Fold alerts under their host, recurring maintenance, run playbooks from alerts, and an Updates page

- Servers can say what they run on (or it is worked out from the Proxmox node); when the host is
  down the guests' offline alerts fold under it: held back, not counted, not notified, still recorded.
- Recurring maintenance schedules made into ordinary windows a week ahead (maintenance_schedules).
- Run a Semaphore template limited to a server from its page or an alert, and watch the run.
- Agents report pending updates (apt, dnf, zypper, Windows Update); an Updates page lists them.
- Migrations 0024 and 0025.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-17 01:35:15 +02:00
1 parent 44e393f172
commit 171511c6db
45 files changed
+6772 -44

No files matched your search

+3
View File
@@ -0,0 +1,3 @@
ALTER TABLE `servers` ADD `parent_server_id` integer;--> statement-breakpoint
ALTER TABLE `servers` ADD `pending_updates` text;--> statement-breakpoint
ALTER TABLE `servers` ADD `updates_pending_since` text;
+13
View File
@@ -0,0 +1,13 @@
CREATE TABLE `maintenance_schedules` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`name` text NOT NULL,
`reason` text,
`targets` text NOT NULL,
`days` text NOT NULL,
`start_time` text NOT NULL,
`duration_minutes` integer NOT NULL,
`timezone` text NOT NULL,
`enabled` integer DEFAULT true NOT NULL,
`created_by` text,
`created_at` text DEFAULT (current_timestamp) NOT NULL
);
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+14
View File
@@ -169,6 +169,20 @@
"when": 1792181083723,
"tag": "0023_premium_cyclops",
"breakpoints": true
},
{
"idx": 24,
"version": "6",
"when": 1792191100783,
"tag": "0024_chubby_jackpot",
"breakpoints": true
},
{
"idx": 25,
"version": "6",
"when": 1792191757544,
"tag": "0025_clean_the_hand",
"breakpoints": true
}
]
}
+30
View File
@@ -282,6 +282,15 @@ export const servers = sqliteTable("servers", {
// card always shows) once a server IS actually linked, so unlinking stays
// reachable.
hideProxmoxLink: integer("hide_proxmox_link", { mode: "boolean" }).notNull().default(false),
// The server this one runs on (a VM on its host), set by an admin. Null means "work it out": a guest linked to a Proxmox node runs on
// the server named after that node. When the parent stops reporting, this server's own offline alert is folded under the parent's.
parentServerId: integer("parent_server_id"),
// What the agent says is waiting to be installed, as JSON: {manager, count, security, rebootRequired, reportedAt}. Null until it reports.
pendingUpdates: text("pending_updates"),
// When updates were first seen waiting (and none were before) — how long the oldest has been waiting. Null when none are.
updatesPendingSince: text("updates_pending_since"),
});
export const scheduledTasks = sqliteTable("scheduled_tasks", {
@@ -505,3 +514,24 @@ export const shareLinks = sqliteTable("share_links", {
expiresAt: text("expires_at"), // ISO; null = never
lastUsedAt: text("last_used_at"), // ISO
});
// ─── Recurring maintenance ──────────────────────────────────────────────────
// "Every Sunday 02:00–04:00 on these servers": a schedule kept here, turned into ordinary maintenance windows a week ahead (each marked
// `schedule:<id>:<date>` in maintenance_windows.source). The windows are what silence alerts; the schedule only says when they come.
export const maintenanceSchedules = sqliteTable("maintenance_schedules", {
id: integer("id").primaryKey({ autoIncrement: true }),
name: text("name").notNull(), // e.g. "Patch night"
reason: text("reason"),
// JSON: {type: "server" | "integration" | "dns_provider", id: number}[] — polymorphic, so no foreign key; a deleted target is skipped.
targets: text("targets").notNull(),
days: text("days").notNull(), // JSON: weekdays it happens on, 0 = Sunday … 6 = Saturday
startTime: text("start_time").notNull(), // "HH:MM" on the clock of `timezone`
durationMinutes: integer("duration_minutes").notNull(),
timezone: text("timezone").notNull(), // IANA name, so "02:00" stays 02:00 across summer and winter time
enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
createdBy: text("created_by"),
createdAt: text("created_at")
.notNull()
.default(sql`(current_timestamp)`),
});
+2
View File
@@ -13,6 +13,7 @@ import { initQuietHoursScheduler } from "./services/quietHoursScheduler.js";
import { initHealthScheduler } from "./services/healthScheduler.js";
import { initAlertTicketScheduler } from "./services/alertTicketScheduler.js";
import { initKumaMaintenanceScheduler } from "./services/kumaMaintenanceScheduler.js";
import { initMaintenanceScheduleScheduler } from "./services/maintenanceScheduleScheduler.js";
{
// Sessions are signed with this secret. Without one, the app would sign them with a value that's in the source code.
@@ -42,6 +43,7 @@ await initQuietHoursScheduler();
await initHealthScheduler();
await initAlertTicketScheduler();
await initKumaMaintenanceScheduler();
await initMaintenanceScheduleScheduler();
createApp().listen(env.port, () => {
console.log(`homelab-manager listening on port ${env.port}`);
+20 -4
View File
@@ -45,12 +45,20 @@ export interface SemaphoreTemplate {
lastTask: SemaphoreTask | null;
}
/** What a run may be told: which hosts of the inventory to run on (ansible's --limit), and a note shown beside the run in Semaphore. */
export interface SemaphoreRunOptions {
limit?: string;
message?: string;
}
export interface SemaphoreAdapter {
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
listTemplatesWithStatus(): Promise<SemaphoreTemplate[]>;
/** Like listTemplatesWithStatus, but says which projects couldn't be read — for callers that must not mistake "couldn't read" for "no templates". */
checkTemplates(): Promise<{ templates: SemaphoreTemplate[]; failedProjectIds: number[] }>;
runTemplate(projectId: number, templateId: number): Promise<SemaphoreTask>;
runTemplate(projectId: number, templateId: number, options?: SemaphoreRunOptions): Promise<SemaphoreTask>;
/** How a run is getting on. */
getTask(projectId: number, taskId: number): Promise<SemaphoreTask>;
}
export function createSemaphoreAdapter(config: SemaphoreConfig): SemaphoreAdapter {
@@ -134,11 +142,19 @@ export function createSemaphoreAdapter(config: SemaphoreConfig): SemaphoreAdapte
return (await checkTemplates()).templates;
}
async function runTemplate(projectId: number, templateId: number): Promise<SemaphoreTask> {
const task = await api("POST", `/project/${projectId}/tasks`, { template_id: templateId });
async function runTemplate(projectId: number, templateId: number, options: SemaphoreRunOptions = {}): Promise<SemaphoreTask> {
const task = await api("POST", `/project/${projectId}/tasks`, {
template_id: templateId,
...(options.limit ? { limit: options.limit } : {}),
...(options.message ? { message: options.message } : {}),
});
return mapTask(task)!;
}
async function getTask(projectId: number, taskId: number): Promise<SemaphoreTask> {
return mapTask(await api("GET", `/project/${projectId}/tasks/${taskId}`))!;
}
async function ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }> {
const start = Date.now();
try {
@@ -149,5 +165,5 @@ export function createSemaphoreAdapter(config: SemaphoreConfig): SemaphoreAdapte
}
}
return withDiagLogging("semaphore", { ping, listTemplatesWithStatus, checkTemplates, runTemplate });
return withDiagLogging("semaphore", { ping, listTemplatesWithStatus, checkTemplates, runTemplate, getTask });
}
+15
View File
@@ -23,7 +23,22 @@ const listeningPortSchema = z.object({
process: text(100).optional(),
});
// What is waiting to be installed. As lenient as the ports: a malformed answer is dropped, never allowed to cost the agent its report.
const updatesSchema = z.object({
manager: text(30).optional(),
count: z.number().int().min(0).max(100_000),
security: z.number().int().min(0).max(100_000).nullable().optional(),
reboot_required: z.boolean().optional(),
});
const systemSchema = z.object({
updates: z
.unknown()
.optional()
.transform((u) => {
const parsed = updatesSchema.safeParse(u);
return parsed.success ? parsed.data : undefined;
}),
ip_addresses: z.array(text(100)).transform(cap(200)).optional(),
cpu: z.object({ model: text(200).optional(), cores: z.number().optional(), load_percent: z.number().nullable().optional() }).optional(),
memory: z.object({ total_bytes: z.number().optional(), used_bytes: z.number().optional() }).optional(),
+34 -3
View File
@@ -633,6 +633,14 @@ integrationsRouter.get("/:id/semaphore/templates", asyncHandler(async (req, res)
}
}));
// Which hosts a run may be limited to: names, addresses and ansible patterns (commas, colons, wildcards) — and nothing that could be
// taken for an option. Semaphore hands the value to ansible as an argument, but it is still only ever shown the characters a host has.
const runBodySchema = z
.object({
limit: z.union([z.literal(""), z.string().trim().regex(/^[A-Za-z0-9][A-Za-z0-9._,:*\-]{0,199}$/, "Use host names or patterns like web01,db*")]).optional(),
})
.default({});
integrationsRouter.post(
"/:id/semaphore/projects/:projectId/templates/:templateId/run",
requireRole("operator"),
@@ -645,16 +653,25 @@ integrationsRouter.post(
if (!Number.isInteger(projectId) || !Number.isInteger(templateId)) {
return res.status(400).json({ error: "invalid_id" });
}
const body = runBodySchema.safeParse(req.body ?? {});
if (!body.success) {
return res.status(400).json({ error: "invalid_body", message: "The hosts to limit it to can only be host names or patterns like web01,db*.", details: body.error.flatten() });
}
try {
const task = await found.adapter.runTemplate(projectId, templateId);
const actor = req.currentUser!;
const limit = body.data.limit || undefined;
const task = await found.adapter.runTemplate(projectId, templateId, {
limit,
message: `Started in Homelab Manager by ${actor.name ?? actor.email ?? actor.oidcSub}${limit ? ` (limited to ${limit})` : ""}`,
});
await recordAudit({
actor: req.currentUser!,
actor,
category: "integration",
action: "run_template",
targetType: "semaphore_template",
targetId: templateId,
detail: { integrationId: found.integration.id, projectId, taskId: task.id },
detail: { integrationId: found.integration.id, projectId, taskId: task.id, ...(limit ? { limit } : {}) },
});
res.status(201).json({ task });
} catch (err) {
@@ -663,6 +680,20 @@ integrationsRouter.post(
}),
);
// How a run is getting on: waiting, running, success or error.
integrationsRouter.get("/:id/semaphore/projects/:projectId/tasks/:taskId", asyncHandler(async (req, res) => {
const found = await requireSemaphoreAdapter(req, res);
if (!found) return;
const projectId = Number(req.params.projectId);
const taskId = Number(req.params.taskId);
if (!Number.isInteger(projectId) || !Number.isInteger(taskId)) return res.status(400).json({ error: "invalid_id" });
try {
res.json({ task: await found.adapter.getTask(projectId, taskId) });
} catch (err) {
res.status(502).json({ error: err instanceof Error ? err.message : String(err) });
}
}));
// ─── Proxmox ─────────────────────────────────────────────────────────────────
async function requireProxmoxAdapter(req: Request, res: Response) {
+101 -1
View File
@@ -2,7 +2,8 @@ import { Router } from "express";
import { and, eq, inArray } from "drizzle-orm";
import { z } from "zod";
import { db } from "../db/client.js";
import { dnsProviders, integrations, maintenanceAliases, maintenanceTargetTypes, maintenanceWindows, servers } from "../db/schema.js";
import { dnsProviders, integrations, maintenanceAliases, maintenanceSchedules, maintenanceTargetTypes, maintenanceWindows, servers } from "../db/schema.js";
import { deleteSchedule, isValidTimeZone, materializeSchedules, occurrencesBetween, removeFutureWindows, targetsOf } from "../services/maintenanceSchedules.js";
import { hostOfAddress } from "../services/kumaTargets.js";
import { invalidateAlertsCache } from "../services/alerts.js";
import { requireAuth, requireRole } from "../auth/middleware.js";
@@ -268,6 +269,105 @@ maintenanceRouter.post("/end-all", requireRole("operator"), asyncHandler(async (
res.json({ ended: active.length });
}));
// ─── Recurring schedules ────────────────────────────────────────────────────
const scheduleSchema = z.object({
name: z.string().trim().min(1).max(80),
reason: z.string().trim().max(200).optional(),
targets: z.array(z.object({ type: z.enum(maintenanceTargetTypes), id: z.number().int() })).min(1).max(100),
days: z.array(z.number().int().min(0).max(6)).min(1).max(7),
startTime: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/, "Use a time like 02:00"),
durationMinutes: z.number().int().min(15).max(3 * 24 * 60),
timezone: z.string().min(1).max(64),
});
maintenanceRouter.get("/schedules", asyncHandler(async (_req, res) => {
const rows = await db.select().from(maintenanceSchedules).orderBy(maintenanceSchedules.name);
const now = new Date();
const out = [];
for (const s of rows) {
const targets = [];
for (const t of targetsOf(s)) {
const d = await describeTarget(t.type, t.id);
if (d) targets.push({ type: t.type, id: t.id, name: d.name, kind: d.kind });
}
const next = s.enabled ? occurrencesBetween(s, now, new Date(now.getTime() + 15 * 24 * 3600_000))[0] : undefined;
out.push({
id: s.id,
name: s.name,
reason: s.reason,
days: JSON.parse(s.days) as number[],
startTime: s.startTime,
durationMinutes: s.durationMinutes,
timezone: s.timezone,
enabled: s.enabled,
createdBy: s.createdBy,
targets,
nextStart: next ? next.start.toISOString() : null,
});
}
res.json({ schedules: out });
}));
maintenanceRouter.post("/schedules", requireRole("operator"), asyncHandler(async (req, res) => {
const parsed = scheduleSchema.safeParse(req.body);
if (!parsed.success) {
return res.status(400).json({ error: "invalid_body", message: "Name it, choose what it covers, the days, a start time and how long.", details: parsed.error.flatten() });
}
const data = parsed.data;
if (!isValidTimeZone(data.timezone)) return res.status(400).json({ error: "invalid_timezone", message: "That isn't a time zone this server knows." });
// Only things that exist, once each.
const seen = new Set<string>();
const targets = [];
for (const t of data.targets) {
const key = `${t.type}:${t.id}`;
if (seen.has(key)) continue;
seen.add(key);
if (!(await describeTarget(t.type, t.id))) return res.status(400).json({ error: "unknown_target", message: "One of the things chosen doesn't exist any more." });
targets.push(t);
}
const actor = req.currentUser!;
const [created] = await db
.insert(maintenanceSchedules)
.values({
name: data.name,
reason: data.reason || null,
targets: JSON.stringify(targets),
days: JSON.stringify([...new Set(data.days)].sort()),
startTime: data.startTime,
durationMinutes: data.durationMinutes,
timezone: data.timezone,
createdBy: actor.name ?? actor.email ?? actor.oidcSub,
})
.returning();
const made = await materializeSchedules(new Date(), undefined, created.id);
await recordAudit({ actor, category: "maintenance", action: "create_schedule", targetType: "maintenance_schedule", targetId: created.id, detail: { name: created.name, targets: targets.length, days: data.days, startTime: data.startTime, durationMinutes: data.durationMinutes } });
res.status(201).json({ id: created.id, windowsMade: made });
}));
maintenanceRouter.patch("/schedules/:id", requireRole("operator"), asyncHandler(async (req, res) => {
const id = Number(req.params.id);
const parsed = z.object({ enabled: z.boolean() }).safeParse(req.body);
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body" });
const [row] = await db.select().from(maintenanceSchedules).where(eq(maintenanceSchedules.id, id)).limit(1);
if (!row) return res.status(404).json({ error: "not_found" });
await db.update(maintenanceSchedules).set({ enabled: parsed.data.enabled }).where(eq(maintenanceSchedules.id, id));
if (parsed.data.enabled) await materializeSchedules(new Date(), undefined, id);
else await removeFutureWindows(id);
await recordAudit({ actor: req.currentUser!, category: "maintenance", action: parsed.data.enabled ? "enable_schedule" : "disable_schedule", targetType: "maintenance_schedule", targetId: id, detail: { name: row.name } });
res.json({ enabled: parsed.data.enabled });
}));
maintenanceRouter.delete("/schedules/:id", requireRole("operator"), asyncHandler(async (req, res) => {
const id = Number(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [row] = await db.select().from(maintenanceSchedules).where(eq(maintenanceSchedules.id, id)).limit(1);
if (!row) return res.status(404).json({ error: "not_found" });
await deleteSchedule(id);
await recordAudit({ actor: req.currentUser!, category: "maintenance", action: "delete_schedule", targetType: "maintenance_schedule", targetId: id, detail: { name: row.name } });
res.status(204).end();
}));
const syncUptimeKumaSchema = z.object({ integrationId: z.number().int().positive() });
/** Reads Uptime Kuma's maintenance schedule now and brings the mirrored windows in line with it (the same pass that runs every 15 minutes when switched on). */
+55 -5
View File
@@ -6,6 +6,8 @@ import { servers, serverLinks, integrations, dnsRecordsCache, dnsZonesCache, dns
import { requireAuth, requireRole } from "../auth/middleware.js";
import { generateApiToken } from "../services/tokens.js";
import { recordAudit } from "../services/audit.js";
import { loadParentRows, resolveParents, wouldLoop } from "../services/serverParents.js";
import { parsePendingUpdates } from "../services/pendingUpdates.js";
import { getServerMetrics, RETENTION_DAYS } from "../services/serverMetrics.js";
import { getServerActivity } from "../services/serverActivity.js";
import { asyncHandler } from "../utils/asyncHandler.js";
@@ -21,9 +23,9 @@ import { getSettings } from "../services/settingsStore.js";
export const serversRouter = Router();
/** A server row as the API returns it: no token hash, and tags as an array rather than the stored JSON. */
function publicServer<T extends { apiTokenHash: string; tags: string | null }>(row: T) {
const { apiTokenHash: _hash, tags, ...rest } = row;
return { ...rest, tags: parseTags(tags) };
function publicServer<T extends { apiTokenHash: string; tags: string | null; pendingUpdates: string | null }>(row: T) {
const { apiTokenHash: _hash, tags, pendingUpdates, ...rest } = row;
return { ...rest, tags: parseTags(tags), pendingUpdates: parsePendingUpdates(pendingUpdates) };
}
serversRouter.use(requireAuth);
@@ -113,6 +115,8 @@ const updateServerSchema = z
proxmoxGuestType: z.enum(["qemu", "lxc"]).nullable().optional(),
proxmoxVmid: z.number().int().nullable().optional(),
hideProxmoxLink: z.boolean().optional(),
// The server this one runs on. Null goes back to working it out from the Proxmox node.
parentServerId: z.number().int().nullable().optional(),
})
.refine(
(data) => {
@@ -148,6 +152,17 @@ serversRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req, res)
}
}
if (parsed.data.parentServerId !== undefined && parsed.data.parentServerId !== null) {
const parentId = parsed.data.parentServerId;
const rows = await loadParentRows();
if (parentId === id || !rows.some((r) => r.id === parentId)) {
return res.status(400).json({ error: "invalid_parent", message: "Choose another server that exists." });
}
if (wouldLoop(rows, id, parentId)) {
return res.status(400).json({ error: "parent_loop", message: "That server already runs on this one, so each would be waiting for the other." });
}
}
const [updated] = await db.update(servers).set(parsed.data).where(eq(servers.id, id)).returning();
await recordAudit({
@@ -156,7 +171,7 @@ serversRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req, res)
action: "update",
targetType: "server",
targetId: id,
detail: { name: updated.name },
detail: { name: updated.name, ...(parsed.data.parentServerId !== undefined ? { parentServerId: parsed.data.parentServerId } : {}) },
});
const serverOut = publicServer(updated);
@@ -173,6 +188,27 @@ serversRouter.get("/summary", asyncHandler(async (_req, res) => {
res.json(buildServerSummary(rows, healthChecks, Date.now(), await activeSubjects()));
}));
// What every server that reports through an agent says is waiting to be installed, for the Updates page. Servers whose agent hasn't
// said (it predates this, or couldn't find out) are listed with no figures rather than left out.
serversRouter.get("/updates", asyncHandler(async (_req, res) => {
const rows = await db
.select({
id: servers.id,
name: servers.name,
osType: servers.osType,
lastSeenAt: servers.lastSeenAt,
pendingUpdates: servers.pendingUpdates,
updatesPendingSince: servers.updatesPendingSince,
})
.from(servers)
.orderBy(servers.name);
res.json({
servers: rows
.filter((r) => r.lastSeenAt !== null)
.map((r) => ({ id: r.id, name: r.name, osType: r.osType, lastSeenAt: r.lastSeenAt, updates: parsePendingUpdates(r.pendingUpdates), pendingSince: r.updatesPendingSince })),
});
}));
// For the Operations > Admin Links page — every server's admin bookmarks in one place, instead of visiting
// each server's own detail page to find them.
serversRouter.get("/links", asyncHandler(async (_req, res) => {
@@ -306,10 +342,21 @@ serversRouter.get("/:id/detail", asyncHandler(async (req, res) => {
memUsedBytes: _memUsedBytes,
listeningPorts: _listeningPorts,
lastPortScan: _lastPortScan,
pendingUpdates: rawUpdates,
...serverOut
} = server;
res.json({ server: { ...serverOut, tags: parseTags(rawTags) }, ipAddresses, hardware, dnsMatches, links });
// What this server runs on, and what runs on it (set by hand, or worked out from its Proxmox node).
const parentRows = await loadParentRows();
const links2 = resolveParents(parentRows);
const nameOf = new Map(parentRows.map((r) => [r.id, r.name]));
const parentLink = links2.get(id);
const relations = {
parent: parentLink ? { id: parentLink.parentId, name: nameOf.get(parentLink.parentId) ?? "", auto: parentLink.auto } : null,
children: [...links2.entries()].filter(([, l]) => l.parentId === id).map(([childId, l]) => ({ id: childId, name: nameOf.get(childId) ?? "", auto: l.auto })).sort((a, b) => a.name.localeCompare(b.name)),
};
res.json({ server: { ...serverOut, tags: parseTags(rawTags), pendingUpdates: parsePendingUpdates(rawUpdates) }, ipAddresses, hardware, dnsMatches, links, relations });
}));
const tagsSchema = z.object({ tags: z.array(z.string().max(100)).max(50) });
@@ -440,6 +487,9 @@ serversRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req, res)
const id = Number(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
// Servers that ran on this one go back to working out where they run (the link is a plain column, not a foreign key).
await db.update(servers).set({ parentServerId: null }).where(eq(servers.parentServerId, id));
const deleted = await db.delete(servers).where(eq(servers.id, id)).returning();
if (deleted.length === 0) return res.status(404).json({ error: "not_found" });
+4
View File
@@ -18,6 +18,10 @@ export interface Alert {
link: string | null;
/** Under an active maintenance window: still a real problem, but notifications for it are held back. */
silenced: boolean;
/** Set when this problem is explained by another one (a guest down because its host is): it is held back like a silenced one, but the reason is this, not maintenance. */
causedBy?: { name: string };
/** On a host's alert: the servers on it whose own alerts are folded under this one. */
affects?: string[];
/** What a maintenance window would have to be on to silence it, when there is such a thing — which is what "Silence" acts on. */
target: { type: "server" | "integration" | "dns_provider"; id: number; name: string } | null;
}
+24 -5
View File
@@ -28,12 +28,13 @@ import { computeSecretStatus } from "./secretStatus.js";
import { getFailingSources } from "./integrationHealthMonitor.js";
import { getSettings } from "./settingsStore.js";
import { sourceLabel } from "./notify.js";
import { coveredByParent, loadParentRows, resolveParents } from "./serverParents.js";
import type { Alert, AlertCategory, AlertSeverity, SourceFailure } from "./alertTypes.js";
export interface AlertsReport {
alerts: Alert[];
/** Active problems by severity — those under a maintenance window are counted apart, not in these. */
counts: { critical: number; warning: number; info: number; silenced: number };
counts: { critical: number; warning: number; info: number; silenced: number; /** Held back because another problem explains them (a guest down because its host is). */ grouped: number };
/** Things that couldn't be checked, and which checks that leaves blind. */
couldntCheck: { name: string; error: string; affects: string[] }[];
/** Context worth knowing about how complete the picture is. */
@@ -115,7 +116,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
return row ? integrationTarget(row.id) : null;
};
function push(a: { category: AlertCategory; severity: AlertSeverity; type: string; message: string; key: string; link?: string | null; silenced?: boolean; target?: Alert["target"] }) {
function push(a: { category: AlertCategory; severity: AlertSeverity; type: string; message: string; key: string; link?: string | null; silenced?: boolean; target?: Alert["target"]; causedBy?: Alert["causedBy"]; affects?: string[] }) {
const meta = SOURCES[a.type];
alerts.push({
id: `${a.category}:${a.key}`,
@@ -124,8 +125,10 @@ export async function collectAlerts(): Promise<AlertsReport> {
source: meta?.label ?? sourceLabel(a.type),
message: a.message,
link: a.link === undefined ? (meta?.link ?? null) : a.link,
silenced: !!a.silenced,
silenced: !!a.silenced || !!a.causedBy,
target: a.target ?? null,
...(a.causedBy ? { causedBy: a.causedBy } : {}),
...(a.affects?.length ? { affects: a.affects } : {}),
});
}
@@ -160,10 +163,24 @@ export async function collectAlerts(): Promise<AlertsReport> {
`Server-offline and server-disk checks are paused for another ${Math.ceil(grace / 60_000)} min after the app restarted, so agents get a chance to report before any server is judged.`,
);
}
for (const c of evaluateHealth(snapshot, healthChecks, now, { skipServers: grace > 0 })) {
const conditions = evaluateHealth(snapshot, healthChecks, now, { skipServers: grace > 0 });
// A server that is down because the one it runs on is down is folded under that one.
const offlineIds = new Set<number>();
for (const c of conditions) {
const m = /^offline:server:(\d+)$/.exec(c.key);
if (m) offlineIds.add(Number(m[1]));
}
const cause = coveredByParent(offlineIds, resolveParents(await loadParentRows()));
const taken = new Map<number, string[]>();
for (const [child, parent] of cause) taken.set(parent, [...(taken.get(parent) ?? []), serverNames.get(child) ?? `#${child}`]);
for (const c of conditions) {
const origin = originOfConditionKey(c.key);
const subject = subjectOfConditionKey(c.key);
const offlineId = /^offline:server:(\d+)$/.exec(c.key)?.[1];
const parentOf = offlineId ? cause.get(Number(offlineId)) : undefined;
push({
causedBy: parentOf !== undefined ? { name: serverNames.get(parentOf) ?? `#${parentOf}` } : undefined,
affects: offlineId ? taken.get(Number(offlineId))?.sort((a, b) => a.localeCompare(b)) : undefined,
category: origin.category,
severity: c.severity ?? "warning",
type: origin.type,
@@ -403,13 +420,15 @@ export async function collectAlerts(): Promise<AlertsReport> {
);
const active = alerts.filter((a) => !a.silenced);
const grouped = alerts.filter((a) => a.causedBy).length;
return {
alerts,
counts: {
critical: active.filter((a) => a.severity === "critical").length,
warning: active.filter((a) => a.severity === "warning").length,
info: active.filter((a) => a.severity === "info").length,
silenced: alerts.length - active.length,
silenced: alerts.length - active.length - grouped,
grouped,
},
couldntCheck: [...blind.entries()].map(([name, v]) => ({ name, error: v.error, affects: [...v.affects] })),
notes,
+54 -9
View File
@@ -8,10 +8,11 @@ import { notifyHealthIssues, notifyHealthRecovered } from "./notify.js";
import { recordAudit } from "./audit.js";
import { getInternalFlag, getSettings, setInternalFlag } from "./settingsStore.js";
import { activeSubjects, subjectOfConditionKey } from "./maintenance.js";
import { coveredByParent, loadParentRows, resolveParents } from "./serverParents.js";
const STATE_FLAG = "healthActiveConditions";
/** After a restart the agents haven't had a chance to report yet (they run every 15 min), so server-derived conditions are held rather than judged. */
const STARTUP_GRACE_MS = 20 * 60 * 1000;
const STARTUP_GRACE_MS = Number(process.env.HEALTH_STARTUP_GRACE_MS ?? 20 * 60 * 1000); // the environment variable is for tests
const PROCESS_START = Date.now();
// ─── Types ──────────────────────────────────────────────────────────────────
@@ -212,7 +213,8 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
// ─── State diff (pure) ──────────────────────────────────────────────────────
export type ActiveState = Record<string, { message: string; source: string }>;
/** `covered`: the problem is explained by another that is being reported (a guest down because its host is) — recorded, but not announced on its own. */
export type ActiveState = Record<string, { message: string; source: string; covered?: boolean }>;
/** `held` is a set of hierarchical sources that couldn't be read this run; matches the source itself or anything beneath it. */
function isHeld(source: string, held: Set<string>): boolean {
@@ -226,6 +228,10 @@ function isHeld(source: string, held: Set<string>): boolean {
* now is resolved — except when its source couldn't be read this run, in
* which case it's carried over untouched. Without that, one failed poll would
* report a recovery and then re-alert the same problem on the next poll.
*
* `covered` are keys of problems explained by another (a guest that is down because its host is). They are kept in the state and
* counted in `newlyDown` / `ended` (what happened, for the record), but are neither announced as new nor as resolved — until the
* explanation goes away while they are still there, when they are announced as new: an outage is never swallowed.
*/
export function diffConditions(
previous: ActiveState,
@@ -233,7 +239,16 @@ export function diffConditions(
held: Set<string> = new Set(),
/** Subjects ("server:3", "integration:1") under an active maintenance window. */
silenced: Set<string> = new Set(),
): { added: HealthCondition[]; resolved: { key: string; message: string }[]; next: ActiveState } {
covered: Set<string> = new Set(),
): {
added: HealthCondition[];
resolved: { key: string; message: string }[];
next: ActiveState;
/** Every problem that began, announced or not. */
newlyDown: HealthCondition[];
/** Every problem that ended, announced or not. */
ended: { key: string; message: string }[];
} {
const isSilenced = (key: string) => {
const subject = subjectOfConditionKey(key);
return subject !== null && silenced.has(subject);
@@ -243,20 +258,24 @@ export function diffConditions(
const visible = current.filter((c) => !isSilenced(c.key));
const next: ActiveState = {};
for (const c of visible) next[c.key] = { message: c.message, source: c.source };
for (const c of visible) next[c.key] = covered.has(c.key) ? { message: c.message, source: c.source, covered: true } : { message: c.message, source: c.source };
const added = visible.filter((c) => !(c.key in previous));
const newlyDown = visible.filter((c) => !(c.key in previous));
// Announced: new and standing alone, or covered until now and no longer.
const added = visible.filter((c) => !covered.has(c.key) && (!(c.key in previous) || previous[c.key].covered));
const resolved: { key: string; message: string }[] = [];
const ended: { key: string; message: string }[] = [];
for (const [key, entry] of Object.entries(previous)) {
if (key in next) continue;
// Held (source unreadable) or silenced: leave it exactly as it was — neither cleared nor re-alerted.
if (isHeld(entry.source, held) || isSilenced(key)) {
next[key] = entry;
} else {
resolved.push({ key, message: entry.message });
ended.push({ key, message: entry.message });
if (!entry.covered) resolved.push({ key, message: entry.message });
}
}
return { added, resolved, next };
return { added, resolved, next, newlyDown, ended };
}
// ─── Collection (I/O) ───────────────────────────────────────────────────────
@@ -346,6 +365,14 @@ export async function recordServerTransitions(added: { key: string; message: str
await write(resolved, { offline: "back_online", disk: "disk_ok" }, false);
}
/** "a, b and c", "a, b, c, d, e and 3 more". */
export function alsoList(names: string[]): string {
const shown = names.slice(0, 5);
const rest = names.length - shown.length;
const head = shown.length > 1 ? `${shown.slice(0, -1).join(", ")} and ${shown[shown.length - 1]}` : shown[0];
return rest > 0 ? `${shown.join(", ")} and ${rest} more` : head;
}
export async function runHealthCheck(now: number = Date.now()): Promise<{ added: number; resolved: number }> {
const { healthChecks } = await getSettings();
const { snapshot, held } = await collectSnapshot();
@@ -354,12 +381,30 @@ export async function runHealthCheck(now: number = Date.now()): Promise<{ added:
if (inGrace) held.add("server");
const current = evaluateHealth(snapshot, healthChecks, now, { skipServers: inGrace });
const { added, resolved, next } = diffConditions(await loadState(), current, held, await activeSubjects(new Date(now)));
// A server that is down because the one it runs on is down is folded under that one: recorded, but not announced on its own.
const offlineIds = new Set<number>();
for (const c of current) {
const m = /^offline:server:(\d+)$/.exec(c.key);
if (m) offlineIds.add(Number(m[1]));
}
const cause = coveredByParent(offlineIds, resolveParents(await loadParentRows()));
const coveredKeys = new Set([...cause.keys()].map((id) => `offline:server:${id}`));
const { added: announced, resolved, next, newlyDown, ended } = diffConditions(await loadState(), current, held, await activeSubjects(new Date(now)), coveredKeys);
await setInternalFlag(STATE_FLAG, JSON.stringify(next));
// A host's message says what else it takes with it.
const names = new Map(snapshot.servers.map((s) => [s.id, s.name]));
const taken = new Map<number, string[]>();
for (const [child, parent] of cause) taken.set(parent, [...(taken.get(parent) ?? []), names.get(child) ?? `#${child}`]);
const added = announced.map((c) => {
const m = /^offline:server:(\d+)$/.exec(c.key);
const children = m ? taken.get(Number(m[1])) : undefined;
return children?.length ? { ...c, message: `${c.message} — also takes down ${alsoList(children)}` } : c;
});
// State is tracked even when the alert toggle is off (the notify functions check it),
// so turning alerts back on doesn't dump every long-standing condition at once.
await recordServerTransitions(added, resolved);
await recordServerTransitions(newlyDown, ended);
if (added.length > 0) await notifyHealthIssues(added);
if (resolved.length > 0) await notifyHealthRecovered(resolved);
return { added: added.length, resolved: resolved.length };
@@ -0,0 +1,24 @@
import { materializeSchedules } from "./maintenanceSchedules.js";
// Windows are made a week ahead, so a pass is only needed now and then: to make the next day's, and to pick up a server added to a
// schedule's list. Quarter-hourly keeps a restart from leaving a gap.
const INTERVAL_MS = 15 * 60 * 1000;
const FIRST_PASS_DELAY_MS = 30 * 1000;
let timer: ReturnType<typeof setInterval> | null = null;
async function pass(): Promise<void> {
try {
const made = await materializeSchedules();
if (made > 0) console.log(`[maintenanceSchedules] made ${made} maintenance window${made === 1 ? "" : "s"} from recurring schedules`);
} catch (err) {
console.error("[maintenanceSchedules]", err instanceof Error ? err.message : err);
}
}
export async function initMaintenanceScheduleScheduler(): Promise<void> {
if (timer) return;
timer = setInterval(() => void pass(), INTERVAL_MS);
setTimeout(() => void pass(), FIRST_PASS_DELAY_MS);
console.log(`[maintenanceSchedules] making windows from recurring schedules a week ahead, checked every ${INTERVAL_MS / 60000} minutes`);
}
+145
View File
@@ -0,0 +1,145 @@
/**
* Recurring maintenance: "every Sunday 02:00–04:00 on these servers". A schedule only says when; the windows that silence alerts are
* ordinary maintenance windows made a week ahead (so they show on the timeline and can be cancelled one at a time, like any other).
* Each is marked `schedule:<id>:<date>`, which is how a later pass knows it has already made it — even if someone cancelled it.
*/
import { and, eq, gt, like } from "drizzle-orm";
import { db } from "../db/client.js";
import { maintenanceSchedules, maintenanceWindows, type MaintenanceTargetType } from "../db/schema.js";
import { invalidateAlertsCache } from "./alerts.js";
import { describeTarget } from "./maintenance.js";
export const HORIZON_DAYS = 7;
const DAY_MS = 24 * 3600_000;
export type ScheduleRow = typeof maintenanceSchedules.$inferSelect;
export interface ScheduleTarget {
type: MaintenanceTargetType;
id: number;
}
export interface Occurrence {
start: Date;
end: Date;
/** The local calendar date it starts on, "2026-10-18" — unique per schedule. */
date: string;
}
/** Whether the system knows this time zone name. */
export function isValidTimeZone(tz: string): boolean {
try {
new Intl.DateTimeFormat("en-US", { timeZone: tz });
return true;
} catch {
return false;
}
}
const parts = (utcMs: number, tz: string) => {
const f = new Intl.DateTimeFormat("en-US", { timeZone: tz, hourCycle: "h23", year: "numeric", month: "2-digit", day: "2-digit", hour: "2-digit", minute: "2-digit", second: "2-digit" });
const out: Record<string, number> = {};
for (const p of f.formatToParts(new Date(utcMs))) if (p.type !== "literal") out[p.type] = Number(p.value);
return out;
};
/** How far the clock in `tz` is ahead of UTC at that moment, in ms. */
function offsetAt(utcMs: number, tz: string): number {
const p = parts(utcMs, tz);
return Date.UTC(p.year, p.month - 1, p.day, p.hour, p.minute, p.second) - Math.floor(utcMs / 1000) * 1000;
}
/** The instant a wall-clock time in `tz` happens. Winter/summer time is settled by asking the zone for its offset at the answer. */
export function zonedTime(year: number, month: number, day: number, hour: number, minute: number, tz: string): Date {
const naive = Date.UTC(year, month - 1, day, hour, minute);
let guess = naive - offsetAt(naive, tz);
guess = naive - offsetAt(guess, tz);
return new Date(guess);
}
const pad = (n: number) => String(n).padStart(2, "0");
/**
* The times a schedule happens that touch [from, to]: those that have not ended by `from` and begin before `to`. A schedule that runs
* past midnight belongs to the day it starts on.
*/
export function occurrencesBetween(schedule: Pick<ScheduleRow, "days" | "startTime" | "durationMinutes" | "timezone">, from: Date, to: Date): Occurrence[] {
const days = new Set<number>(JSON.parse(schedule.days) as number[]);
const [hh, mm] = schedule.startTime.split(":").map(Number);
const out: Occurrence[] = [];
// Walk the calendar dates around the range (a day either side covers a schedule that started yesterday, or zone differences).
const first = parts(from.getTime() - DAY_MS, schedule.timezone);
let cursor = Date.UTC(first.year, first.month - 1, first.day);
const last = to.getTime() + DAY_MS;
for (; cursor <= last; cursor += DAY_MS) {
const d = new Date(cursor);
if (!days.has(d.getUTCDay())) continue;
const y = d.getUTCFullYear();
const m = d.getUTCMonth() + 1;
const day = d.getUTCDate();
const start = zonedTime(y, m, day, hh, mm, schedule.timezone);
const end = new Date(start.getTime() + schedule.durationMinutes * 60_000);
if (end > from && start < to) out.push({ start, end, date: `${y}-${pad(m)}-${pad(day)}` });
}
return out;
}
export function targetsOf(schedule: Pick<ScheduleRow, "targets">): ScheduleTarget[] {
try {
const v = JSON.parse(schedule.targets);
return Array.isArray(v) ? v.filter((t): t is ScheduleTarget => !!t && typeof t.id === "number" && typeof t.type === "string") : [];
} catch {
return [];
}
}
export const sourceOf = (scheduleId: number, date: string) => `schedule:${scheduleId}:${date}`;
/**
* Makes the windows for the coming week that don't exist yet. Safe to run as often as you like: a window already made (including one
* that was cancelled) is left as it is. Returns how many it made.
*/
export async function materializeSchedules(now: Date = new Date(), horizonDays: number = HORIZON_DAYS, onlyScheduleId?: number): Promise<number> {
const schedules = (await db.select().from(maintenanceSchedules)).filter((s) => s.enabled && (onlyScheduleId === undefined || s.id === onlyScheduleId));
const horizon = new Date(now.getTime() + horizonDays * DAY_MS);
let made = 0;
for (const schedule of schedules) {
const occurrences = occurrencesBetween(schedule, now, horizon);
if (occurrences.length === 0) continue;
const existing = await db.select({ source: maintenanceWindows.source, targetType: maintenanceWindows.targetType, targetId: maintenanceWindows.targetId }).from(maintenanceWindows).where(like(maintenanceWindows.source, `schedule:${schedule.id}:%`));
const have = new Set(existing.map((w) => `${w.source}|${w.targetType}:${w.targetId}`));
for (const target of targetsOf(schedule)) {
if (!(await describeTarget(target.type, target.id))) continue; // since removed: nothing left to silence
for (const occ of occurrences) {
const source = sourceOf(schedule.id, occ.date);
if (have.has(`${source}|${target.type}:${target.id}`)) continue;
await db.insert(maintenanceWindows).values({
targetType: target.type,
targetId: target.id,
reason: schedule.reason ? `${schedule.name}: ${schedule.reason}` : schedule.name,
startedAt: occ.start.toISOString(),
endsAt: occ.end.toISOString(),
createdBy: `Recurring: ${schedule.name}`,
source,
});
made++;
}
}
}
if (made > 0) invalidateAlertsCache();
return made;
}
/** Takes back the windows a schedule made that haven't begun — when it is switched off or removed. Ones under way end on their own. */
export async function removeFutureWindows(scheduleId: number, now: Date = new Date()): Promise<number> {
const gone = await db
.delete(maintenanceWindows)
.where(and(like(maintenanceWindows.source, `schedule:${scheduleId}:%`), gt(maintenanceWindows.startedAt, now.toISOString())))
.returning({ id: maintenanceWindows.id });
if (gone.length > 0) invalidateAlertsCache();
return gone.length;
}
export async function deleteSchedule(scheduleId: number): Promise<void> {
await removeFutureWindows(scheduleId);
await db.delete(maintenanceSchedules).where(eq(maintenanceSchedules.id, scheduleId));
}
+62
View File
@@ -0,0 +1,62 @@
/**
* What each server says is waiting to be installed: how many updates, how many of them security fixes, whether a restart is due — and
* how long updates have been waiting, which the agent can't know and so is worked out here from one report to the next.
*/
/** What an agent reports about updates, as it arrives. */
export interface IncomingUpdates {
/** apt, dnf, zypper, windows-update … — what it asked. */
manager?: string;
count: number;
/** Null when this package manager can't say which are security fixes. */
security?: number | null;
reboot_required?: boolean;
}
export interface PendingUpdates {
manager: string;
count: number;
security: number | null;
rebootRequired: boolean;
/** ISO — when the agent last told us. */
reportedAt: string;
}
export function parsePendingUpdates(stored: string | null): PendingUpdates | null {
if (!stored) return null;
try {
const v = JSON.parse(stored);
if (!v || typeof v !== "object" || typeof v.count !== "number") return null;
return {
manager: typeof v.manager === "string" ? v.manager : "",
count: v.count,
security: typeof v.security === "number" ? v.security : null,
rebootRequired: v.rebootRequired === true,
reportedAt: typeof v.reportedAt === "string" ? v.reportedAt : "",
};
} catch {
return null;
}
}
/**
* The columns to store after a report. Updates waiting since "since" keep that moment for as long as some are still waiting; when
* none are, it is cleared, and the next time some appear it starts again from then.
*/
export function nextUpdateState(
previous: { pendingUpdates: string | null; updatesPendingSince: string | null },
incoming: IncomingUpdates,
now: string,
): { pendingUpdates: string; updatesPendingSince: string | null } {
const before = parsePendingUpdates(previous.pendingUpdates);
const waiting = incoming.count > 0;
const since = waiting ? (before && before.count > 0 && previous.updatesPendingSince ? previous.updatesPendingSince : now) : null;
const record: PendingUpdates = {
manager: incoming.manager ?? "",
count: incoming.count,
security: typeof incoming.security === "number" ? incoming.security : null,
rebootRequired: incoming.reboot_required === true,
reportedAt: now,
};
return { pendingUpdates: JSON.stringify(record), updatesPendingSince: since };
}
+91
View File
@@ -0,0 +1,91 @@
/**
* Which server runs on which: a virtual machine on its Proxmox host, a container on a Docker box. When the host stops reporting, the
* things on it stop too, and seven alerts about seven guests only hide the one that matters — so those alerts are folded under the
* host's. The link is either set by hand ("runs on") or worked out: a server linked to a Proxmox guest runs on the server named after
* that guest's Proxmox node, when there is exactly one such server.
*/
import { db } from "../db/client.js";
import { servers } from "../db/schema.js";
export interface ParentRow {
id: number;
name: string;
hostname: string | null;
parentServerId: number | null;
proxmoxNode: string | null;
}
export interface ParentLink {
parentId: number;
/** Worked out from the Proxmox node rather than set by hand. */
auto: boolean;
}
const lc = (s: string | null | undefined) => (s ?? "").trim().toLowerCase();
/** Whether a server is the machine a Proxmox node name refers to: by its name, its hostname, or the first part of a fully qualified hostname. */
function isNode(row: ParentRow, node: string): boolean {
const n = lc(node);
if (!n) return false;
const host = lc(row.hostname);
return lc(row.name) === n || host === n || host.split(".")[0] === n;
}
/** Each server's parent, where it has one. A link set by hand wins; otherwise the unique server that is its Proxmox node. */
export function resolveParents(rows: ParentRow[]): Map<number, ParentLink> {
const byId = new Map(rows.map((r) => [r.id, r]));
const out = new Map<number, ParentLink>();
for (const row of rows) {
if (row.parentServerId !== null && row.parentServerId !== row.id && byId.has(row.parentServerId)) {
out.set(row.id, { parentId: row.parentServerId, auto: false });
continue;
}
if (row.proxmoxNode) {
const candidates = rows.filter((r) => r.id !== row.id && isNode(r, row.proxmoxNode!));
if (candidates.length === 1) out.set(row.id, { parentId: candidates[0].id, auto: true });
}
}
return out;
}
/**
* Of the servers that are down, those whose cause is another server that is down too: each maps to the topmost ancestor that is down —
* the root cause. A chain (guest → VM host → the box it hangs off) is followed all the way up; a loop in the links covers nothing.
*/
export function coveredByParent(offline: Set<number>, parents: Map<number, ParentLink>): Map<number, number> {
const covered = new Map<number, number>();
for (const id of offline) {
const seen = new Set<number>([id]);
let root: number | null = null;
let looped = false;
let link = parents.get(id);
while (link && !seen.has(link.parentId)) {
if (offline.has(link.parentId)) root = link.parentId;
seen.add(link.parentId);
link = parents.get(link.parentId);
}
if (link && seen.has(link.parentId)) looped = true;
// A loop explains nothing (each would be waiting for the other), so its members stand alone and are announced.
if (root !== null && !looped) covered.set(id, root);
}
return covered;
}
/** Would making `parentId` the parent of `id` close a loop? True if `id` is `parentId` or already sits above it. */
export function wouldLoop(rows: ParentRow[], id: number, parentId: number): boolean {
const parents = resolveParents(rows);
const seen = new Set<number>();
let cur: number | undefined = parentId;
while (cur !== undefined && !seen.has(cur)) {
if (cur === id) return true;
seen.add(cur);
cur = parents.get(cur)?.parentId;
}
return false;
}
export async function loadParentRows(): Promise<ParentRow[]> {
return db
.select({ id: servers.id, name: servers.name, hostname: servers.hostname, parentServerId: servers.parentServerId, proxmoxNode: servers.proxmoxNode })
.from(servers);
}
+6
View File
@@ -2,6 +2,7 @@ import { and, eq, notInArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { scheduledTasks, servers } from "../db/schema.js";
import { recordServerMetrics } from "./serverMetrics.js";
import { nextUpdateState, type IncomingUpdates } from "./pendingUpdates.js";
export interface IncomingTask {
scheduleType: "cron" | "systemd_timer" | "windows_task";
@@ -20,6 +21,8 @@ export interface IncomingSystemInfo {
memory?: { total_bytes?: number; used_bytes?: number };
disks?: { mount: string; size_bytes: number; used_bytes: number }[];
listening_ports?: { protocol: "tcp" | "udp"; port: number; address: string; process?: string }[];
/** Updates waiting to be installed. Absent from an agent that doesn't report them (or couldn't find out this time). */
updates?: IncomingUpdates;
}
export interface AgentReport {
@@ -98,10 +101,13 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
}
const system = report.system;
const [before] = await db.select({ pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince }).from(servers).where(eq(servers.id, serverId));
await db
.update(servers)
.set({
lastSeenAt: now,
// Only stored when the agent reported it, so an agent that predates this (or couldn't tell this time) doesn't wipe what it said before.
...(system?.updates && before ? nextUpdateState(before, system.updates, now) : {}),
...(report.hostname ? { hostname: report.hostname } : {}),
// An agent knows what it runs on better than whoever registered the server did.
...(report.osType === "linux" || report.osType === "windows" ? { osType: report.osType } : {}),
+202
View File
@@ -0,0 +1,202 @@
/**
* Recurring maintenance: when a schedule happens (across summer and winter time), and the windows made from it.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-sched-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, maintenanceWindows, maintenanceSchedules, auditLog } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { maintenanceRouter } = await import("../src/routes/maintenance.js");
const { occurrencesBetween, zonedTime, materializeSchedules, isValidTimeZone } = await import("../src/services/maintenanceSchedules.js");
const { listActiveWindows } = await import("../src/services/maintenance.js");
const { eq, like } = await import("drizzle-orm");
await runMigrations();
const sched = (days: number[], startTime: string, durationMinutes: number, timezone: string) => ({ days: JSON.stringify(days), startTime, durationMinutes, timezone });
const iso = (d: Date) => d.toISOString();
describe("when a schedule happens", () => {
test("a wall-clock time follows summer and winter time", () => {
// Sundays in Stockholm: 18 October is still summer time (UTC+2), 1 November is winter time (UTC+1).
const s = sched([0], "02:00", 120, "Europe/Stockholm");
const occ = occurrencesBetween(s, new Date("2026-10-15T00:00:00Z"), new Date("2026-11-03T00:00:00Z"));
const byDate = Object.fromEntries(occ.map((o) => [o.date, [iso(o.start), iso(o.end)]]));
assert.deepEqual(byDate["2026-10-18"], ["2026-10-18T00:00:00.000Z", "2026-10-18T02:00:00.000Z"]);
assert.deepEqual(byDate["2026-11-01"], ["2026-11-01T01:00:00.000Z", "2026-11-01T03:00:00.000Z"]);
assert.deepEqual(Object.keys(byDate).sort(), ["2026-10-18", "2026-10-25", "2026-11-01"]);
});
test("it is the same wall-clock time in another zone, and a time that doesn't exist that night still happens", () => {
assert.equal(iso(zonedTime(2026, 7, 1, 2, 0, "America/New_York")), "2026-07-01T06:00:00.000Z");
assert.equal(iso(zonedTime(2026, 1, 1, 2, 0, "America/New_York")), "2026-01-01T07:00:00.000Z");
// 29 March 2026: clocks in Stockholm jump from 02:00 to 03:00, so 02:30 isn't a time that night.
const gap = zonedTime(2026, 3, 29, 2, 30, "Europe/Stockholm");
assert.ok(Math.abs(gap.getTime() - Date.parse("2026-03-29T01:00:00Z")) <= 3600_000);
assert.equal(zonedTime(2026, 10, 18, 0, 0, "UTC").toISOString(), "2026-10-18T00:00:00.000Z");
});
test("one that runs past midnight belongs to the day it starts, and one under way counts", () => {
const s = sched([6], "23:00", 180, "UTC"); // Saturday 23:00 for three hours
const [occ] = occurrencesBetween(s, new Date("2026-10-18T00:30:00Z"), new Date("2026-10-18T06:00:00Z"));
assert.equal(occ.date, "2026-10-17");
assert.equal(iso(occ.start), "2026-10-17T23:00:00.000Z");
assert.equal(iso(occ.end), "2026-10-18T02:00:00.000Z");
assert.equal(occurrencesBetween(s, new Date("2026-10-18T02:00:00Z"), new Date("2026-10-18T06:00:00Z")).length, 0, "already over");
});
test("every day, and only the days chosen", () => {
const week = occurrencesBetween(sched([0, 1, 2, 3, 4, 5, 6], "04:00", 60, "UTC"), new Date("2026-10-12T00:00:00Z"), new Date("2026-10-19T00:00:00Z"));
assert.equal(week.length, 7);
const weekdays = occurrencesBetween(sched([1, 3], "04:00", 60, "UTC"), new Date("2026-10-12T00:00:00Z"), new Date("2026-10-19T00:00:00Z"));
assert.deepEqual(weekdays.map((o) => o.date), ["2026-10-12", "2026-10-14"]);
});
test("time zone names are checked", () => {
assert.equal(isValidTimeZone("Europe/Stockholm"), true);
assert.equal(isValidTimeZone("Mars/Olympus"), false);
});
});
describe("the schedules and their windows", () => {
let app: Server;
let base = "";
let who = "operator";
let a = 0;
let b = 0;
before(async () => {
await db.insert(users).values([
{ oidcSub: "operator", name: "Olga", role: "operator" },
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
]);
const made = await db
.insert(servers)
.values([
{ name: "elsa", apiTokenHash: "a", apiTokenPrefix: "a" },
{ name: "anna", apiTokenHash: "b", apiTokenPrefix: "b" },
])
.returning({ id: servers.id });
a = made[0].id;
b = made[1].id;
const app0 = express();
app0.use(express.json());
app0.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
app0.use("/api/maintenance", maintenanceRouter);
app = await new Promise<Server>((resolve) => {
const s = app0.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/maintenance`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
const call = (method: string, path: string, body?: unknown) =>
fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
const good = { name: "Patch night", reason: "OS updates", targets: [{ type: "server", id: 0 }], days: [0, 1, 2, 3, 4, 5, 6], startTime: "03:00", durationMinutes: 60, timezone: "UTC" };
const windowsOf = async (id: number) => db.select().from(maintenanceWindows).where(like(maintenanceWindows.source, `schedule:${id}:%`));
let scheduleId = 0;
test("making a schedule makes the coming week's windows, once each, for every server on it", async () => {
const res = await call("POST", "/schedules", { ...good, targets: [{ type: "server", id: a }, { type: "server", id: b }, { type: "server", id: a }] });
assert.equal(res.status, 201);
const body = await res.json();
scheduleId = body.id;
const windows = await windowsOf(scheduleId);
assert.ok(body.windowsMade === windows.length && (windows.length === 14 || windows.length === 16), `made ${windows.length}`);
assert.equal(new Set(windows.map((w) => `${w.targetId}|${w.source}`)).size, windows.length, "no duplicates");
assert.ok(windows.every((w) => w.reason === "Patch night: OS updates" && w.createdBy === "Recurring: Patch night"));
assert.ok(windows.every((w) => w.endsAt > w.startedAt));
// running again makes nothing more
assert.equal(await materializeSchedules(), 0);
assert.equal((await windowsOf(scheduleId)).length, windows.length);
});
test("a window cancelled by hand isn't made again, and the list says what is next", async () => {
const before = await windowsOf(scheduleId);
const one = before.find((w) => w.startedAt > new Date().toISOString())!;
await db.update(maintenanceWindows).set({ endsAt: one.startedAt }).where(eq(maintenanceWindows.id, one.id));
assert.equal(await materializeSchedules(), 0);
assert.equal((await windowsOf(scheduleId)).length, before.length);
const list = await (await call("GET", "/schedules")).json();
assert.equal(list.schedules.length, 1);
assert.equal(list.schedules[0].targets.length, 2);
assert.deepEqual(list.schedules[0].days, [0, 1, 2, 3, 4, 5, 6]);
assert.ok(Date.parse(list.schedules[0].nextStart) > 0);
});
test("a window made from a schedule silences while it runs", async () => {
const now = Date.now();
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: a, reason: "x", startedAt: new Date(now - 60_000).toISOString(), endsAt: new Date(now + 600_000).toISOString(), source: `schedule:${scheduleId}:2000-01-01` });
assert.ok((await listActiveWindows()).some((w) => w.source === `schedule:${scheduleId}:2000-01-01`));
await db.delete(maintenanceWindows).where(eq(maintenanceWindows.source, `schedule:${scheduleId}:2000-01-01`));
});
test("switching it off takes back what hasn't begun, and on makes it again", async () => {
assert.equal((await call("PATCH", `/schedules/${scheduleId}`, { enabled: false })).status, 200);
assert.equal((await windowsOf(scheduleId)).filter((w) => w.startedAt > new Date().toISOString()).length, 0);
assert.equal(await materializeSchedules(), 0, "off means nothing is made");
assert.equal((await call("PATCH", `/schedules/${scheduleId}`, { enabled: true })).status, 200);
assert.ok((await windowsOf(scheduleId)).length >= 12);
});
test("only operators change schedules; a bad one is refused", async () => {
who = "viewer";
assert.equal((await call("POST", "/schedules", { ...good, targets: [{ type: "server", id: a }] })).status, 403);
assert.equal((await call("PATCH", `/schedules/${scheduleId}`, { enabled: false })).status, 403);
assert.equal((await call("DELETE", `/schedules/${scheduleId}`)).status, 403);
assert.equal((await call("GET", "/schedules")).status, 200, "anyone signed in can see them");
who = "operator";
const target = [{ type: "server", id: a }];
for (const bad of [
{ ...good, targets: target, name: "" },
{ ...good, targets: [] },
{ ...good, targets: target, days: [] },
{ ...good, targets: target, days: [7] },
{ ...good, targets: target, startTime: "25:00" },
{ ...good, targets: target, startTime: "3am" },
{ ...good, targets: target, durationMinutes: 5 },
{ ...good, targets: target, timezone: "Mars/Olympus" },
{ ...good, targets: [{ type: "server", id: 99999 }] },
]) {
assert.equal((await call("POST", "/schedules", bad)).status, 400, JSON.stringify(bad).slice(0, 80));
}
});
test("removing a schedule takes back what hasn't begun and is in the audit log", async () => {
assert.equal((await call("DELETE", `/schedules/${scheduleId}`)).status, 204);
assert.equal((await db.select().from(maintenanceSchedules)).length, 0);
assert.equal((await windowsOf(scheduleId)).filter((w) => w.startedAt > new Date().toISOString()).length, 0);
assert.equal((await call("DELETE", `/schedules/${scheduleId}`)).status, 404);
const actions = (await db.select().from(auditLog)).filter((e) => e.category === "maintenance").map((e) => e.action);
for (const wanted of ["create_schedule", "disable_schedule", "enable_schedule", "delete_schedule"]) assert.ok(actions.includes(wanted), wanted);
});
});
+306
View File
@@ -0,0 +1,306 @@
/**
* Folding a guest's "offline" alert under its host's, and what servers say is waiting to be installed.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-parents-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
// No waiting after a restart before servers are judged, or nothing here could be "offline".
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { diffConditions, alsoList } = await import("../src/services/healthMonitor.js");
const { resolveParents, coveredByParent, wouldLoop } = await import("../src/services/serverParents.js");
const { nextUpdateState, parsePendingUpdates } = await import("../src/services/pendingUpdates.js");
const { serversRouter } = await import("../src/routes/servers.js");
const { agentReportRouter } = await import("../src/routes/agentReport.js");
const { hashToken } = await import("../src/services/tokens.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const row = (id: number, name: string, extra: Partial<{ hostname: string | null; parentServerId: number | null; proxmoxNode: string | null }> = {}) => ({
id,
name,
hostname: null,
parentServerId: null,
proxmoxNode: null,
...extra,
});
describe("resolveParents", () => {
test("a link set by hand wins, and a Proxmox node that is exactly one server is found", () => {
const rows = [row(1, "pve1", { hostname: "pve1.lab.example" }), row(2, "web", { proxmoxNode: "PVE1" }), row(3, "db", { parentServerId: 2, proxmoxNode: "pve1" }), row(4, "lone")];
const parents = resolveParents(rows);
assert.deepEqual(parents.get(2), { parentId: 1, auto: true });
assert.deepEqual(parents.get(3), { parentId: 2, auto: false }, "by hand beats the node");
assert.equal(parents.get(4), undefined);
assert.equal(parents.get(1), undefined);
});
test("two servers that could be the node, or none, give no parent; a dead link is ignored", () => {
const rows = [row(1, "pve1"), row(2, "pve1", { hostname: "other" }), row(3, "web", { proxmoxNode: "pve1" }), row(4, "x", { proxmoxNode: "nothing" }), row(5, "y", { parentServerId: 99 }), row(6, "z", { parentServerId: 6 })];
const parents = resolveParents(rows);
assert.equal(parents.get(3), undefined, "ambiguous");
assert.equal(parents.get(4), undefined);
assert.equal(parents.get(5), undefined);
assert.equal(parents.get(6), undefined, "not its own parent");
});
test("a server is not its own node", () => {
assert.equal(resolveParents([row(1, "pve1", { proxmoxNode: "pve1" })]).size, 0);
});
});
describe("coveredByParent", () => {
const parents = resolveParents([row(1, "host"), row(2, "vm", { parentServerId: 1 }), row(3, "ct", { parentServerId: 2 }), row(4, "other", { parentServerId: 1 })]);
test("a guest is covered only while something above it is down, and by the topmost one", () => {
assert.deepEqual([...coveredByParent(new Set([1, 2, 3]), parents)].sort(), [[2, 1], [3, 1]]);
assert.deepEqual([...coveredByParent(new Set([2, 3]), parents)], [[3, 2]], "the host is up: the VM is the cause");
assert.deepEqual([...coveredByParent(new Set([3]), parents)], [], "nothing above it is down");
assert.deepEqual([...coveredByParent(new Set([1]), parents)], []);
assert.deepEqual([...coveredByParent(new Set([1, 3]), parents)], [[3, 1]], "an up VM in between doesn't hide the cause");
});
test("a loop covers nothing, so nothing is swallowed", () => {
const loop = resolveParents([row(1, "a", { parentServerId: 2 }), row(2, "b", { parentServerId: 1 }), row(3, "c", { parentServerId: 1 })]);
assert.equal(coveredByParent(new Set([1, 2, 3]), loop).size, 0);
});
test("wouldLoop spots a link that would close a loop", () => {
const rows = [row(1, "host"), row(2, "vm", { parentServerId: 1 })];
assert.equal(wouldLoop(rows, 1, 2), true);
assert.equal(wouldLoop(rows, 1, 1), true);
assert.equal(wouldLoop(rows, 2, 1), false);
});
});
describe("diffConditions with covered problems", () => {
const c = (key: string) => ({ key, source: "server", message: key });
const names = (list: { key: string }[]) => list.map((x) => x.key).sort();
test("a problem explained by another is recorded but not announced, and neither is its end", () => {
const first = diffConditions({}, [c("offline:server:1"), c("offline:server:2")], new Set(), new Set(), new Set(["offline:server:2"]));
assert.deepEqual(names(first.added), ["offline:server:1"]);
assert.deepEqual(names(first.newlyDown), ["offline:server:1", "offline:server:2"], "both are down, for the record");
assert.equal(first.next["offline:server:2"].covered, true);
const later = diffConditions(first.next, [c("offline:server:1")], new Set(), new Set(), new Set());
assert.deepEqual(names(later.resolved), [], "the guest's end isn't announced either");
assert.deepEqual(names(later.ended), ["offline:server:2"], "but it did end");
});
test("when the explanation goes away and the problem is still there, it is announced as new", () => {
const first = diffConditions({}, [c("offline:server:1"), c("offline:server:2")], new Set(), new Set(), new Set(["offline:server:2"]));
const hostBack = diffConditions(first.next, [c("offline:server:2")], new Set(), new Set(), new Set());
assert.deepEqual(names(hostBack.added), ["offline:server:2"]);
assert.deepEqual(names(hostBack.resolved), ["offline:server:1"]);
assert.deepEqual(names(hostBack.newlyDown), [], "it was already down; nothing began");
// and only once
const again = diffConditions(hostBack.next, [c("offline:server:2")], new Set(), new Set(), new Set());
assert.deepEqual(again.added, []);
});
test("with nothing covered it works as before", () => {
const run = diffConditions({}, [c("a"), c("b")]);
assert.deepEqual(names(run.added), ["a", "b"]);
assert.deepEqual(names(run.newlyDown), ["a", "b"]);
const gone = diffConditions(run.next, [c("a")]);
assert.deepEqual(names(gone.resolved), ["b"]);
assert.deepEqual(names(gone.ended), ["b"]);
});
test("a list of names reads as words", () => {
assert.equal(alsoList(["a"]), "a");
assert.equal(alsoList(["a", "b", "c"]), "a, b and c");
assert.equal(alsoList(["a", "b", "c", "d", "e", "f", "g"]), "a, b, c, d, e and 2 more");
});
});
describe("the alerts, folded", () => {
const hoursAgo = (n: number) => new Date(Date.now() - n * 3600_000).toISOString();
let hostId = 0;
before(async () => {
const made = await db
.insert(servers)
.values([
{ name: "pve1", apiTokenHash: "h", apiTokenPrefix: "h", lastSeenAt: hoursAgo(2) },
{ name: "web", apiTokenHash: "w", apiTokenPrefix: "w", lastSeenAt: hoursAgo(2), proxmoxNode: "pve1" },
{ name: "db", apiTokenHash: "d", apiTokenPrefix: "d", lastSeenAt: hoursAgo(2), proxmoxNode: "pve1" },
{ name: "elsewhere", apiTokenHash: "e", apiTokenPrefix: "e", lastSeenAt: hoursAgo(2) },
])
.returning({ id: servers.id });
hostId = made[0].id;
});
test("the host's alert says what it takes with it, and the guests' are held back, not counted", async () => {
const report = await getAlerts(true);
const offline = report.alerts.filter((a) => a.category === "offline");
assert.equal(offline.length, 4);
const host = offline.find((a) => a.message.startsWith("pve1"))!;
assert.deepEqual(host.affects, ["db", "web"]);
assert.equal(host.silenced, false);
const web = offline.find((a) => a.message.startsWith("web"))!;
assert.deepEqual(web.causedBy, { name: "pve1" });
assert.equal(web.silenced, true, "held back like a silenced alert");
assert.equal(offline.find((a) => a.message.startsWith("elsewhere"))!.causedBy, undefined);
assert.equal(report.counts.critical, 2, "pve1 and the unrelated one");
assert.equal(report.counts.grouped, 2);
assert.equal(report.counts.silenced, 0, "none of this is maintenance");
});
test("when the host reports again the guests that are still down stand alone", async () => {
await db.update(servers).set({ lastSeenAt: new Date().toISOString() }).where(eq(servers.id, hostId));
invalidateAlertsCache(); // a look under ten seconds old is reused otherwise
const report = await getAlerts(true);
assert.equal(report.counts.critical, 3);
assert.equal(report.counts.grouped, 0);
assert.ok(report.alerts.every((a) => !a.causedBy));
});
});
describe("what servers say is waiting to be installed", () => {
test("how long updates have been waiting carries from report to report", () => {
const t0 = "2026-10-01T00:00:00.000Z";
const t1 = "2026-10-05T00:00:00.000Z";
const t2 = "2026-10-09T00:00:00.000Z";
const t3 = "2026-10-12T00:00:00.000Z";
const first = nextUpdateState({ pendingUpdates: null, updatesPendingSince: null }, { manager: "apt", count: 4, security: 1 }, t0);
assert.equal(first.updatesPendingSince, t0);
const second = nextUpdateState(first, { manager: "apt", count: 6, security: 2, reboot_required: true }, t1);
assert.equal(second.updatesPendingSince, t0, "still waiting: the clock keeps running");
assert.deepEqual(parsePendingUpdates(second.pendingUpdates), { manager: "apt", count: 6, security: 2, rebootRequired: true, reportedAt: t1 });
const cleared = nextUpdateState(second, { manager: "apt", count: 0, security: 0 }, t2);
assert.equal(cleared.updatesPendingSince, null);
assert.equal(nextUpdateState(cleared, { manager: "apt", count: 2 }, t3).updatesPendingSince, t3, "and starts again when some appear");
assert.equal(parsePendingUpdates(nextUpdateState(cleared, { count: 2 }, t3).pendingUpdates)?.security, null, "unknown security is unknown, not zero");
});
test("garbage is no updates, not an error", () => {
assert.equal(parsePendingUpdates(null), null);
assert.equal(parsePendingUpdates("{"), null);
assert.equal(parsePendingUpdates(JSON.stringify({ count: "many" })), null);
});
});
describe("the routes", () => {
let app: Server;
let base = "";
let who = "admin";
let hostId = 0;
let guestId = 0;
before(async () => {
await db.insert(users).values([
{ oidcSub: "admin", name: "Ada", role: "admin" },
{ oidcSub: "operator", name: "Olga", role: "operator" },
]);
const [host] = await db.insert(servers).values({ name: "rack", apiTokenHash: hashToken("host-token"), apiTokenPrefix: "ht", lastSeenAt: new Date().toISOString() }).returning();
const [guest] = await db.insert(servers).values({ name: "vm1", apiTokenHash: hashToken("guest-token"), apiTokenPrefix: "gt", lastSeenAt: new Date().toISOString() }).returning();
hostId = host.id;
guestId = guest.id;
const a = express();
a.use("/api/agent/report", agentReportRouter);
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/servers", serversRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
const call = (method: string, path: string, body?: unknown) =>
fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
const report = (token: string, system: unknown) =>
fetch(`${base}/agent/report`, { method: "POST", headers: { "content-type": "application/json", authorization: `Bearer ${token}` }, body: JSON.stringify({ hostname: "h", os_type: "linux", system, tasks: [] }) });
test("an agent's update counts are stored, a report without them keeps what was said, and a garbled answer costs nothing", async () => {
assert.equal((await report("guest-token", { updates: { manager: "apt", count: 7, security: 2, reboot_required: true } })).status, 202);
let [s] = await db.select().from(servers).where(eq(servers.id, guestId));
assert.deepEqual(parsePendingUpdates(s.pendingUpdates), { manager: "apt", count: 7, security: 2, rebootRequired: true, reportedAt: parsePendingUpdates(s.pendingUpdates)!.reportedAt });
const since = s.updatesPendingSince;
assert.ok(since);
assert.equal((await report("guest-token", { cpu: { cores: 2 } })).status, 202);
[s] = await db.select().from(servers).where(eq(servers.id, guestId));
assert.equal(parsePendingUpdates(s.pendingUpdates)?.count, 7, "an agent that said nothing doesn't wipe it");
assert.equal((await report("guest-token", { updates: { count: "lots" }, cpu: { cores: 4 } })).status, 202, "the report itself still goes through");
[s] = await db.select().from(servers).where(eq(servers.id, guestId));
assert.equal(s.cpuCores, 4);
assert.equal(parsePendingUpdates(s.pendingUpdates)?.count, 7);
assert.equal(s.updatesPendingSince, since);
assert.equal((await report("host-token", { updates: { manager: "dnf", count: 0 } })).status, 202);
});
test("the updates list shows every server that reports, with or without figures", async () => {
await db.insert(servers).values({ name: "silent", apiTokenHash: "s", apiTokenPrefix: "s" }); // never reported
const body = await (await call("GET", "/servers/updates")).json();
const byName = new Map(body.servers.map((x: { name: string }) => [x.name, x]));
assert.ok(!byName.has("silent"));
const vm = byName.get("vm1") as { updates: { count: number; security: number; rebootRequired: boolean }; pendingSince: string };
assert.equal(vm.updates.count, 7);
assert.equal(vm.updates.rebootRequired, true);
assert.ok(vm.pendingSince);
assert.equal((byName.get("rack") as { pendingSince: string | null }).pendingSince, null, "nothing waiting");
});
test("an admin says what a server runs on; a loop or a stranger is refused; it shows on both pages", async () => {
assert.equal((await call("PATCH", `/servers/${guestId}`, { parentServerId: hostId })).status, 200);
const detail = await (await call("GET", `/servers/${guestId}/detail`)).json();
assert.deepEqual(detail.relations.parent, { id: hostId, name: "rack", auto: false });
const hostDetail = await (await call("GET", `/servers/${hostId}/detail`)).json();
assert.deepEqual(hostDetail.relations.children, [{ id: guestId, name: "vm1", auto: false }]);
assert.equal(detail.server.pendingUpdates.count, 7, "the detail shows what is waiting, as a figure rather than stored text");
const loop = await call("PATCH", `/servers/${hostId}`, { parentServerId: guestId });
assert.equal(loop.status, 400);
assert.equal((await loop.json()).error, "parent_loop");
assert.equal((await call("PATCH", `/servers/${guestId}`, { parentServerId: guestId })).status, 400);
assert.equal((await call("PATCH", `/servers/${guestId}`, { parentServerId: 99999 })).status, 400);
who = "operator";
assert.equal((await call("PATCH", `/servers/${guestId}`, { parentServerId: null })).status, 403);
who = "admin";
assert.equal((await call("PATCH", `/servers/${guestId}`, { parentServerId: null })).status, 200);
assert.equal((await (await call("GET", `/servers/${guestId}/detail`)).json()).relations.parent, null, "back to working it out");
});
test("removing the host sets the servers on it free", async () => {
await call("PATCH", `/servers/${guestId}`, { parentServerId: hostId });
assert.equal((await call("DELETE", `/servers/${hostId}`)).status, 204);
const [s] = await db.select().from(servers).where(eq(servers.id, guestId));
assert.equal(s.parentServerId, null);
});
});
+143
View File
@@ -0,0 +1,143 @@
/**
* Running a Semaphore template from the app, limited to one host, and watching how the run is getting on.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import http from "node:http";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-sem-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, integrations, auditLog } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { integrationsRouter } = await import("../src/routes/integrations.js");
await runMigrations();
describe("running a Semaphore template", () => {
let semaphore: http.Server;
let app: http.Server;
let base = "";
let who = "operator";
let integrationId = 0;
const received: { method: string; url: string; body: Record<string, unknown> | null; auth: string | undefined }[] = [];
before(async () => {
semaphore = await new Promise<http.Server>((resolve) => {
const s = http.createServer((req, res) => {
let raw = "";
req.on("data", (c) => (raw += c));
req.on("end", () => {
received.push({ method: req.method ?? "", url: req.url ?? "", body: raw ? JSON.parse(raw) : null, auth: req.headers.authorization });
res.setHeader("content-type", "application/json");
if (req.method === "POST" && req.url === "/api/project/3/tasks") return res.end(JSON.stringify({ id: 77, status: "waiting", created: "2026-10-17T10:00:00Z" }));
if (req.method === "GET" && req.url === "/api/project/3/tasks/77") return res.end(JSON.stringify({ id: 77, status: "running", start: "2026-10-17T10:00:05Z" }));
res.statusCode = 404;
res.end(JSON.stringify({ error: "not here" }));
});
});
s.listen(0, "127.0.0.1", () => resolve(s));
});
const url = `http://127.0.0.1:${(semaphore.address() as AddressInfo).port}`;
await db.insert(users).values([
{ oidcSub: "operator", name: "Olga", role: "operator" },
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
]);
const [row] = await db.insert(integrations).values({ type: "semaphore", name: "Semaphore", baseUrl: url, config: JSON.stringify({ url, token: "secret-token" }) }).returning();
integrationId = row.id;
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/integrations", integrationsRouter);
app = await new Promise<http.Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/integrations/${integrationId}/semaphore`;
});
after(() => {
semaphore.close();
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
const run = (body?: unknown) =>
fetch(`${base}/projects/3/templates/9/run`, { method: "POST", headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
test("a run limited to a host tells Semaphore which, and who started it", async () => {
const res = await run({ limit: "elsa" });
assert.equal(res.status, 201);
assert.equal((await res.json()).task.id, 77);
const sent = received.find((r) => r.method === "POST")!;
assert.equal(sent.url, "/api/project/3/tasks");
assert.equal(sent.auth, "Bearer secret-token");
assert.equal(sent.body?.template_id, 9);
assert.equal(sent.body?.limit, "elsa");
assert.match(String(sent.body?.message), /Olga/);
assert.match(String(sent.body?.message), /limited to elsa/);
});
test("a run with no limit sends none, and the audit log records what ran and where", async () => {
received.length = 0;
assert.equal((await run()).status, 201);
assert.equal((await run({})).status, 201);
for (const r of received) {
assert.equal(r.body?.limit, undefined);
assert.doesNotMatch(String(r.body?.message), /limited/);
}
const entries = (await db.select().from(auditLog)).filter((e) => e.action === "run_template");
assert.equal(entries.length, 3);
assert.match(entries[0].detail ?? "", /"limit":"elsa"/);
assert.doesNotMatch(entries[1].detail ?? "", /limit/);
assert.ok(entries.every((e) => !(e.detail ?? "").includes("secret-token")));
});
test("anything that isn't a host or a pattern is refused before it reaches Semaphore", async () => {
received.length = 0;
for (const limit of ["--extra-vars x=1", "-e foo", "elsa; reboot", "$(id)", "a b", "", "x".repeat(201), "../etc"]) {
const res = await run({ limit });
assert.equal(res.status, limit === "" ? 201 : 400, JSON.stringify(limit).slice(0, 40));
}
// the empty one is "no limit"; nothing else was sent on
assert.equal(received.filter((r) => r.method === "POST").length, 1);
for (const ok of ["web01,db*", "elsa.lab.example", "10.0.0.5", "group:&web"]) {
if (ok.includes("&")) continue; // not a pattern this accepts
assert.equal((await run({ limit: ok })).status, 201, ok);
}
});
test("a viewer can't run anything", async () => {
who = "viewer";
received.length = 0;
assert.equal((await run({ limit: "elsa" })).status, 403);
assert.equal(received.length, 0);
who = "operator";
});
test("how a run is getting on can be asked for, and a missing run is reported", async () => {
const res = await fetch(`${base}/projects/3/tasks/77`);
assert.equal(res.status, 200);
assert.deepEqual((await res.json()).task, { id: 77, status: "running", created: null, start: "2026-10-17T10:00:05Z", end: null });
assert.equal((await fetch(`${base}/projects/3/tasks/999`)).status, 502);
assert.equal((await fetch(`${base}/projects/x/tasks/77`)).status, 400);
});
});