Files
Homelab-manager/server/src/services/maintenance.ts
T
bobbanandClaude Sonnet 5.5 93aaf5476d Lidarr/Readarr, integration inspector, app versions, grouped alerts
Lidarr and Readarr join the shared *arr reader. Integrations get an Inspect button that
runs the reader once and shows what each request returned as shapes (never values) and
where the result is empty. Operations -> App versions compares each application's version
with its newest GitHub release (lookup off until an admin switches it on; optional alert).
The Alerts page can group by source, kind of problem or type of alert. The shared HTTP
helper no longer reuses connections (stale-socket resets). Migration 0078; docs and tests
included.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-29 20:07:36 +01:00

168 lines
8.6 KiB
TypeScript

import { and, eq, gt, lt, lte } from "drizzle-orm";
import { db } from "../db/client.js";
import { dnsProviders, integrations, maintenanceWindows, networkDevices, servers } from "../db/schema.js";
export type ActiveWindow = typeof maintenanceWindows.$inferSelect;
/** Windows that have started and not yet ended. ISO strings compare correctly as text, so these are plain string comparisons. */
export async function listActiveWindows(now: Date = new Date()): Promise<ActiveWindow[]> {
const t = now.toISOString();
return db.select().from(maintenanceWindows).where(and(lte(maintenanceWindows.startedAt, t), gt(maintenanceWindows.endsAt, t)));
}
/** Windows that are set up but haven't begun — the schedule mirrored from another system. They silence nothing until they start. */
export async function listUpcomingWindows(now: Date = new Date()): Promise<ActiveWindow[]> {
// A window cancelled before it began has its end set back to its start, and is not a window any more.
return db.select().from(maintenanceWindows).where(and(gt(maintenanceWindows.startedAt, now.toISOString()), gt(maintenanceWindows.endsAt, maintenanceWindows.startedAt)));
}
/**
* The identity a health condition or alert belongs to, as "server:3" /
* "integration:1" / "dns_provider:2", so it can be matched against active windows.
* Derived from the condition key (which already encodes it) rather than stored,
* so conditions persisted by an earlier version still map correctly.
*/
export function subjectOfConditionKey(key: string): string | null {
let m = /^(?:offline|disk|failed-units|smart|zfs|new-port):server:(\d+)(?::|$)/.exec(key);
if (m) return `server:${m[1]}`;
m = /^rule:\d+:server:(\d+)(?::|$)/.exec(key);
if (m) return `server:${m[1]}`;
m = /^baseline:(?:cpu|mem):server:(\d+)$/.exec(key);
if (m) return `server:${m[1]}`;
m = /^container-(?:restarting|unhealthy|stopped|created|host):(\d+):/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^docker-(?:vuln(?:-high)?|image-age):(\d+):/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^jellyfin-(?:restart|task|logins):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^nextcloud-(?:maintenance|upgrade|disk|cron|checks):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^speedtest-(?:slow|failing):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^hetzner-(?:off|stuck|rescue|traffic|forecast|backup|lb|cert|fip):(\d+):/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^fragnet-(?:disk|memory|backup):(\d+):/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^(?:sonarr|radarr|prowlarr|lidarr|readarr)-(?:health|queue|disk):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^version-behind:(\d+)$/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^adguard-(?:stopped|dns|protection|filters):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^grafana-(?:firing|rule-error|ds):(\d+):/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^netbox-(?:workers):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^n8n-(?:failed):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^paperless-(?:component|failed|disk):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^immich-(?:disk|jobs):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^prometheus-(?:target|alert):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^authentik-(?:outpost|outpost-version|cert|token|worker|logins|errors):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^duplicati-(?:failed|stale):\d+:server:(\d+)$/.exec(key);
if (m) return `server:${m[1]}`;
m = /^(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:(\d+)$/.exec(key);
if (m) return `server:${m[1]}`;
m = /^offline:device:(\d+)$/.exec(key);
if (m) return `device:${m[1]}`;
m = /^disk:proxmox:(\d+):/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^(?:synology-volume|synology-disk|disk:synology):(\d+):/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^automation:(?:semaphore|gitea):(\d+):/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^ups-(?:battery|low|replace|load):(\d+)$/.exec(key);
if (m) return `integration:${m[1]}`;
return null;
}
export async function activeSubjects(now: Date = new Date()): Promise<Set<string>> {
return new Set((await listActiveWindows(now)).map((w) => `${w.targetType}:${w.targetId}`));
}
export async function isInMaintenance(targetType: ActiveWindow["targetType"], targetId: number, now: Date = new Date()): Promise<boolean> {
return (await activeSubjects(now)).has(`${targetType}:${targetId}`);
}
/**
* Integration-failure alerts are tracked per service TYPE ("proxmox",
* "cloudflare", ...), not per configured instance, so a window on any
* integration or DNS provider of that type silences that type's failure alerts.
* (With two integrations of one type, a real failure on the un-windowed one is
* silenced too while the window is open — a known consequence of that
* granularity, stated on the Maintenance page.)
*/
export async function isSourceInMaintenance(source: string, now: Date = new Date()): Promise<boolean> {
const windows = await listActiveWindows(now);
if (windows.length === 0) return false;
for (const w of windows) {
if (w.targetType === "integration") {
const [row] = await db.select({ type: integrations.type }).from(integrations).where(eq(integrations.id, w.targetId)).limit(1);
if (row?.type === source) return true;
} else if (w.targetType === "dns_provider") {
const [row] = await db.select({ type: dnsProviders.providerType }).from(dnsProviders).where(eq(dnsProviders.id, w.targetId)).limit(1);
if (row?.type === source) return true;
}
}
return false;
}
export interface StartWindowResult {
window: ActiveWindow;
target: { name: string; kind: string };
/** True if this extended an already-active window rather than starting a new one. */
extended: boolean;
}
/**
* Starts a maintenance window for a target, or — if one is already active for it — extends it instead of
* stacking a second one, pruning long-expired rows along the way. Returns null if the target doesn't exist.
* Shared by the manual "Start maintenance" form and the Uptime Kuma import, so both silence alerts identically;
* callers still record their own audit entry, since only they know the acting user.
*/
export async function startOrExtendWindow(
targetType: ActiveWindow["targetType"],
targetId: number,
minutes: number,
reason: string | null,
createdBy: string | null,
now: Date = new Date(),
): Promise<StartWindowResult | null> {
const target = await describeTarget(targetType, targetId);
if (!target) return null;
const endsAt = new Date(now.getTime() + minutes * 60_000).toISOString();
const existing = (await listActiveWindows(now)).find((w) => w.targetType === targetType && w.targetId === targetId);
let window: ActiveWindow;
if (existing) {
[window] = await db.update(maintenanceWindows).set({ endsAt, reason, createdBy }).where(eq(maintenanceWindows.id, existing.id)).returning();
} else {
// Long-expired rows are just clutter; clear them out whenever a new one is added.
await db.delete(maintenanceWindows).where(lt(maintenanceWindows.endsAt, new Date(now.getTime() - 24 * 3600_000).toISOString()));
[window] = await db.insert(maintenanceWindows).values({ targetType, targetId, reason, startedAt: now.toISOString(), endsAt, createdBy }).returning();
}
return { window, target, extended: !!existing };
}
/** Display name for a window's target, or null if the target no longer exists. */
export async function describeTarget(targetType: ActiveWindow["targetType"], targetId: number): Promise<{ name: string; kind: string } | null> {
if (targetType === "server") {
const [r] = await db.select({ name: servers.name }).from(servers).where(eq(servers.id, targetId)).limit(1);
return r ? { name: r.name, kind: "Server" } : null;
}
if (targetType === "device") {
const [r] = await db.select({ name: networkDevices.name, kind: networkDevices.kind }).from(networkDevices).where(eq(networkDevices.id, targetId)).limit(1);
return r ? { name: r.name, kind: `Network device (${r.kind.replace("_", " ")})` } : null;
}
if (targetType === "integration") {
const [r] = await db.select({ name: integrations.name, type: integrations.type }).from(integrations).where(eq(integrations.id, targetId)).limit(1);
return r ? { name: r.name, kind: `Integration (${r.type})` } : null;
}
const [r] = await db.select({ name: dnsProviders.name, type: dnsProviders.providerType }).from(dnsProviders).where(eq(dnsProviders.id, targetId)).limit(1);
return r ? { name: r.name, kind: `DNS provider (${r.type})` } : null;
}