Files
Homelab-manager/server/src/services/automationMonitor.ts
T
bobbanandClaude Sonnet 5.5 447f33fff6 Add an Alerts page under Operations listing everything that's wrong now
One list of the current problems across servers and integrations, instead
of waiting for a notification or visiting each page: servers that stopped
reporting, full or nearly full disks and volumes (critical from 95%),
Synology volume/disk problems, failed or uncovered Proxmox backups,
failed Proxmox Backup Server verifications, container image updates,
expired or expiring secrets/domains/Tailscale keys, failed Semaphore and
Gitea runs, Uptime Kuma monitors that are down, overdue osTicket tickets,
and integrations whose calls keep failing. Visible to every role, with
severity and kind filters, search, sorting, CSV export and "Check now".

It runs the same checks that send the notifications rather than a second
copy of them: the detection in the health, automation, Proxmox backup,
PBS, Docker update and Tailscale key checks is pulled out into shared
collectors that both the schedulers and the page call, so the two can't
disagree about what counts as a problem. Notification behaviour is
unchanged, including the scheduled backup checks skipping integrations
under a maintenance window. Unlike the notifications the page ignores the
on/off toggles, and keeps problems under a maintenance window, marked
silenced and counted apart.

It reads live, so a result is reused for a minute (and Refresh can't
re-run everything more than once every ten seconds), and every source has
a 20 s limit so one hung integration can't hang the page. Anything it
couldn't read is called out at the top instead of looking like all clear,
and server checks pause for the same 20 minutes after a restart as the
notifications do, with a note saying so.

Also gives the newer integrations (PBS, osTicket, Uptime Kuma, phpIPAM)
proper names in "integration down" notifications instead of their ids.

Verified through the real routes against a scratch database with fake
backends (offline and full-disk servers, secrets and domains, a silenced
server, a fake PBS with failed verification, a hanging integration, a
refused one, a failing-calls streak, caching, the restart grace period,
auth), and by rendering the real page against that data in a browser:
filters, search, silenced toggle, sorting, Check now, dark mode.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-02 23:07:57 +02:00

159 lines
7.6 KiB
TypeScript

import { and, eq, inArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations } from "../db/schema.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createSemaphoreAdapter } from "../integrations/semaphore/adapter.js";
import { createGiteaAdapter } from "../integrations/gitea/adapter.js";
import { diffConditions, type ActiveState, type HealthCondition } from "./healthMonitor.js";
import { activeSubjects } from "./maintenance.js";
import { notifyAutomationFailed, notifyAutomationRecovered } from "./notify.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
const STATE_FLAG = "automationActiveConditions";
/**
* What a piece of automation's most recent run tells us. "unknown" covers everything that is neither a clear
* pass nor a clear failure — still running, waiting, cancelled, skipped, stopped by hand — and means "don't
* change what we were saying": a run in progress must not clear a failure it hasn't yet fixed, and a manual
* stop isn't a failure.
*/
export type RunOutcome = "failed" | "passed" | "unknown";
export interface AutomationItem {
/** Stable identity: the same template/repo always has the same key. */
key: string;
/** Hierarchical source, "semaphore:3:12:45" — held when the run's outcome isn't known. */
source: string;
outcome: RunOutcome;
/** What to say when it has failed. */
message: string;
}
// ─── Classification (pure) ──────────────────────────────────────────────────
export function classifySemaphoreStatus(status: string | null | undefined): RunOutcome {
if (status === "error") return "failed";
if (status === "success") return "passed";
return "unknown"; // waiting, starting, running, stopping, stopped, rejected, confirmed, ...
}
export function classifyGiteaRun(run: { status: string; conclusion: string | null }): RunOutcome {
if (run.conclusion === "failure" || run.status === "failure") return "failed";
if (run.conclusion === "success" || run.status === "success") return "passed";
return "unknown"; // running, waiting, blocked, cancelled, skipped, ...
}
function endedSuffix(end: string | null): string {
if (!end) return "";
const t = Date.parse(end);
return Number.isFinite(t) ? ` (${new Date(t).toISOString().slice(0, 16).replace("T", " ")} UTC)` : "";
}
/** Splits the items into the failures to report and the sources whose state must be left alone this pass. */
export function evaluateAutomation(items: AutomationItem[]): { conditions: HealthCondition[]; held: Set<string> } {
const conditions: HealthCondition[] = [];
const held = new Set<string>();
for (const item of items) {
if (item.outcome === "failed") conditions.push({ key: item.key, source: item.source, message: item.message });
else if (item.outcome === "unknown") held.add(item.source);
}
return { conditions, held };
}
// ─── Collection (I/O) ───────────────────────────────────────────────────────
export async function collectAutomation(): Promise<{ items: AutomationItem[]; held: Set<string>; readable: number }> {
const items: AutomationItem[] = [];
const held = new Set<string>();
let readable = 0;
const rows = await db
.select({ id: integrations.id, name: integrations.name, type: integrations.type })
.from(integrations)
.where(and(eq(integrations.enabled, true), inArray(integrations.type, ["semaphore", "gitea"])));
for (const row of rows) {
try {
const loaded = await loadIntegrationConfig(row.id);
if (!loaded) continue;
if (row.type === "semaphore") {
const { templates, failedProjectIds } = await createSemaphoreAdapter(loaded.config as any).checkTemplates();
// A project that couldn't be listed contributes no templates — that's "couldn't read", not "all fixed".
for (const pid of failedProjectIds) held.add(`semaphore:${row.id}:${pid}`);
for (const t of templates) {
if (!t.lastTask) continue;
items.push({
key: `automation:semaphore:${row.id}:${t.projectId}:${t.id}`,
source: `semaphore:${row.id}:${t.projectId}:${t.id}`,
outcome: classifySemaphoreStatus(t.lastTask.status),
message: `${row.name} / ${t.projectName} / ${t.name}: run #${t.lastTask.id} failed${endedSuffix(t.lastTask.end)}`,
});
}
} else {
const repos = await createGiteaAdapter(loaded.config as any).listReposWithStatus();
for (const r of repos) {
if (!r.hasActions) continue;
const source = `gitea:${row.id}:${r.fullName}`;
// The adapter reports a run it couldn't fetch as null, the same as "no runs yet" — either way there's nothing to judge.
if (!r.latestRun) {
held.add(source);
continue;
}
const run = r.latestRun;
items.push({
key: `automation:gitea:${row.id}:${r.fullName}`,
source,
outcome: classifyGiteaRun(run),
message: `${row.name} / ${r.fullName}: "${run.displayTitle || "workflow"}" (run #${run.runNumber}) failed${run.headBranch ? ` on ${run.headBranch}` : ""}${run.htmlUrl ? ` — ${run.htmlUrl}` : ""}`,
});
}
}
readable++;
} catch (err) {
console.error(`[automation] couldn't read ${row.type} integration ${row.id}:`, err instanceof Error ? err.message : err);
held.add(`${row.type}:${row.id}`);
}
}
return { items, held, readable };
}
// ─── The scheduled pass ─────────────────────────────────────────────────────
async function loadState(): Promise<ActiveState | null> {
const raw = await getInternalFlag(STATE_FLAG);
if (raw === null) return null;
try {
return JSON.parse(raw);
} catch {
return {};
}
}
/**
* Reports each template/repo whose latest run failed, once, and again when it succeeds. State-based like the
* health monitor: a template that keeps failing every night alerts on the first failure, not every night.
*
* The very first pass only records what is already failing, without announcing it — on a fresh install (or
* the first time this feature runs) that would otherwise be a wall of alerts about failures that are months old.
*/
export async function runAutomationCheck(now: number = Date.now()): Promise<{ added: number; resolved: number; baseline: boolean }> {
const { items, held: readHeld, readable } = await collectAutomation();
const stored = await loadState();
// Nothing could be read at all (or nothing is configured): don't spend the "first pass" on an empty picture.
if (stored === null && readable === 0) return { added: 0, resolved: 0, baseline: false };
const { conditions, held: unknownHeld } = evaluateAutomation(items);
const held = new Set([...readHeld, ...unknownHeld]);
const { added, resolved, next } = diffConditions(stored ?? {}, conditions, held, await activeSubjects(new Date(now)));
await setInternalFlag(STATE_FLAG, JSON.stringify(next));
if (stored === null) return { added: 0, resolved: 0, baseline: true };
// State is tracked even with the alert toggle off (the notify functions check it), so turning it back on isn't a flood.
if (added.length > 0) await notifyAutomationFailed(added);
if (resolved.length > 0) await notifyAutomationRecovered(resolved);
return { added: added.length, resolved: resolved.length, baseline: false };
}