Files
Homelab-manager/server/src/services/alertTickets.ts
T
bobbanandClaude Sonnet 5.5 5dc6cea69c Host health from the Linux agent, agent self-update, and disk forecast in /metrics
- Linux agent v4: failed systemd services, SMART disk health, ZFS pool state, pacman updates;
  SMART/ZFS and self-update run in the ExecStartPre=-+ step outside the sandbox
- Server: hosts' health stored (servers.host_health, migration 0029), alerts under a new
  "Failing services" kind plus disk/ZFS alerts, Host health card, Prometheus series
- Agent self-update (Linux v4, Windows v3): admin switch on the Servers page, checksum-verified
  download, atomic replace; GET /api/agent/report/update
- homelab_server_disk_days_to_full in /metrics
- Tests (server, real PowerShell against a fake server) and docs

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-18 00:54:24 +02:00

442 lines
21 KiB
TypeScript

/**
* Opening an osTicket ticket for an alert.
*
* The Alerts page already works out everything that is wrong right now. This looks at that list every 15 minutes and, for a
* problem that is new and has lasted, opens one ticket — then remembers it, so the same problem never opens a second one however
* many checks see it. What it deliberately does not do: close, reopen or comment on a ticket (osTicket's API can't), or open one
* for something under a maintenance window.
*
* planTicketRun() is the decision, a pure function of "the alerts now", "what was remembered" and the settings. Everything else
* here is the doing: reading and writing that memory and calling osTicket.
*/
import { and, desc, eq, inArray, isNull, lt, or, gte } from "drizzle-orm";
import { db } from "../db/client.js";
import { alertTickets, integrations, users } from "../db/schema.js";
import { env } from "../env.js";
import { getAlerts } from "./alerts.js";
import type { Alert, AlertCategory } from "./alertTypes.js";
import { recordAudit } from "./audit.js";
import { getInternalFlag, getSettings, setInternalFlag, type AlertTicketSettings } from "./settingsStore.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createOsTicketTicketApi, hasTicketApi, ticketUrl, type CreatedTicket, type NewTicket, type OsTicketApiConfig } from "../integrations/osticket/ticketApi.js";
type CurrentUser = typeof users.$inferSelect;
export type TicketRow = typeof alertTickets.$inferSelect;
type Report = Awaited<ReturnType<typeof getAlerts>>;
const BASELINE_FLAG = "alertTicketsBaselineDone";
const LAST_RUN_FLAG = "alertTicketsLastRun";
/** How long a cleared problem is remembered after it cleared. */
const KEEP_CLEARED_DAYS = 180;
// ─── Deciding ───────────────────────────────────────────────────────────────
/** What the ticket feature never opens a ticket about, whatever the settings: the ticketing system itself. */
function isAboutTicketing(alert: Alert): boolean {
// A ticket about osTicket being down, or about overdue tickets, can't help: it would go to the very thing that is down (or add to the pile).
return alert.category === "tickets" || alert.id === "integration:failing:osticket";
}
export function qualifies(alert: Alert, settings: AlertTicketSettings): boolean {
if (alert.silenced) return false; // under a maintenance window: expected, not a reason to wake anyone
if (isAboutTicketing(alert)) return false;
if (!settings.severities.includes(alert.severity)) return false;
return settings.categories.length === 0 || settings.categories.includes(alert.category);
}
export type Op =
| { kind: "insertPending"; alert: Alert }
| { kind: "insertBaseline"; alert: Alert }
| { kind: "bump"; row: TicketRow; alert: Alert } // another check in a row, not yet enough
| { kind: "touch"; row: TicketRow } // still there; nothing else to do
| { kind: "reactivate"; row: TicketRow } // cleared a moment ago and is back: the same problem
| { kind: "clear"; row: TicketRow } // gone
| { kind: "dropPending"; row: TicketRow }; // a blip that went away before it lasted
export interface Candidate {
alert: Alert;
/** The pending row to turn into a ticketed one, or null for a problem with no row yet (confirmChecks of 1). */
row: TicketRow | null;
}
export interface Plan {
ops: Op[];
/** Alerts that have now lasted long enough and should open a ticket. */
toTicket: Candidate[];
}
export function planTicketRun(input: {
alerts: Alert[];
/** The latest row for each alert id that has one. */
rows: TicketRow[];
settings: AlertTicketSettings;
now: Date;
/** The first check since this was switched on: record what is already wrong without opening anything for it. */
baseline: boolean;
/** Everything could be read. Only then does "not in the list" mean "gone" — otherwise it may just mean "couldn't look". */
complete: boolean;
}): Plan {
const { alerts, rows, settings, now, baseline, complete } = input;
const byKey = new Map(rows.map((r) => [r.alertKey, r]));
const present = new Set(alerts.map((a) => a.id));
const ops: Op[] = [];
const toTicket: Candidate[] = [];
for (const alert of alerts) {
const row = byKey.get(alert.id);
const wanted = qualifies(alert, settings);
if (row && !row.clearedAt) {
if (row.status !== "pending") {
ops.push({ kind: "touch", row });
} else if (!wanted) {
ops.push({ kind: "touch", row }); // there, but silenced or filtered out right now: wait, don't count it
} else if (row.seenCount + 1 >= settings.confirmChecks) {
toTicket.push({ alert, row });
} else {
ops.push({ kind: "bump", row, alert });
}
continue;
}
if (!wanted) continue;
if (row?.clearedAt && now.getTime() - Date.parse(row.clearedAt) < settings.reopenAfterMinutes * 60_000) {
ops.push({ kind: "reactivate", row }); // flapping: the same problem, not a new one
continue;
}
// A new occurrence.
if (baseline) ops.push({ kind: "insertBaseline", alert });
else if (settings.confirmChecks <= 1) toTicket.push({ alert, row: null });
else ops.push({ kind: "insertPending", alert });
}
if (complete) {
for (const row of rows) {
if (row.clearedAt || present.has(row.alertKey)) continue;
ops.push({ kind: row.status === "pending" ? "dropPending" : "clear", row });
}
}
return { ops, toTicket };
}
// ─── Writing the ticket ─────────────────────────────────────────────────────
const CATEGORY_LABELS: Record<AlertCategory, string> = {
offline: "Server down",
disk: "Disk & storage",
services: "Failing services",
backup: "Backups",
updates: "Updates",
expiry: "Expiry",
automation: "Automation",
integration: "Integration failing",
monitoring: "Monitoring",
tickets: "Tickets",
};
const SEVERITY_LABELS = { critical: "Critical", warning: "Warning", info: "Info" } as const;
const stamp = (iso: string) => `${iso.slice(0, 16).replace("T", " ")} UTC`;
const absolute = (path: string) => `${env.appBaseUrl.replace(/\/+$/, "")}${path}`;
const oneLine = (text: string, max: number) => (text.replace(/\s+/g, " ").trim().length > max ? `${text.replace(/\s+/g, " ").trim().slice(0, max - 1)}…` : text.replace(/\s+/g, " ").trim());
export function buildTicket(alert: Alert, firstSeenAt: string): { subject: string; message: string } {
const lines = [
"Homelab Manager found a problem.",
"",
`Severity: ${SEVERITY_LABELS[alert.severity]}`,
`Kind: ${CATEGORY_LABELS[alert.category] ?? alert.category}`,
`Source: ${alert.source}`,
`Problem: ${alert.message}`,
`First seen: ${stamp(firstSeenAt)} (it checks every 15 minutes)`,
"",
...(alert.link ? [`Details: ${absolute(alert.link)}`] : []),
`Everything that is wrong right now: ${absolute("/alerts")}`,
"",
`Opened automatically by Homelab Manager (alert ${alert.id}). It is not updated or closed automatically — close it here when it is dealt with.`,
];
return { subject: oneLine(`[Homelab] ${SEVERITY_LABELS[alert.severity]}: ${alert.message}`, 200), message: lines.join("\n") };
}
export function buildSummaryTicket(alerts: { alert: Alert; firstSeenAt: string }[]): { subject: string; message: string } {
const lines = [
`Homelab Manager found ${alerts.length} problems at once. They are in one ticket so a single outage doesn't fill the queue with dozens.`,
"",
...alerts.map(({ alert }) => `- [${SEVERITY_LABELS[alert.severity]}] ${alert.source}: ${alert.message}`),
"",
`Everything that is wrong right now: ${absolute("/alerts")}`,
"",
"Opened automatically by Homelab Manager. It is not updated or closed automatically — close it here when it is dealt with.",
];
return { subject: `[Homelab] ${alerts.length} problems detected`, message: lines.join("\n") };
}
// ─── Doing ──────────────────────────────────────────────────────────────────
export interface RunSummary {
at: string;
skipped?: "disabled" | "busy";
/** Why nothing could be done (missing setup, osTicket refusing). */
error?: string;
/** Tickets opened in osTicket (a summary ticket counts as one). */
ticketsOpened: number;
/** Alerts those tickets cover. */
alertsCovered: number;
/** Problems recorded without a ticket because they were already there when this was switched on. */
baseline: number;
/** Problems seen but not yet in enough checks in a row. */
pending: number;
cleared: number;
/** Tickets that couldn't be opened this time; they are tried again at the next check. */
failed: number;
}
export interface RunDeps {
getReport(): Promise<Report>;
createTicket(ticket: NewTicket): Promise<CreatedTicket>;
now(): Date;
}
let running = false;
const empty = (now: Date): RunSummary => ({ at: now.toISOString(), ticketsOpened: 0, alertsCovered: 0, baseline: 0, pending: 0, cleared: 0, failed: 0 });
/** The osTicket integration to use, with what it needs to open tickets. Throws a sentence an admin can act on. */
async function ticketClient(settings: AlertTicketSettings) {
if (!settings.integrationId) throw new Error("No osTicket integration is chosen for tickets.");
const loaded = await loadIntegrationConfig(settings.integrationId);
if (!loaded || loaded.integration.type !== "osticket") throw new Error("The chosen osTicket integration no longer exists.");
if (!loaded.integration.enabled) throw new Error("The chosen osTicket integration is switched off.");
const config = loaded.config as OsTicketApiConfig;
if (!hasTicketApi(config)) throw new Error("The osTicket integration has no ticket API address and key. Edit it under Integrations and fill them in.");
if (!settings.requesterEmail.trim()) throw new Error("No requester email is set — osTicket needs one for every ticket.");
return createOsTicketTicketApi(config);
}
async function latestRows(now: Date, reopenAfterMinutes: number): Promise<TicketRow[]> {
const since = new Date(now.getTime() - Math.max(reopenAfterMinutes, 60) * 60_000).toISOString();
const rows = await db
.select()
.from(alertTickets)
.where(or(isNull(alertTickets.clearedAt), gte(alertTickets.clearedAt, since)))
.orderBy(desc(alertTickets.id));
const latest = new Map<string, TicketRow>();
for (const row of rows) if (!latest.has(row.alertKey)) latest.set(row.alertKey, row);
return [...latest.values()];
}
export async function runAlertTicketCheck(options: { actor?: CurrentUser; deps?: Partial<RunDeps> } = {}): Promise<RunSummary> {
const nowFn = options.deps?.now ?? (() => new Date());
const now = nowFn();
const summary = empty(now);
const { alertTickets: settings } = await getSettings();
if (!settings.enabled) {
// Switched off: the next time it's switched on counts as a fresh start, so what's wrong by then is the baseline.
await setInternalFlag(BASELINE_FLAG, "");
return { ...summary, skipped: "disabled" };
}
if (running) return { ...summary, skipped: "busy" };
running = true;
try {
let client: { createTicket(ticket: NewTicket): Promise<CreatedTicket> };
try {
client = options.deps?.createTicket ? { createTicket: options.deps.createTicket } : await ticketClient(settings);
} catch (err) {
summary.error = err instanceof Error ? err.message : String(err);
await setInternalFlag(LAST_RUN_FLAG, JSON.stringify(summary));
return summary;
}
const report = await (options.deps?.getReport ?? (() => getAlerts(true)))();
const complete = report.couldntCheck.length === 0;
const baseline = settings.existingProblems === "ignore" && (await getInternalFlag(BASELINE_FLAG)) !== "1";
const rows = await latestRows(now, settings.reopenAfterMinutes);
const plan = planTicketRun({ alerts: report.alerts, rows, settings, now, baseline, complete });
const iso = now.toISOString();
for (const op of plan.ops) {
switch (op.kind) {
case "insertPending":
case "insertBaseline":
await db.insert(alertTickets).values({
alertKey: op.alert.id,
status: op.kind === "insertPending" ? "pending" : "baseline",
seenCount: 1,
integrationId: settings.integrationId,
severity: op.alert.severity,
message: op.alert.message,
firstSeenAt: iso,
lastSeenAt: iso,
});
if (op.kind === "insertPending") summary.pending++;
else summary.baseline++;
break;
case "bump":
await db.update(alertTickets).set({ seenCount: op.row.seenCount + 1, lastSeenAt: iso, severity: op.alert.severity, message: op.alert.message }).where(eq(alertTickets.id, op.row.id));
summary.pending++;
break;
case "touch":
await db.update(alertTickets).set({ lastSeenAt: iso }).where(eq(alertTickets.id, op.row.id));
if (op.row.status === "pending") summary.pending++;
break;
case "reactivate":
await db.update(alertTickets).set({ clearedAt: null, lastSeenAt: iso }).where(eq(alertTickets.id, op.row.id));
break;
case "clear":
await db.update(alertTickets).set({ clearedAt: iso }).where(eq(alertTickets.id, op.row.id));
summary.cleared++;
break;
case "dropPending":
await db.delete(alertTickets).where(eq(alertTickets.id, op.row.id));
break;
}
}
// Open the tickets: one each, or — when a lot appear at once — one for all of them.
const requester = { name: settings.requesterName.trim() || "Homelab Manager", email: settings.requesterEmail.trim() };
const common = { ...requester, alertStaff: settings.notifyStaff, topicId: settings.topicId, priorityId: settings.priorityId };
const firstSeen = (c: Candidate) => c.row?.firstSeenAt ?? iso;
const groups: Candidate[][] = plan.toTicket.length > settings.summaryAbove ? [plan.toTicket] : plan.toTicket.map((c) => [c]);
for (const group of groups) {
const content =
group.length === 1 ? buildTicket(group[0].alert, firstSeen(group[0])) : buildSummaryTicket(group.map((c) => ({ alert: c.alert, firstSeenAt: firstSeen(c) })));
let created: CreatedTicket;
try {
created = await client.createTicket({ ...common, ...content });
} catch (err) {
summary.failed += group.length;
summary.error ??= err instanceof Error ? err.message : String(err);
console.error("[alertTickets] couldn't open a ticket:", err instanceof Error ? err.message : err);
continue; // nothing was recorded, so the next check tries again
}
const grouped = group.length > 1;
for (const c of group) {
const values = { status: "open" as const, ticketNumber: created.number, grouped, ticketedAt: iso, lastSeenAt: iso, severity: c.alert.severity, message: c.alert.message, integrationId: settings.integrationId };
if (c.row) await db.update(alertTickets).set(values).where(eq(alertTickets.id, c.row.id));
else await db.insert(alertTickets).values({ alertKey: c.alert.id, seenCount: 1, firstSeenAt: iso, ...values });
await recordAudit({
actor: options.actor,
category: "alert_ticket",
action: "create",
targetType: "alert",
targetId: c.alert.id,
detail: { ticket: created.number, severity: c.alert.severity, ...(grouped ? { summaryOf: group.length } : {}), integrationId: settings.integrationId },
});
}
summary.ticketsOpened++;
summary.alertsCovered += group.length;
}
// The baseline is only complete if everything could be read: a source that was down would otherwise hide problems that then
// look "new" — and get tickets — once it comes back.
if (baseline && complete) await setInternalFlag(BASELINE_FLAG, "1");
await db.delete(alertTickets).where(and(lt(alertTickets.clearedAt, new Date(now.getTime() - KEEP_CLEARED_DAYS * 86_400_000).toISOString())));
await setInternalFlag(LAST_RUN_FLAG, JSON.stringify(summary));
return summary;
} finally {
running = false;
}
}
/** The next check starts afresh: it records what is wrong at that moment as the baseline instead of opening tickets for it. */
export async function resetBaseline(): Promise<void> {
await setInternalFlag(BASELINE_FLAG, "");
}
/** A ticket that says it is a test, to check the address, the key and where tickets land — without waiting for a real problem. */
export async function sendTestTicket(actor: CurrentUser): Promise<CreatedTicket> {
const { alertTickets: settings } = await getSettings();
const client = await ticketClient(settings);
const created = await client.createTicket({
name: settings.requesterName.trim() || "Homelab Manager",
email: settings.requesterEmail.trim(),
alertStaff: settings.notifyStaff,
topicId: settings.topicId,
priorityId: settings.priorityId,
subject: "[Homelab] Test ticket from Homelab Manager",
message: `This is a test. ${actor.name ?? actor.email ?? "An administrator"} asked Homelab Manager to open it from Settings → Alert tickets, to check that tickets can be created here.\n\nNothing is wrong. You can close this ticket.\n\n${absolute("/settings/alert-tickets")}`,
});
await recordAudit({ actor, category: "alert_ticket", action: "test", targetType: "integration", targetId: settings.integrationId ?? undefined, detail: { ticket: created.number } });
return created;
}
// ─── What the pages show ────────────────────────────────────────────────────
export interface AlertTicketInfo {
number: string | null;
/** The ticket in osTicket's staff panel, when the address is known. */
url: string | null;
/** Part of a summary ticket that covers several alerts. */
grouped: boolean;
openedAt: string | null;
}
/** The ticket opened for each of these alerts that has one and is still active, by alert id. */
export async function ticketsForAlerts(alertIds: string[]): Promise<Map<string, AlertTicketInfo>> {
const result = new Map<string, AlertTicketInfo>();
if (alertIds.length === 0) return result;
const rows = await db
.select()
.from(alertTickets)
.where(and(inArray(alertTickets.alertKey, alertIds), eq(alertTickets.status, "open"), isNull(alertTickets.clearedAt)))
.orderBy(desc(alertTickets.id));
const apiUrls = new Map<number, string | null>();
async function apiUrlOf(integrationId: number | null): Promise<string | null> {
if (integrationId === null) return null;
if (!apiUrls.has(integrationId)) {
const [row] = await db.select({ config: integrations.config }).from(integrations).where(eq(integrations.id, integrationId)).limit(1);
let url: string | null = null;
try {
const parsed = row?.config ? (JSON.parse(row.config) as { apiUrl?: unknown }) : {};
url = typeof parsed.apiUrl === "string" && parsed.apiUrl.trim() ? parsed.apiUrl : null;
} catch {
url = null;
}
apiUrls.set(integrationId, url);
}
return apiUrls.get(integrationId) ?? null;
}
for (const row of rows) {
if (result.has(row.alertKey)) continue;
const base = await apiUrlOf(row.integrationId);
result.set(row.alertKey, { number: row.ticketNumber, url: base && row.ticketNumber ? ticketUrl(base, row.ticketNumber) : null, grouped: row.grouped, openedAt: row.ticketedAt });
}
return result;
}
export interface AlertTicketStatus {
lastRun: RunSummary | null;
/** Active problems with a ticket / waiting to last long enough / already there when this was switched on. */
open: number;
pending: number;
baseline: number;
/** The next check will record what's wrong as the baseline instead of opening tickets for it. */
baselinePending: boolean;
}
export async function getAlertTicketStatus(): Promise<AlertTicketStatus> {
const active = await db.select({ status: alertTickets.status }).from(alertTickets).where(isNull(alertTickets.clearedAt));
let lastRun: RunSummary | null = null;
try {
const raw = await getInternalFlag(LAST_RUN_FLAG);
lastRun = raw ? (JSON.parse(raw) as RunSummary) : null;
} catch {
lastRun = null;
}
const { alertTickets: settings } = await getSettings();
return {
lastRun,
open: active.filter((r) => r.status === "open").length,
pending: active.filter((r) => r.status === "pending").length,
baseline: active.filter((r) => r.status === "baseline").length,
baselinePending: settings.enabled && settings.existingProblems === "ignore" && (await getInternalFlag(BASELINE_FLAG)) !== "1",
};
}