/** * Opening an osTicket ticket for an alert. * * The Alerts page already works out everything that is wrong right now. This looks at that list every 15 minutes and, for a * problem that is new and has lasted, opens one ticket — then remembers it, so the same problem never opens a second one however * many checks see it. What it deliberately does not do: close, reopen or comment on a ticket (osTicket's API can't), or open one * for something under a maintenance window. * * planTicketRun() is the decision, a pure function of "the alerts now", "what was remembered" and the settings. Everything else * here is the doing: reading and writing that memory and calling osTicket. */ import { and, desc, eq, inArray, isNull, lt, or, gte } from "drizzle-orm"; import { db } from "../db/client.js"; import { alertTickets, integrations, users } from "../db/schema.js"; import { env } from "../env.js"; import { getAlerts } from "./alerts.js"; import type { Alert, AlertCategory } from "./alertTypes.js"; import { recordAudit } from "./audit.js"; import { getInternalFlag, getSettings, setInternalFlag, type AlertTicketSettings } from "./settingsStore.js"; import { loadIntegrationConfig } from "../integrations/loadIntegration.js"; import { createOsTicketTicketApi, hasTicketApi, ticketUrl, type CreatedTicket, type NewTicket, type OsTicketApiConfig } from "../integrations/osticket/ticketApi.js"; type CurrentUser = typeof users.$inferSelect; export type TicketRow = typeof alertTickets.$inferSelect; type Report = Awaited>; const BASELINE_FLAG = "alertTicketsBaselineDone"; const LAST_RUN_FLAG = "alertTicketsLastRun"; /** How long a cleared problem is remembered after it cleared. */ const KEEP_CLEARED_DAYS = 180; // ─── Deciding ─────────────────────────────────────────────────────────────── /** What the ticket feature never opens a ticket about, whatever the settings: the ticketing system itself. */ function isAboutTicketing(alert: Alert): boolean { // A ticket about osTicket being down, or about overdue tickets, can't help: it would go to the very thing that is down (or add to the pile). return alert.category === "tickets" || alert.id === "integration:failing:osticket"; } export function qualifies(alert: Alert, settings: AlertTicketSettings): boolean { if (alert.silenced) return false; // under a maintenance window: expected, not a reason to wake anyone if (isAboutTicketing(alert)) return false; if (!settings.severities.includes(alert.severity)) return false; return settings.categories.length === 0 || settings.categories.includes(alert.category); } export type Op = | { kind: "insertPending"; alert: Alert } | { kind: "insertBaseline"; alert: Alert } | { kind: "bump"; row: TicketRow; alert: Alert } // another check in a row, not yet enough | { kind: "touch"; row: TicketRow } // still there; nothing else to do | { kind: "reactivate"; row: TicketRow } // cleared a moment ago and is back: the same problem | { kind: "clear"; row: TicketRow } // gone | { kind: "dropPending"; row: TicketRow }; // a blip that went away before it lasted export interface Candidate { alert: Alert; /** The pending row to turn into a ticketed one, or null for a problem with no row yet (confirmChecks of 1). */ row: TicketRow | null; } export interface Plan { ops: Op[]; /** Alerts that have now lasted long enough and should open a ticket. */ toTicket: Candidate[]; } export function planTicketRun(input: { alerts: Alert[]; /** The latest row for each alert id that has one. */ rows: TicketRow[]; settings: AlertTicketSettings; now: Date; /** The first check since this was switched on: record what is already wrong without opening anything for it. */ baseline: boolean; /** Everything could be read. Only then does "not in the list" mean "gone" — otherwise it may just mean "couldn't look". */ complete: boolean; }): Plan { const { alerts, rows, settings, now, baseline, complete } = input; const byKey = new Map(rows.map((r) => [r.alertKey, r])); const present = new Set(alerts.map((a) => a.id)); const ops: Op[] = []; const toTicket: Candidate[] = []; for (const alert of alerts) { const row = byKey.get(alert.id); const wanted = qualifies(alert, settings); if (row && !row.clearedAt) { if (row.status !== "pending") { ops.push({ kind: "touch", row }); } else if (!wanted) { ops.push({ kind: "touch", row }); // there, but silenced or filtered out right now: wait, don't count it } else if (row.seenCount + 1 >= settings.confirmChecks) { toTicket.push({ alert, row }); } else { ops.push({ kind: "bump", row, alert }); } continue; } if (!wanted) continue; if (row?.clearedAt && now.getTime() - Date.parse(row.clearedAt) < settings.reopenAfterMinutes * 60_000) { ops.push({ kind: "reactivate", row }); // flapping: the same problem, not a new one continue; } // A new occurrence. if (baseline) ops.push({ kind: "insertBaseline", alert }); else if (settings.confirmChecks <= 1) toTicket.push({ alert, row: null }); else ops.push({ kind: "insertPending", alert }); } if (complete) { for (const row of rows) { if (row.clearedAt || present.has(row.alertKey)) continue; ops.push({ kind: row.status === "pending" ? "dropPending" : "clear", row }); } } return { ops, toTicket }; } // ─── Writing the ticket ───────────────────────────────────────────────────── const CATEGORY_LABELS: Record = { offline: "Server down", disk: "Disk & storage", services: "Failing services", backup: "Backups", updates: "Updates", expiry: "Expiry", automation: "Automation", integration: "Integration failing", monitoring: "Monitoring", tickets: "Tickets", }; const SEVERITY_LABELS = { critical: "Critical", warning: "Warning", info: "Info" } as const; const stamp = (iso: string) => `${iso.slice(0, 16).replace("T", " ")} UTC`; const absolute = (path: string) => `${env.appBaseUrl.replace(/\/+$/, "")}${path}`; const oneLine = (text: string, max: number) => (text.replace(/\s+/g, " ").trim().length > max ? `${text.replace(/\s+/g, " ").trim().slice(0, max - 1)}…` : text.replace(/\s+/g, " ").trim()); export function buildTicket(alert: Alert, firstSeenAt: string): { subject: string; message: string } { const lines = [ "Homelab Manager found a problem.", "", `Severity: ${SEVERITY_LABELS[alert.severity]}`, `Kind: ${CATEGORY_LABELS[alert.category] ?? alert.category}`, `Source: ${alert.source}`, `Problem: ${alert.message}`, `First seen: ${stamp(firstSeenAt)} (it checks every 15 minutes)`, "", ...(alert.link ? [`Details: ${absolute(alert.link)}`] : []), `Everything that is wrong right now: ${absolute("/alerts")}`, "", `Opened automatically by Homelab Manager (alert ${alert.id}). It is not updated or closed automatically — close it here when it is dealt with.`, ]; return { subject: oneLine(`[Homelab] ${SEVERITY_LABELS[alert.severity]}: ${alert.message}`, 200), message: lines.join("\n") }; } export function buildSummaryTicket(alerts: { alert: Alert; firstSeenAt: string }[]): { subject: string; message: string } { const lines = [ `Homelab Manager found ${alerts.length} problems at once. They are in one ticket so a single outage doesn't fill the queue with dozens.`, "", ...alerts.map(({ alert }) => `- [${SEVERITY_LABELS[alert.severity]}] ${alert.source}: ${alert.message}`), "", `Everything that is wrong right now: ${absolute("/alerts")}`, "", "Opened automatically by Homelab Manager. It is not updated or closed automatically — close it here when it is dealt with.", ]; return { subject: `[Homelab] ${alerts.length} problems detected`, message: lines.join("\n") }; } // ─── Doing ────────────────────────────────────────────────────────────────── export interface RunSummary { at: string; skipped?: "disabled" | "busy"; /** Why nothing could be done (missing setup, osTicket refusing). */ error?: string; /** Tickets opened in osTicket (a summary ticket counts as one). */ ticketsOpened: number; /** Alerts those tickets cover. */ alertsCovered: number; /** Problems recorded without a ticket because they were already there when this was switched on. */ baseline: number; /** Problems seen but not yet in enough checks in a row. */ pending: number; cleared: number; /** Tickets that couldn't be opened this time; they are tried again at the next check. */ failed: number; } export interface RunDeps { getReport(): Promise; createTicket(ticket: NewTicket): Promise; now(): Date; } let running = false; const empty = (now: Date): RunSummary => ({ at: now.toISOString(), ticketsOpened: 0, alertsCovered: 0, baseline: 0, pending: 0, cleared: 0, failed: 0 }); /** The osTicket integration to use, with what it needs to open tickets. Throws a sentence an admin can act on. */ async function ticketClient(settings: AlertTicketSettings) { if (!settings.integrationId) throw new Error("No osTicket integration is chosen for tickets."); const loaded = await loadIntegrationConfig(settings.integrationId); if (!loaded || loaded.integration.type !== "osticket") throw new Error("The chosen osTicket integration no longer exists."); if (!loaded.integration.enabled) throw new Error("The chosen osTicket integration is switched off."); const config = loaded.config as OsTicketApiConfig; if (!hasTicketApi(config)) throw new Error("The osTicket integration has no ticket API address and key. Edit it under Integrations and fill them in."); if (!settings.requesterEmail.trim()) throw new Error("No requester email is set — osTicket needs one for every ticket."); return createOsTicketTicketApi(config); } async function latestRows(now: Date, reopenAfterMinutes: number): Promise { const since = new Date(now.getTime() - Math.max(reopenAfterMinutes, 60) * 60_000).toISOString(); const rows = await db .select() .from(alertTickets) .where(or(isNull(alertTickets.clearedAt), gte(alertTickets.clearedAt, since))) .orderBy(desc(alertTickets.id)); const latest = new Map(); for (const row of rows) if (!latest.has(row.alertKey)) latest.set(row.alertKey, row); return [...latest.values()]; } export async function runAlertTicketCheck(options: { actor?: CurrentUser; deps?: Partial } = {}): Promise { const nowFn = options.deps?.now ?? (() => new Date()); const now = nowFn(); const summary = empty(now); const { alertTickets: settings } = await getSettings(); if (!settings.enabled) { // Switched off: the next time it's switched on counts as a fresh start, so what's wrong by then is the baseline. await setInternalFlag(BASELINE_FLAG, ""); return { ...summary, skipped: "disabled" }; } if (running) return { ...summary, skipped: "busy" }; running = true; try { let client: { createTicket(ticket: NewTicket): Promise }; try { client = options.deps?.createTicket ? { createTicket: options.deps.createTicket } : await ticketClient(settings); } catch (err) { summary.error = err instanceof Error ? err.message : String(err); await setInternalFlag(LAST_RUN_FLAG, JSON.stringify(summary)); return summary; } const report = await (options.deps?.getReport ?? (() => getAlerts(true)))(); const complete = report.couldntCheck.length === 0; const baseline = settings.existingProblems === "ignore" && (await getInternalFlag(BASELINE_FLAG)) !== "1"; const rows = await latestRows(now, settings.reopenAfterMinutes); const plan = planTicketRun({ alerts: report.alerts, rows, settings, now, baseline, complete }); const iso = now.toISOString(); for (const op of plan.ops) { switch (op.kind) { case "insertPending": case "insertBaseline": await db.insert(alertTickets).values({ alertKey: op.alert.id, status: op.kind === "insertPending" ? "pending" : "baseline", seenCount: 1, integrationId: settings.integrationId, severity: op.alert.severity, message: op.alert.message, firstSeenAt: iso, lastSeenAt: iso, }); if (op.kind === "insertPending") summary.pending++; else summary.baseline++; break; case "bump": await db.update(alertTickets).set({ seenCount: op.row.seenCount + 1, lastSeenAt: iso, severity: op.alert.severity, message: op.alert.message }).where(eq(alertTickets.id, op.row.id)); summary.pending++; break; case "touch": await db.update(alertTickets).set({ lastSeenAt: iso }).where(eq(alertTickets.id, op.row.id)); if (op.row.status === "pending") summary.pending++; break; case "reactivate": await db.update(alertTickets).set({ clearedAt: null, lastSeenAt: iso }).where(eq(alertTickets.id, op.row.id)); break; case "clear": await db.update(alertTickets).set({ clearedAt: iso }).where(eq(alertTickets.id, op.row.id)); summary.cleared++; break; case "dropPending": await db.delete(alertTickets).where(eq(alertTickets.id, op.row.id)); break; } } // Open the tickets: one each, or — when a lot appear at once — one for all of them. const requester = { name: settings.requesterName.trim() || "Homelab Manager", email: settings.requesterEmail.trim() }; const common = { ...requester, alertStaff: settings.notifyStaff, topicId: settings.topicId, priorityId: settings.priorityId }; const firstSeen = (c: Candidate) => c.row?.firstSeenAt ?? iso; const groups: Candidate[][] = plan.toTicket.length > settings.summaryAbove ? [plan.toTicket] : plan.toTicket.map((c) => [c]); for (const group of groups) { const content = group.length === 1 ? buildTicket(group[0].alert, firstSeen(group[0])) : buildSummaryTicket(group.map((c) => ({ alert: c.alert, firstSeenAt: firstSeen(c) }))); let created: CreatedTicket; try { created = await client.createTicket({ ...common, ...content }); } catch (err) { summary.failed += group.length; summary.error ??= err instanceof Error ? err.message : String(err); console.error("[alertTickets] couldn't open a ticket:", err instanceof Error ? err.message : err); continue; // nothing was recorded, so the next check tries again } const grouped = group.length > 1; for (const c of group) { const values = { status: "open" as const, ticketNumber: created.number, grouped, ticketedAt: iso, lastSeenAt: iso, severity: c.alert.severity, message: c.alert.message, integrationId: settings.integrationId }; if (c.row) await db.update(alertTickets).set(values).where(eq(alertTickets.id, c.row.id)); else await db.insert(alertTickets).values({ alertKey: c.alert.id, seenCount: 1, firstSeenAt: iso, ...values }); await recordAudit({ actor: options.actor, category: "alert_ticket", action: "create", targetType: "alert", targetId: c.alert.id, detail: { ticket: created.number, severity: c.alert.severity, ...(grouped ? { summaryOf: group.length } : {}), integrationId: settings.integrationId }, }); } summary.ticketsOpened++; summary.alertsCovered += group.length; } // The baseline is only complete if everything could be read: a source that was down would otherwise hide problems that then // look "new" — and get tickets — once it comes back. if (baseline && complete) await setInternalFlag(BASELINE_FLAG, "1"); await db.delete(alertTickets).where(and(lt(alertTickets.clearedAt, new Date(now.getTime() - KEEP_CLEARED_DAYS * 86_400_000).toISOString()))); await setInternalFlag(LAST_RUN_FLAG, JSON.stringify(summary)); return summary; } finally { running = false; } } /** The next check starts afresh: it records what is wrong at that moment as the baseline instead of opening tickets for it. */ export async function resetBaseline(): Promise { await setInternalFlag(BASELINE_FLAG, ""); } /** A ticket that says it is a test, to check the address, the key and where tickets land — without waiting for a real problem. */ export async function sendTestTicket(actor: CurrentUser): Promise { const { alertTickets: settings } = await getSettings(); const client = await ticketClient(settings); const created = await client.createTicket({ name: settings.requesterName.trim() || "Homelab Manager", email: settings.requesterEmail.trim(), alertStaff: settings.notifyStaff, topicId: settings.topicId, priorityId: settings.priorityId, subject: "[Homelab] Test ticket from Homelab Manager", message: `This is a test. ${actor.name ?? actor.email ?? "An administrator"} asked Homelab Manager to open it from Settings → Notifications → Alert tickets, to check that tickets can be created here.\n\nNothing is wrong. You can close this ticket.\n\n${absolute("/settings/alert-tickets")}`, }); await recordAudit({ actor, category: "alert_ticket", action: "test", targetType: "integration", targetId: settings.integrationId ?? undefined, detail: { ticket: created.number } }); return created; } // ─── What the pages show ──────────────────────────────────────────────────── export interface AlertTicketInfo { number: string | null; /** The ticket in osTicket's staff panel, when the address is known. */ url: string | null; /** Part of a summary ticket that covers several alerts. */ grouped: boolean; openedAt: string | null; } /** The ticket opened for each of these alerts that has one and is still active, by alert id. */ export async function ticketsForAlerts(alertIds: string[]): Promise> { const result = new Map(); if (alertIds.length === 0) return result; const rows = await db .select() .from(alertTickets) .where(and(inArray(alertTickets.alertKey, alertIds), eq(alertTickets.status, "open"), isNull(alertTickets.clearedAt))) .orderBy(desc(alertTickets.id)); const apiUrls = new Map(); async function apiUrlOf(integrationId: number | null): Promise { if (integrationId === null) return null; if (!apiUrls.has(integrationId)) { const [row] = await db.select({ config: integrations.config }).from(integrations).where(eq(integrations.id, integrationId)).limit(1); let url: string | null = null; try { const parsed = row?.config ? (JSON.parse(row.config) as { apiUrl?: unknown }) : {}; url = typeof parsed.apiUrl === "string" && parsed.apiUrl.trim() ? parsed.apiUrl : null; } catch { url = null; } apiUrls.set(integrationId, url); } return apiUrls.get(integrationId) ?? null; } for (const row of rows) { if (result.has(row.alertKey)) continue; const base = await apiUrlOf(row.integrationId); result.set(row.alertKey, { number: row.ticketNumber, url: base && row.ticketNumber ? ticketUrl(base, row.ticketNumber) : null, grouped: row.grouped, openedAt: row.ticketedAt }); } return result; } export interface AlertTicketStatus { lastRun: RunSummary | null; /** Active problems with a ticket / waiting to last long enough / already there when this was switched on. */ open: number; pending: number; baseline: number; /** The next check will record what's wrong as the baseline instead of opening tickets for it. */ baselinePending: boolean; } export async function getAlertTicketStatus(): Promise { const active = await db.select({ status: alertTickets.status }).from(alertTickets).where(isNull(alertTickets.clearedAt)); let lastRun: RunSummary | null = null; try { const raw = await getInternalFlag(LAST_RUN_FLAG); lastRun = raw ? (JSON.parse(raw) as RunSummary) : null; } catch { lastRun = null; } const { alertTickets: settings } = await getSettings(); return { lastRun, open: active.filter((r) => r.status === "open").length, pending: active.filter((r) => r.status === "pending").length, baseline: active.filter((r) => r.status === "baseline").length, baselinePending: settings.enabled && settings.existingProblems === "ignore" && (await getInternalFlag(BASELINE_FLAG)) !== "1", }; }