- Linux agent v4: failed systemd services, SMART disk health, ZFS pool state, pacman updates; SMART/ZFS and self-update run in the ExecStartPre=-+ step outside the sandbox - Server: hosts' health stored (servers.host_health, migration 0029), alerts under a new "Failing services" kind plus disk/ZFS alerts, Host health card, Prometheus series - Agent self-update (Linux v4, Windows v3): admin switch on the Servers page, checksum-verified download, atomic replace; GET /api/agent/report/update - homelab_server_disk_days_to_full in /metrics - Tests (server, real PowerShell against a fake server) and docs Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
442 lines
21 KiB
TypeScript
442 lines
21 KiB
TypeScript
/**
|
|
* Opening an osTicket ticket for an alert.
|
|
*
|
|
* The Alerts page already works out everything that is wrong right now. This looks at that list every 15 minutes and, for a
|
|
* problem that is new and has lasted, opens one ticket — then remembers it, so the same problem never opens a second one however
|
|
* many checks see it. What it deliberately does not do: close, reopen or comment on a ticket (osTicket's API can't), or open one
|
|
* for something under a maintenance window.
|
|
*
|
|
* planTicketRun() is the decision, a pure function of "the alerts now", "what was remembered" and the settings. Everything else
|
|
* here is the doing: reading and writing that memory and calling osTicket.
|
|
*/
|
|
import { and, desc, eq, inArray, isNull, lt, or, gte } from "drizzle-orm";
|
|
import { db } from "../db/client.js";
|
|
import { alertTickets, integrations, users } from "../db/schema.js";
|
|
import { env } from "../env.js";
|
|
import { getAlerts } from "./alerts.js";
|
|
import type { Alert, AlertCategory } from "./alertTypes.js";
|
|
import { recordAudit } from "./audit.js";
|
|
import { getInternalFlag, getSettings, setInternalFlag, type AlertTicketSettings } from "./settingsStore.js";
|
|
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
|
import { createOsTicketTicketApi, hasTicketApi, ticketUrl, type CreatedTicket, type NewTicket, type OsTicketApiConfig } from "../integrations/osticket/ticketApi.js";
|
|
|
|
type CurrentUser = typeof users.$inferSelect;
|
|
export type TicketRow = typeof alertTickets.$inferSelect;
|
|
type Report = Awaited<ReturnType<typeof getAlerts>>;
|
|
|
|
const BASELINE_FLAG = "alertTicketsBaselineDone";
|
|
const LAST_RUN_FLAG = "alertTicketsLastRun";
|
|
/** How long a cleared problem is remembered after it cleared. */
|
|
const KEEP_CLEARED_DAYS = 180;
|
|
|
|
// ─── Deciding ───────────────────────────────────────────────────────────────
|
|
|
|
/** What the ticket feature never opens a ticket about, whatever the settings: the ticketing system itself. */
|
|
function isAboutTicketing(alert: Alert): boolean {
|
|
// A ticket about osTicket being down, or about overdue tickets, can't help: it would go to the very thing that is down (or add to the pile).
|
|
return alert.category === "tickets" || alert.id === "integration:failing:osticket";
|
|
}
|
|
|
|
export function qualifies(alert: Alert, settings: AlertTicketSettings): boolean {
|
|
if (alert.silenced) return false; // under a maintenance window: expected, not a reason to wake anyone
|
|
if (isAboutTicketing(alert)) return false;
|
|
if (!settings.severities.includes(alert.severity)) return false;
|
|
return settings.categories.length === 0 || settings.categories.includes(alert.category);
|
|
}
|
|
|
|
export type Op =
|
|
| { kind: "insertPending"; alert: Alert }
|
|
| { kind: "insertBaseline"; alert: Alert }
|
|
| { kind: "bump"; row: TicketRow; alert: Alert } // another check in a row, not yet enough
|
|
| { kind: "touch"; row: TicketRow } // still there; nothing else to do
|
|
| { kind: "reactivate"; row: TicketRow } // cleared a moment ago and is back: the same problem
|
|
| { kind: "clear"; row: TicketRow } // gone
|
|
| { kind: "dropPending"; row: TicketRow }; // a blip that went away before it lasted
|
|
|
|
export interface Candidate {
|
|
alert: Alert;
|
|
/** The pending row to turn into a ticketed one, or null for a problem with no row yet (confirmChecks of 1). */
|
|
row: TicketRow | null;
|
|
}
|
|
|
|
export interface Plan {
|
|
ops: Op[];
|
|
/** Alerts that have now lasted long enough and should open a ticket. */
|
|
toTicket: Candidate[];
|
|
}
|
|
|
|
export function planTicketRun(input: {
|
|
alerts: Alert[];
|
|
/** The latest row for each alert id that has one. */
|
|
rows: TicketRow[];
|
|
settings: AlertTicketSettings;
|
|
now: Date;
|
|
/** The first check since this was switched on: record what is already wrong without opening anything for it. */
|
|
baseline: boolean;
|
|
/** Everything could be read. Only then does "not in the list" mean "gone" — otherwise it may just mean "couldn't look". */
|
|
complete: boolean;
|
|
}): Plan {
|
|
const { alerts, rows, settings, now, baseline, complete } = input;
|
|
const byKey = new Map(rows.map((r) => [r.alertKey, r]));
|
|
const present = new Set(alerts.map((a) => a.id));
|
|
const ops: Op[] = [];
|
|
const toTicket: Candidate[] = [];
|
|
|
|
for (const alert of alerts) {
|
|
const row = byKey.get(alert.id);
|
|
const wanted = qualifies(alert, settings);
|
|
|
|
if (row && !row.clearedAt) {
|
|
if (row.status !== "pending") {
|
|
ops.push({ kind: "touch", row });
|
|
} else if (!wanted) {
|
|
ops.push({ kind: "touch", row }); // there, but silenced or filtered out right now: wait, don't count it
|
|
} else if (row.seenCount + 1 >= settings.confirmChecks) {
|
|
toTicket.push({ alert, row });
|
|
} else {
|
|
ops.push({ kind: "bump", row, alert });
|
|
}
|
|
continue;
|
|
}
|
|
|
|
if (!wanted) continue;
|
|
|
|
if (row?.clearedAt && now.getTime() - Date.parse(row.clearedAt) < settings.reopenAfterMinutes * 60_000) {
|
|
ops.push({ kind: "reactivate", row }); // flapping: the same problem, not a new one
|
|
continue;
|
|
}
|
|
|
|
// A new occurrence.
|
|
if (baseline) ops.push({ kind: "insertBaseline", alert });
|
|
else if (settings.confirmChecks <= 1) toTicket.push({ alert, row: null });
|
|
else ops.push({ kind: "insertPending", alert });
|
|
}
|
|
|
|
if (complete) {
|
|
for (const row of rows) {
|
|
if (row.clearedAt || present.has(row.alertKey)) continue;
|
|
ops.push({ kind: row.status === "pending" ? "dropPending" : "clear", row });
|
|
}
|
|
}
|
|
return { ops, toTicket };
|
|
}
|
|
|
|
// ─── Writing the ticket ─────────────────────────────────────────────────────
|
|
|
|
const CATEGORY_LABELS: Record<AlertCategory, string> = {
|
|
offline: "Server down",
|
|
disk: "Disk & storage",
|
|
services: "Failing services",
|
|
backup: "Backups",
|
|
updates: "Updates",
|
|
expiry: "Expiry",
|
|
automation: "Automation",
|
|
integration: "Integration failing",
|
|
monitoring: "Monitoring",
|
|
tickets: "Tickets",
|
|
};
|
|
const SEVERITY_LABELS = { critical: "Critical", warning: "Warning", info: "Info" } as const;
|
|
|
|
const stamp = (iso: string) => `${iso.slice(0, 16).replace("T", " ")} UTC`;
|
|
const absolute = (path: string) => `${env.appBaseUrl.replace(/\/+$/, "")}${path}`;
|
|
const oneLine = (text: string, max: number) => (text.replace(/\s+/g, " ").trim().length > max ? `${text.replace(/\s+/g, " ").trim().slice(0, max - 1)}…` : text.replace(/\s+/g, " ").trim());
|
|
|
|
export function buildTicket(alert: Alert, firstSeenAt: string): { subject: string; message: string } {
|
|
const lines = [
|
|
"Homelab Manager found a problem.",
|
|
"",
|
|
`Severity: ${SEVERITY_LABELS[alert.severity]}`,
|
|
`Kind: ${CATEGORY_LABELS[alert.category] ?? alert.category}`,
|
|
`Source: ${alert.source}`,
|
|
`Problem: ${alert.message}`,
|
|
`First seen: ${stamp(firstSeenAt)} (it checks every 15 minutes)`,
|
|
"",
|
|
...(alert.link ? [`Details: ${absolute(alert.link)}`] : []),
|
|
`Everything that is wrong right now: ${absolute("/alerts")}`,
|
|
"",
|
|
`Opened automatically by Homelab Manager (alert ${alert.id}). It is not updated or closed automatically — close it here when it is dealt with.`,
|
|
];
|
|
return { subject: oneLine(`[Homelab] ${SEVERITY_LABELS[alert.severity]}: ${alert.message}`, 200), message: lines.join("\n") };
|
|
}
|
|
|
|
export function buildSummaryTicket(alerts: { alert: Alert; firstSeenAt: string }[]): { subject: string; message: string } {
|
|
const lines = [
|
|
`Homelab Manager found ${alerts.length} problems at once. They are in one ticket so a single outage doesn't fill the queue with dozens.`,
|
|
"",
|
|
...alerts.map(({ alert }) => `- [${SEVERITY_LABELS[alert.severity]}] ${alert.source}: ${alert.message}`),
|
|
"",
|
|
`Everything that is wrong right now: ${absolute("/alerts")}`,
|
|
"",
|
|
"Opened automatically by Homelab Manager. It is not updated or closed automatically — close it here when it is dealt with.",
|
|
];
|
|
return { subject: `[Homelab] ${alerts.length} problems detected`, message: lines.join("\n") };
|
|
}
|
|
|
|
// ─── Doing ──────────────────────────────────────────────────────────────────
|
|
|
|
export interface RunSummary {
|
|
at: string;
|
|
skipped?: "disabled" | "busy";
|
|
/** Why nothing could be done (missing setup, osTicket refusing). */
|
|
error?: string;
|
|
/** Tickets opened in osTicket (a summary ticket counts as one). */
|
|
ticketsOpened: number;
|
|
/** Alerts those tickets cover. */
|
|
alertsCovered: number;
|
|
/** Problems recorded without a ticket because they were already there when this was switched on. */
|
|
baseline: number;
|
|
/** Problems seen but not yet in enough checks in a row. */
|
|
pending: number;
|
|
cleared: number;
|
|
/** Tickets that couldn't be opened this time; they are tried again at the next check. */
|
|
failed: number;
|
|
}
|
|
|
|
export interface RunDeps {
|
|
getReport(): Promise<Report>;
|
|
createTicket(ticket: NewTicket): Promise<CreatedTicket>;
|
|
now(): Date;
|
|
}
|
|
|
|
let running = false;
|
|
|
|
const empty = (now: Date): RunSummary => ({ at: now.toISOString(), ticketsOpened: 0, alertsCovered: 0, baseline: 0, pending: 0, cleared: 0, failed: 0 });
|
|
|
|
/** The osTicket integration to use, with what it needs to open tickets. Throws a sentence an admin can act on. */
|
|
async function ticketClient(settings: AlertTicketSettings) {
|
|
if (!settings.integrationId) throw new Error("No osTicket integration is chosen for tickets.");
|
|
const loaded = await loadIntegrationConfig(settings.integrationId);
|
|
if (!loaded || loaded.integration.type !== "osticket") throw new Error("The chosen osTicket integration no longer exists.");
|
|
if (!loaded.integration.enabled) throw new Error("The chosen osTicket integration is switched off.");
|
|
const config = loaded.config as OsTicketApiConfig;
|
|
if (!hasTicketApi(config)) throw new Error("The osTicket integration has no ticket API address and key. Edit it under Integrations and fill them in.");
|
|
if (!settings.requesterEmail.trim()) throw new Error("No requester email is set — osTicket needs one for every ticket.");
|
|
return createOsTicketTicketApi(config);
|
|
}
|
|
|
|
async function latestRows(now: Date, reopenAfterMinutes: number): Promise<TicketRow[]> {
|
|
const since = new Date(now.getTime() - Math.max(reopenAfterMinutes, 60) * 60_000).toISOString();
|
|
const rows = await db
|
|
.select()
|
|
.from(alertTickets)
|
|
.where(or(isNull(alertTickets.clearedAt), gte(alertTickets.clearedAt, since)))
|
|
.orderBy(desc(alertTickets.id));
|
|
const latest = new Map<string, TicketRow>();
|
|
for (const row of rows) if (!latest.has(row.alertKey)) latest.set(row.alertKey, row);
|
|
return [...latest.values()];
|
|
}
|
|
|
|
export async function runAlertTicketCheck(options: { actor?: CurrentUser; deps?: Partial<RunDeps> } = {}): Promise<RunSummary> {
|
|
const nowFn = options.deps?.now ?? (() => new Date());
|
|
const now = nowFn();
|
|
const summary = empty(now);
|
|
|
|
const { alertTickets: settings } = await getSettings();
|
|
if (!settings.enabled) {
|
|
// Switched off: the next time it's switched on counts as a fresh start, so what's wrong by then is the baseline.
|
|
await setInternalFlag(BASELINE_FLAG, "");
|
|
return { ...summary, skipped: "disabled" };
|
|
}
|
|
if (running) return { ...summary, skipped: "busy" };
|
|
running = true;
|
|
|
|
try {
|
|
let client: { createTicket(ticket: NewTicket): Promise<CreatedTicket> };
|
|
try {
|
|
client = options.deps?.createTicket ? { createTicket: options.deps.createTicket } : await ticketClient(settings);
|
|
} catch (err) {
|
|
summary.error = err instanceof Error ? err.message : String(err);
|
|
await setInternalFlag(LAST_RUN_FLAG, JSON.stringify(summary));
|
|
return summary;
|
|
}
|
|
|
|
const report = await (options.deps?.getReport ?? (() => getAlerts(true)))();
|
|
const complete = report.couldntCheck.length === 0;
|
|
const baseline = settings.existingProblems === "ignore" && (await getInternalFlag(BASELINE_FLAG)) !== "1";
|
|
const rows = await latestRows(now, settings.reopenAfterMinutes);
|
|
const plan = planTicketRun({ alerts: report.alerts, rows, settings, now, baseline, complete });
|
|
const iso = now.toISOString();
|
|
|
|
for (const op of plan.ops) {
|
|
switch (op.kind) {
|
|
case "insertPending":
|
|
case "insertBaseline":
|
|
await db.insert(alertTickets).values({
|
|
alertKey: op.alert.id,
|
|
status: op.kind === "insertPending" ? "pending" : "baseline",
|
|
seenCount: 1,
|
|
integrationId: settings.integrationId,
|
|
severity: op.alert.severity,
|
|
message: op.alert.message,
|
|
firstSeenAt: iso,
|
|
lastSeenAt: iso,
|
|
});
|
|
if (op.kind === "insertPending") summary.pending++;
|
|
else summary.baseline++;
|
|
break;
|
|
case "bump":
|
|
await db.update(alertTickets).set({ seenCount: op.row.seenCount + 1, lastSeenAt: iso, severity: op.alert.severity, message: op.alert.message }).where(eq(alertTickets.id, op.row.id));
|
|
summary.pending++;
|
|
break;
|
|
case "touch":
|
|
await db.update(alertTickets).set({ lastSeenAt: iso }).where(eq(alertTickets.id, op.row.id));
|
|
if (op.row.status === "pending") summary.pending++;
|
|
break;
|
|
case "reactivate":
|
|
await db.update(alertTickets).set({ clearedAt: null, lastSeenAt: iso }).where(eq(alertTickets.id, op.row.id));
|
|
break;
|
|
case "clear":
|
|
await db.update(alertTickets).set({ clearedAt: iso }).where(eq(alertTickets.id, op.row.id));
|
|
summary.cleared++;
|
|
break;
|
|
case "dropPending":
|
|
await db.delete(alertTickets).where(eq(alertTickets.id, op.row.id));
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Open the tickets: one each, or — when a lot appear at once — one for all of them.
|
|
const requester = { name: settings.requesterName.trim() || "Homelab Manager", email: settings.requesterEmail.trim() };
|
|
const common = { ...requester, alertStaff: settings.notifyStaff, topicId: settings.topicId, priorityId: settings.priorityId };
|
|
const firstSeen = (c: Candidate) => c.row?.firstSeenAt ?? iso;
|
|
|
|
const groups: Candidate[][] = plan.toTicket.length > settings.summaryAbove ? [plan.toTicket] : plan.toTicket.map((c) => [c]);
|
|
for (const group of groups) {
|
|
const content =
|
|
group.length === 1 ? buildTicket(group[0].alert, firstSeen(group[0])) : buildSummaryTicket(group.map((c) => ({ alert: c.alert, firstSeenAt: firstSeen(c) })));
|
|
let created: CreatedTicket;
|
|
try {
|
|
created = await client.createTicket({ ...common, ...content });
|
|
} catch (err) {
|
|
summary.failed += group.length;
|
|
summary.error ??= err instanceof Error ? err.message : String(err);
|
|
console.error("[alertTickets] couldn't open a ticket:", err instanceof Error ? err.message : err);
|
|
continue; // nothing was recorded, so the next check tries again
|
|
}
|
|
const grouped = group.length > 1;
|
|
for (const c of group) {
|
|
const values = { status: "open" as const, ticketNumber: created.number, grouped, ticketedAt: iso, lastSeenAt: iso, severity: c.alert.severity, message: c.alert.message, integrationId: settings.integrationId };
|
|
if (c.row) await db.update(alertTickets).set(values).where(eq(alertTickets.id, c.row.id));
|
|
else await db.insert(alertTickets).values({ alertKey: c.alert.id, seenCount: 1, firstSeenAt: iso, ...values });
|
|
await recordAudit({
|
|
actor: options.actor,
|
|
category: "alert_ticket",
|
|
action: "create",
|
|
targetType: "alert",
|
|
targetId: c.alert.id,
|
|
detail: { ticket: created.number, severity: c.alert.severity, ...(grouped ? { summaryOf: group.length } : {}), integrationId: settings.integrationId },
|
|
});
|
|
}
|
|
summary.ticketsOpened++;
|
|
summary.alertsCovered += group.length;
|
|
}
|
|
|
|
// The baseline is only complete if everything could be read: a source that was down would otherwise hide problems that then
|
|
// look "new" — and get tickets — once it comes back.
|
|
if (baseline && complete) await setInternalFlag(BASELINE_FLAG, "1");
|
|
|
|
await db.delete(alertTickets).where(and(lt(alertTickets.clearedAt, new Date(now.getTime() - KEEP_CLEARED_DAYS * 86_400_000).toISOString())));
|
|
await setInternalFlag(LAST_RUN_FLAG, JSON.stringify(summary));
|
|
return summary;
|
|
} finally {
|
|
running = false;
|
|
}
|
|
}
|
|
|
|
/** The next check starts afresh: it records what is wrong at that moment as the baseline instead of opening tickets for it. */
|
|
export async function resetBaseline(): Promise<void> {
|
|
await setInternalFlag(BASELINE_FLAG, "");
|
|
}
|
|
|
|
/** A ticket that says it is a test, to check the address, the key and where tickets land — without waiting for a real problem. */
|
|
export async function sendTestTicket(actor: CurrentUser): Promise<CreatedTicket> {
|
|
const { alertTickets: settings } = await getSettings();
|
|
const client = await ticketClient(settings);
|
|
const created = await client.createTicket({
|
|
name: settings.requesterName.trim() || "Homelab Manager",
|
|
email: settings.requesterEmail.trim(),
|
|
alertStaff: settings.notifyStaff,
|
|
topicId: settings.topicId,
|
|
priorityId: settings.priorityId,
|
|
subject: "[Homelab] Test ticket from Homelab Manager",
|
|
message: `This is a test. ${actor.name ?? actor.email ?? "An administrator"} asked Homelab Manager to open it from Settings → Alert tickets, to check that tickets can be created here.\n\nNothing is wrong. You can close this ticket.\n\n${absolute("/settings/alert-tickets")}`,
|
|
});
|
|
await recordAudit({ actor, category: "alert_ticket", action: "test", targetType: "integration", targetId: settings.integrationId ?? undefined, detail: { ticket: created.number } });
|
|
return created;
|
|
}
|
|
|
|
// ─── What the pages show ────────────────────────────────────────────────────
|
|
|
|
export interface AlertTicketInfo {
|
|
number: string | null;
|
|
/** The ticket in osTicket's staff panel, when the address is known. */
|
|
url: string | null;
|
|
/** Part of a summary ticket that covers several alerts. */
|
|
grouped: boolean;
|
|
openedAt: string | null;
|
|
}
|
|
|
|
/** The ticket opened for each of these alerts that has one and is still active, by alert id. */
|
|
export async function ticketsForAlerts(alertIds: string[]): Promise<Map<string, AlertTicketInfo>> {
|
|
const result = new Map<string, AlertTicketInfo>();
|
|
if (alertIds.length === 0) return result;
|
|
const rows = await db
|
|
.select()
|
|
.from(alertTickets)
|
|
.where(and(inArray(alertTickets.alertKey, alertIds), eq(alertTickets.status, "open"), isNull(alertTickets.clearedAt)))
|
|
.orderBy(desc(alertTickets.id));
|
|
|
|
const apiUrls = new Map<number, string | null>();
|
|
async function apiUrlOf(integrationId: number | null): Promise<string | null> {
|
|
if (integrationId === null) return null;
|
|
if (!apiUrls.has(integrationId)) {
|
|
const [row] = await db.select({ config: integrations.config }).from(integrations).where(eq(integrations.id, integrationId)).limit(1);
|
|
let url: string | null = null;
|
|
try {
|
|
const parsed = row?.config ? (JSON.parse(row.config) as { apiUrl?: unknown }) : {};
|
|
url = typeof parsed.apiUrl === "string" && parsed.apiUrl.trim() ? parsed.apiUrl : null;
|
|
} catch {
|
|
url = null;
|
|
}
|
|
apiUrls.set(integrationId, url);
|
|
}
|
|
return apiUrls.get(integrationId) ?? null;
|
|
}
|
|
|
|
for (const row of rows) {
|
|
if (result.has(row.alertKey)) continue;
|
|
const base = await apiUrlOf(row.integrationId);
|
|
result.set(row.alertKey, { number: row.ticketNumber, url: base && row.ticketNumber ? ticketUrl(base, row.ticketNumber) : null, grouped: row.grouped, openedAt: row.ticketedAt });
|
|
}
|
|
return result;
|
|
}
|
|
|
|
export interface AlertTicketStatus {
|
|
lastRun: RunSummary | null;
|
|
/** Active problems with a ticket / waiting to last long enough / already there when this was switched on. */
|
|
open: number;
|
|
pending: number;
|
|
baseline: number;
|
|
/** The next check will record what's wrong as the baseline instead of opening tickets for it. */
|
|
baselinePending: boolean;
|
|
}
|
|
|
|
export async function getAlertTicketStatus(): Promise<AlertTicketStatus> {
|
|
const active = await db.select({ status: alertTickets.status }).from(alertTickets).where(isNull(alertTickets.clearedAt));
|
|
let lastRun: RunSummary | null = null;
|
|
try {
|
|
const raw = await getInternalFlag(LAST_RUN_FLAG);
|
|
lastRun = raw ? (JSON.parse(raw) as RunSummary) : null;
|
|
} catch {
|
|
lastRun = null;
|
|
}
|
|
const { alertTickets: settings } = await getSettings();
|
|
return {
|
|
lastRun,
|
|
open: active.filter((r) => r.status === "open").length,
|
|
pending: active.filter((r) => r.status === "pending").length,
|
|
baseline: active.filter((r) => r.status === "baseline").length,
|
|
baselinePending: settings.enabled && settings.existingProblems === "ignore" && (await getInternalFlag(BASELINE_FLAG)) !== "1",
|
|
};
|
|
}
|