Files
Homelab-manager/server/src/services/alerts.ts
T
bobbanandClaude Sonnet 5.5 9caa414829 Dependency map, impact preview in maintenance, and a calmer Alerts page
- Map: a Dependencies view showing what would go down with a picked box and what it relies on
- Maintenance: lists what depends on a target and can silence those too (same time, audited); the recurring form and update drawer show the same note
- Alerts: a headline on where things stand, held-back alerts nested under their cause, severity edge on rows; caused-by now carries the explaining alert's id

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-18 19:17:09 +02:00

525 lines
26 KiB
TypeScript

/**
* Everything that's wrong right now, in one list — the Alerts page. Nothing here decides what counts as a problem: it asks
* the same checks that send the notifications (health, backups, updates, expiry, automation, ...) and turns what they find
* into a flat list, so the page and the notifications can't disagree. Unlike the notifications it ignores the per-event
* on/off toggles (the page is for looking at, not for being interrupted by) and keeps problems that are under a
* maintenance window, flagged as silenced rather than dropped.
*
* It reads live (a handful of API calls per integration), so a result is kept for a short while rather than re-run for
* every viewer, and every source has a time limit so one hung integration can't hang the page. A source that can't be
* read is reported as such — silence from it must never look like "all clear".
*/
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, secrets, servers } from "../db/schema.js";
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
import { recordFromReport } from "./incidents.js";
import { reconcileAcks } from "./alertAcks.js";
import { lastAttempt as lastSelfBackup, newestSnapshot, STALE_AFTER_HOURS as SELF_BACKUP_STALE_HOURS } from "./selfBackup.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createUptimeKumaAdapter } from "../integrations/uptimekuma/adapter.js";
import { createOsTicketAdapter } from "../integrations/osticket/adapter.js";
import { activeSubjects, isSourceInMaintenance, subjectOfConditionKey } from "./maintenance.js";
import { collectSnapshot, evaluateHealth, startupGraceRemainingMs } from "./healthMonitor.js";
import { collectAutomation, evaluateAutomation } from "./automationMonitor.js";
import { collectDockerUpdates } from "./dockerUpdateScheduler.js";
import { collectProxmoxBackupProblems } from "./proxmoxBackupScheduler.js";
import { collectPbsProblems } from "./pbsVerificationScheduler.js";
import { collectTailscaleKeyExpiries } from "./tailscaleKeyExpiryScheduler.js";
import { collectDomainAlerts } from "./domainMonitor.js";
import { computeSecretStatus } from "./secretStatus.js";
import { getFailingSources } from "./integrationHealthMonitor.js";
import { getSettings } from "./settingsStore.js";
import { sourceLabel } from "./notify.js";
import { alertIdOfNode, dependencyCover, integrationTypeCause, takenByType } from "./dependencies.js";
import type { Alert, AlertCategory, AlertSeverity, SourceFailure } from "./alertTypes.js";
export interface AlertsReport {
alerts: Alert[];
/** Active problems by severity — those under a maintenance window are counted apart, not in these. */
counts: { critical: number; warning: number; info: number; silenced: number; /** Held back because another problem explains them (a guest down because its host is). */ grouped: number };
/** Things that couldn't be checked, and which checks that leaves blind. */
couldntCheck: { name: string; error: string; affects: string[] }[];
/** Context worth knowing about how complete the picture is. */
notes: string[];
generatedAt: string;
}
/** Where each kind of source lives in the app, and what to call it. */
const SOURCES: Record<string, { label: string; link: string }> = {
server: { label: "Server", link: "/servers" },
proxmox: { label: "Proxmox", link: "/proxmox" },
synology: { label: "Synology", link: "/synology" },
semaphore: { label: "Semaphore", link: "/semaphore" },
gitea: { label: "Gitea", link: "/gitea" },
dockhand: { label: "Docker", link: "/docker" },
tailscale: { label: "Tailscale", link: "/tailscale" },
homelab: { label: "Homelab Manager", link: "/settings/backup" },
pbs: { label: "Proxmox Backup", link: "/pbs" },
uptimekuma: { label: "Uptime Kuma", link: "/uptime-kuma" },
osticket: { label: "osTicket", link: "/osticket" },
secrets: { label: "Secrets", link: "/secrets" },
domains: { label: "Domains", link: "/domains" },
};
const SOURCE_TIMEOUT_MS = 20_000;
/** How long a result is reused. */
const CACHE_MS = 60_000;
/** Pressing Refresh over and over shouldn't hammer every integration — a result younger than this is reused even then. */
const MIN_REFRESH_MS = 10_000;
const SEVERITY_ORDER: Record<AlertSeverity, number> = { critical: 0, warning: 1, info: 2 };
/** Which kind of source, and which one, a health/automation condition key belongs to. */
function originOfConditionKey(key: string): { type: string; id: number | null; category: AlertCategory } {
let m = /^(?:offline|disk|failed-units|smart|zfs):server:(\d+)/.exec(key);
if (m) return { type: "server", id: Number(m[1]), category: key.startsWith("offline") ? "offline" : key.startsWith("failed-units") ? "services" : "disk" };
m = /^disk:proxmox:(\d+):/.exec(key);
if (m) return { type: "proxmox", id: Number(m[1]), category: "disk" };
m = /^(?:synology-volume|synology-disk|disk:synology):(\d+):/.exec(key);
if (m) return { type: "synology", id: Number(m[1]), category: "disk" };
m = /^automation:(semaphore|gitea):(\d+):/.exec(key);
if (m) return { type: m[1], id: Number(m[2]), category: "automation" };
return { type: "server", id: null, category: "disk" };
}
function withTimeout<T>(work: Promise<T>): Promise<T> {
let timer: ReturnType<typeof setTimeout>;
const limit = new Promise<never>((_, reject) => {
timer = setTimeout(() => reject(new Error(`timed out after ${SOURCE_TIMEOUT_MS / 1000}s`)), SOURCE_TIMEOUT_MS);
});
return Promise.race([work, limit]).finally(() => clearTimeout(timer));
}
const errorText = (err: unknown) => (err instanceof Error ? err.message : String(err));
const plural = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
export async function collectAlerts(): Promise<AlertsReport> {
const alerts: Alert[] = [];
const notes: string[] = [];
const blind = new Map<string, { error: string; affects: Set<string> }>();
const subjects = await activeSubjects();
const intRows = await db.select({ id: integrations.id, name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations);
const intById = new Map(intRows.map((r) => [r.id, r]));
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((r) => [r.id, r.name]));
/** The thing a maintenance window would go on to silence an alert about `subject` ("server:3", "integration:1"). */
const targetOf = (type: "server" | "integration" | "dns_provider", id: number): Alert["target"] => ({
type,
id,
name: (type === "server" ? serverNames.get(id) : type === "integration" ? intById.get(id)?.name : undefined) ?? `${type} #${id}`,
});
const targetOfSubject = (subject: string | null): Alert["target"] => {
const m = /^(server|integration|dns_provider):(\d+)$/.exec(subject ?? "");
return m ? targetOf(m[1] as "server" | "integration" | "dns_provider", Number(m[2])) : null;
};
const integrationTarget = (id: number): Alert["target"] => targetOf("integration", id);
/** Integration-down alerts are per kind of service; a window on any integration of that kind silences them, so offer the first. */
const firstIntegrationOfType = (type: string): Alert["target"] => {
const row = intRows.find((r) => r.type === type && r.enabled);
return row ? integrationTarget(row.id) : null;
};
function push(a: { category: AlertCategory; severity: AlertSeverity; type: string; message: string; key: string; link?: string | null; silenced?: boolean; target?: Alert["target"]; causedBy?: Alert["causedBy"]; affects?: string[] }) {
const meta = SOURCES[a.type];
alerts.push({
id: `${a.category}:${a.key}`,
severity: a.severity,
category: a.category,
source: meta?.label ?? sourceLabel(a.type),
message: a.message,
link: a.link === undefined ? (meta?.link ?? null) : a.link,
silenced: !!a.silenced || !!a.causedBy,
target: a.target ?? null,
...(a.causedBy ? { causedBy: a.causedBy } : {}),
...(a.affects?.length ? { affects: a.affects } : {}),
});
}
function cannotCheck(name: string, error: string, check: string) {
const entry = blind.get(name) ?? { error, affects: new Set<string>() };
entry.affects.add(check);
blind.set(name, entry);
}
const cannotCheckIntegration = (f: SourceFailure, check: string) => {
const row = intById.get(f.integrationId);
cannotCheck(`${row ? (SOURCES[row.type]?.label ?? row.type) : "Integration"} “${f.integrationName}”`, f.message, check);
};
const silencedIntegration = (id: number) => subjects.has(`integration:${id}`);
// One entry per check. A check that blows up, or hangs, only costs its own section of the page.
async function check(name: string, work: () => Promise<void>) {
try {
await withTimeout(work());
} catch (err) {
cannotCheck(name, errorText(err), name);
}
}
// The servers found to be offline, for the integration failures that one may explain: the server check fills it in (and it is empty if that check fails).
let offlineKnown!: (ids: Set<number>) => void;
const offlineServers = new Promise<Set<number>>((resolve) => (offlineKnown = resolve));
await Promise.all([
check("Server and storage health", async () => {
const { healthChecks } = await getSettings();
const { snapshot, held } = await collectSnapshot();
const now = Date.now();
const grace = startupGraceRemainingMs(now);
if (grace > 0) {
notes.push(
`Server-offline and server-disk checks are paused for another ${Math.ceil(grace / 60_000)} min after the app restarted, so agents get a chance to report before any server is judged.`,
);
}
const conditions = evaluateHealth(snapshot, healthChecks, now, { skipServers: grace > 0 });
// A server that is down because the one it runs on is down is folded under that one.
const offlineIds = new Set<number>();
for (const c of conditions) {
const m = /^offline:server:(\d+)$/.exec(c.key);
if (m) offlineIds.add(Number(m[1]));
}
// (Or because of a service it needs that is failing, or a server it needs that needs one.)
offlineKnown(offlineIds);
const cover = await dependencyCover(offlineIds, getFailingSources().map((f) => f.source));
for (const c of conditions) {
const origin = originOfConditionKey(c.key);
const subject = subjectOfConditionKey(c.key);
const offlineId = /^offline:server:(\d+)$/.exec(c.key)?.[1];
const parentOf = offlineId ? cover.cause.get(`server:${offlineId}`) : undefined;
push({
causedBy: parentOf !== undefined ? { name: cover.graph.names.get(parentOf) ?? parentOf, alertId: alertIdOfNode(parentOf, cover.graph) ?? undefined } : undefined,
affects: offlineId ? cover.taken.get(`server:${offlineId}`)?.sort((a, b) => a.localeCompare(b)) : undefined,
category: origin.category,
severity: c.severity ?? "warning",
type: origin.type,
message: c.message,
key: c.key,
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}` : undefined,
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
});
}
// Integrations the check couldn't read this time. (A whole-integration entry covers its nodes, so skip those.)
for (const h of held) {
const m = /^(proxmox|synology):(\d+)(?::(.+))?$/.exec(h);
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
const row = intById.get(Number(m[2]));
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
}
}).finally(() => offlineKnown(new Set())),
check("Automation runs", async () => {
const { items, held } = await collectAutomation();
for (const c of evaluateAutomation(items).conditions) {
const origin = originOfConditionKey(c.key);
const subject = subjectOfConditionKey(c.key);
push({ category: "automation", severity: "warning", type: origin.type, message: c.message, key: c.key, silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject) });
}
for (const h of held) {
const m = /^(semaphore|gitea):(\d+)$/.exec(h);
if (!m) continue;
const row = intById.get(Number(m[2]));
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”`, "couldn't be read — see the Diagnostic Log", "Automation runs");
}
}),
check("Proxmox backups", async () => {
const { failures, uncovered, sourceFailures } = await collectProxmoxBackupProblems({ skipSilenced: false });
for (const f of failures) {
push({
category: "backup",
severity: "critical",
type: "proxmox",
message: `Latest backup on ${f.node}${f.guestId ? ` (guest ${f.guestId})` : ""} didn't succeed [${f.integrationName}]: ${f.status}`,
key: `proxmox-backup:${f.integrationId}:${f.node}`,
silenced: f.silenced,
target: integrationTarget(f.integrationId),
});
}
for (const u of uncovered) {
push({
category: "backup",
severity: "warning",
type: "proxmox",
message: `${u.guestName} (#${u.vmid}) on ${u.node} isn't covered by any backup job [${u.integrationName}]`,
key: `proxmox-uncovered:${u.integrationId}:${u.vmid}`,
silenced: u.silenced,
target: integrationTarget(u.integrationId),
});
}
sourceFailures.forEach((f) => cannotCheckIntegration(f, "Proxmox backups"));
}),
// A guest whose newest backup is older than its limit — which Proxmox's own "latest run failed" alert can't see, because that only
// looks at the most recent run per node.
check("Backup coverage", async () => {
const now = new Date();
const report = await getBackupCoverage({ now, rules: true });
for (const s of staleBackups(report, now, subjects)) {
push({
category: "backup",
severity: s.severity,
type: "proxmox",
message: s.message,
key: s.key,
link: "/backups",
silenced: s.silenced,
target: s.guest.server ? targetOf("server", s.guest.server.id) : integrationTarget(s.guest.integrationId),
});
}
}),
// Homelab Manager's own backups: if they are switched on, there should be a recent one. This is what turns a backup that has quietly
// stopped working — a full disk, a folder that went read-only — into something somebody hears about before they need it.
check("Homelab Manager's own backups", async () => {
const { selfBackup } = await getSettings();
if (!selfBackup.enabled) return;
const newest = newestSnapshot();
const last = await lastSelfBackup();
const failing = last && !last.ok ? ` The last attempt failed: ${last.error}` : "";
if (!newest) {
if (failing) push({ category: "backup", severity: "warning", type: "homelab", message: `Homelab Manager has no backup of its own yet.${failing}`, key: "self-backup" });
return;
}
const ageHours = (Date.now() - Date.parse(newest.at)) / 3600_000;
if (ageHours <= SELF_BACKUP_STALE_HOURS) return;
push({
category: "backup",
severity: ageHours >= 24 * 7 ? "critical" : "warning",
type: "homelab",
message: `The newest backup of Homelab Manager's own database is ${ageHours < 72 ? `${Math.round(ageHours)} h` : `${Math.floor(ageHours / 24)} days`} old.${failing}`,
key: "self-backup",
});
}),
check("Backup verification", async () => {
const { problems, sourceFailures } = await collectPbsProblems({ skipSilenced: false });
for (const p of problems) {
push({
category: "backup",
severity: p.error ? "warning" : "critical",
type: "pbs",
message: p.error
? `Datastore "${p.datastore}" couldn't be read [${p.integrationName}]: ${p.error}`
: `Datastore "${p.datastore}" has ${plural(p.failedCount, "snapshot")} that failed verification [${p.integrationName}]`,
key: `pbs:${p.integrationId}:${p.datastore}`,
silenced: p.silenced,
target: integrationTarget(p.integrationId),
});
}
sourceFailures.forEach((f) => cannotCheckIntegration(f, "Backup verification"));
}),
check("Image updates", async () => {
const { items, failures } = await collectDockerUpdates();
for (const u of items) {
push({
category: "updates",
severity: "info",
type: "dockhand",
message: `${u.containerName} [${u.environmentName}, ${u.integrationName}] has an image update available${u.newerVersion ? ` → ${u.newerVersion}` : ""}`,
key: `docker:${u.integrationId}:${u.environmentName}:${u.containerName}`,
silenced: silencedIntegration(u.integrationId),
target: integrationTarget(u.integrationId),
});
}
failures.forEach((f) => cannotCheckIntegration(f, "Image updates"));
}),
check("Tailscale keys", async () => {
const { items, failures } = await collectTailscaleKeyExpiries();
for (const k of items) {
push({
category: "expiry",
severity: k.daysLeft < 0 ? "critical" : "warning",
type: "tailscale",
message: k.daysLeft < 0 ? `Key for ${k.deviceLabel} [${k.integrationName}] has expired` : `Key for ${k.deviceLabel} [${k.integrationName}] expires in ${plural(k.daysLeft, "day")}`,
key: `tailscale-key:${k.integrationId}:${k.deviceLabel}`,
silenced: silencedIntegration(k.integrationId),
target: integrationTarget(k.integrationId),
});
}
failures.forEach((f) => cannotCheckIntegration(f, "Tailscale keys"));
}),
check("Uptime Kuma", async () => {
for (const row of intRows.filter((r) => r.type === "uptimekuma" && r.enabled)) {
try {
const loaded = await loadIntegrationConfig(row.id);
if (!loaded) continue;
for (const m of await createUptimeKumaAdapter(loaded.config as any).listMonitors()) {
if (m.status !== "down") continue;
push({
category: "monitoring",
severity: "critical",
type: "uptimekuma",
message: `Monitor "${m.name}" is down${m.target ? ` (${m.target}${m.port ? `:${m.port}` : ""})` : ""} [${row.name}]`,
key: `kuma:${row.id}:${m.id}`,
silenced: silencedIntegration(row.id),
target: integrationTarget(row.id),
});
}
} catch (err) {
cannotCheckIntegration({ integrationId: row.id, integrationName: row.name, message: errorText(err) }, "Uptime Kuma");
}
}
}),
check("osTicket", async () => {
for (const row of intRows.filter((r) => r.type === "osticket" && r.enabled)) {
try {
const loaded = await loadIntegrationConfig(row.id);
if (!loaded) continue;
const overdue = (await createOsTicketAdapter(loaded.config as any).listOpenTickets()).filter((t) => t.isOverdue).length;
if (overdue > 0) {
push({
category: "tickets",
severity: "warning",
type: "osticket",
message: `${plural(overdue, "open ticket")} ${overdue === 1 ? "is" : "are"} overdue [${row.name}]`,
key: `osticket:${row.id}`,
silenced: silencedIntegration(row.id),
target: integrationTarget(row.id),
});
}
} catch (err) {
cannotCheckIntegration({ integrationId: row.id, integrationName: row.name, message: errorText(err) }, "osTicket");
}
}
}),
check("Secrets", async () => {
for (const s of await db.select().from(secrets)) {
const status = computeSecretStatus(s.expiryDate, s.warnDays);
if (status.status === "expired") {
push({ category: "expiry", severity: "critical", type: "secrets", message: `Secret "${s.name}" expired ${plural(Math.abs(status.daysLeft), "day")} ago (${s.expiryDate})`, key: `secret:${s.id}` });
} else if (status.status === "expiring") {
push({ category: "expiry", severity: "warning", type: "secrets", message: `Secret "${s.name}" expires in ${plural(status.daysLeft, "day")} (${s.expiryDate})`, key: `secret:${s.id}` });
}
if (s.checkHost && s.lastCheckError) {
push({
category: "expiry",
severity: "warning",
type: "secrets",
message: `Couldn't read the live certificate for "${s.name}" (${s.checkHost}:${s.checkPort ?? 443}) — the expiry shown may be stale: ${s.lastCheckError}`,
key: `secret-check:${s.id}`,
});
}
}
}),
check("Domains", async () => {
const { expiring, staleChecks } = await collectDomainAlerts();
for (const d of expiring) {
push({
category: "expiry",
severity: d.status === "expired" ? "critical" : "warning",
type: "domains",
message: d.status === "expired" ? `Domain ${d.name} expired on ${d.expiresAt}` : `Domain ${d.name} expires in ${plural(d.daysLeft, "day")} (${d.expiresAt})`,
key: `domain:${d.name}`,
});
}
for (const s of staleChecks) {
push({ category: "expiry", severity: "info", type: "domains", message: `Couldn't refresh the registration for ${s.name} — the expiry shown may be stale: ${s.error}`, key: `domain-stale:${s.name}` });
}
}),
// Not a notification: a disk that is on course to fill is something to see coming, so it is shown here, from the server's own history.
check("Disk forecasts", async () => {
const { healthChecks } = await getSettings();
for (const s of await db.select({ id: servers.id, name: servers.name, thresholds: servers.thresholds }).from(servers)) {
// A server may have its own forecast window (0 turns it off for that server), its own limit, and disks it is told to leave alone.
const own = parseThresholds(s.thresholds);
const forecastDays = effectiveThresholds(healthChecks, own).diskForecastDays;
if (!(forecastDays > 0)) continue;
const series = await getServerMetrics(s.id, 14);
for (const f of disksFillingSoon(series, forecastDays, (mount) => diskLimit(own, mount, healthChecks.diskUsagePercent))) {
push({
category: "disk",
severity: f.daysToFull <= 3 ? "critical" : "warning",
type: "server",
message: `Disk ${f.mount} on ${s.name} is filling by about ${f.perDay.toFixed(1)} points a day — full in about ${plural(f.daysToFull, "day")} (${Math.round(f.latest)}% used now)`,
key: `disk-forecast:${s.id}:${f.mount}`,
link: `/servers/${s.id}`,
silenced: subjects.has(`server:${s.id}`),
target: targetOf("server", s.id),
});
}
}
}),
check("Integration failures", async () => {
const { notifications } = await getSettings();
const failing = getFailingSources();
// A failure explained by something it needs that is down is folded under that, like a guest under its host.
const cover = failing.length > 0 ? await dependencyCover(await offlineServers, failing.map((f) => f.source)).catch(() => null) : null;
for (const f of failing) {
const cause = cover ? integrationTypeCause(f.source, cover) : null;
const affects = cover ? takenByType(f.source, cover) : [];
push({
causedBy: cause ? { name: cause.name, alertId: (cover && alertIdOfNode(cause.key, cover.graph)) || undefined } : undefined,
affects: affects.length > 0 ? affects.sort((a, b) => a.localeCompare(b)) : undefined,
category: "integration",
severity: f.alerted ? "critical" : "warning",
type: f.source,
message: `${sourceLabel(f.source)} has failed its last ${plural(f.consecutiveFailures, "call")} in a row${f.alerted ? "" : ` (a notification goes out after ${notifications.integrationFailureThreshold})`} — see the Diagnostic Log`,
key: `failing:${f.source}`,
link: null,
silenced: await isSourceInMaintenance(f.source),
target: firstIntegrationOfType(f.source),
});
}
}),
]);
alerts.sort(
(a, b) =>
Number(a.silenced) - Number(b.silenced) ||
SEVERITY_ORDER[a.severity] - SEVERITY_ORDER[b.severity] ||
a.source.localeCompare(b.source) ||
a.message.localeCompare(b.message),
);
const active = alerts.filter((a) => !a.silenced);
const grouped = alerts.filter((a) => a.causedBy).length;
return {
alerts,
counts: {
critical: active.filter((a) => a.severity === "critical").length,
warning: active.filter((a) => a.severity === "warning").length,
info: active.filter((a) => a.severity === "info").length,
silenced: alerts.length - active.length - grouped,
grouped,
},
couldntCheck: [...blind.entries()].map(([name, v]) => ({ name, error: v.error, affects: [...v.affects] })),
notes,
generatedAt: new Date().toISOString(),
};
}
let cache: { at: number; report: AlertsReport } | null = null;
/** Forget the stored result, so the next look is a fresh one — after something changes what it would say (a maintenance window starting or ending). */
export function invalidateAlertsCache(): void {
cache = null;
}
let inFlight: Promise<AlertsReport> | null = null;
/** The current alerts, reusing a recent result unless `force` asks for a fresh one (and even then not more than once every few seconds). */
export async function getAlerts(force: boolean): Promise<AlertsReport & { cached: boolean }> {
const age = cache ? Date.now() - cache.at : Infinity;
if (cache && age < (force ? MIN_REFRESH_MS : CACHE_MS)) return { ...cache.report, cached: true };
inFlight ??= collectAlerts()
.then((report) => {
cache = { at: Date.now(), report };
// Remembered for the incident record and the acknowledgements — in the background, and never allowed to hold up or break the answer.
void recordFromReport(report).then(() => reconcileAcks(new Set(report.alerts.map((a) => a.id)))).catch((err) => console.error("[alerts] couldn't update the incident record:", err));
return report;
})
.finally(() => {
inFlight = null;
});
return { ...(await inFlight), cached: false };
}