A job whose last three runs in a row all finished with warnings is flagged and raises an alert (on by default, switchable off per job). The size of the destination, and its quota where the storage says, are read from the reports and kept with each run; the Stored column shows the growth and, where the room is known, when it will be full, with an alert inside the disk-forecast window. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
779 lines
43 KiB
TypeScript
779 lines
43 KiB
TypeScript
/**
|
|
* Everything that's wrong right now, in one list — the Alerts page. Nothing here decides what counts as a problem: it asks
|
|
* the same checks that send the notifications (health, backups, updates, expiry, automation, ...) and turns what they find
|
|
* into a flat list, so the page and the notifications can't disagree. Unlike the notifications it ignores the per-event
|
|
* on/off toggles (the page is for looking at, not for being interrupted by) and keeps problems that are under a
|
|
* maintenance window, flagged as silenced rather than dropped.
|
|
*
|
|
* It reads live (a handful of API calls per integration), so a result is kept for a short while rather than re-run for
|
|
* every viewer, and every source has a time limit so one hung integration can't hang the page. A source that can't be
|
|
* read is reported as such — silence from it must never look like "all clear".
|
|
*/
|
|
import { eq } from "drizzle-orm";
|
|
import { db } from "../db/client.js";
|
|
import { endpointChecks, haEntities, haWatched, heartbeats, integrations, networkDevices, secrets, servers } from "../db/schema.js";
|
|
import { disksFillingSoon, getForecastSeries } from "./serverMetrics.js";
|
|
import { storageFillingSoon, storageOverview } from "./storageHistory.js";
|
|
import { daysOfSupportLeft, lifecycleOf } from "./osLifecycle.js";
|
|
import { npmIntegrationIds, npmProblems, readNpm } from "./npmReader.js";
|
|
import { haProblems, haSyncStatuses, isHaFresh } from "./homeAssistantSync.js";
|
|
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
|
|
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
|
|
import { recordFromReport } from "./incidents.js";
|
|
import { reconcileAcks } from "./alertAcks.js";
|
|
import { flappingKeys, keyOfAlertId } from "./flapGuard.js";
|
|
import { loadSnoozes } from "./alertSnoozes.js";
|
|
import { lastAttempt as lastSelfBackup, newestSnapshot, STALE_AFTER_HOURS as SELF_BACKUP_STALE_HOURS } from "./selfBackup.js";
|
|
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
|
import { createUptimeKumaAdapter } from "../integrations/uptimekuma/adapter.js";
|
|
import { createOsTicketAdapter } from "../integrations/osticket/adapter.js";
|
|
import { activeSubjects, isSourceInMaintenance, subjectOfConditionKey } from "./maintenance.js";
|
|
import { collectSnapshot, evaluateHealth, startupGraceRemainingMs } from "./healthMonitor.js";
|
|
import { collectAutomation, evaluateAutomation } from "./automationMonitor.js";
|
|
import { collectDockerUpdates } from "./dockerUpdateScheduler.js";
|
|
import { collectProxmoxBackupProblems } from "./proxmoxBackupScheduler.js";
|
|
import { collectPbsProblems } from "./pbsVerificationScheduler.js";
|
|
import { collectTailscaleKeyExpiries } from "./tailscaleKeyExpiryScheduler.js";
|
|
import { collectDomainAlerts } from "./domainMonitor.js";
|
|
import { computeSecretStatus } from "./secretStatus.js";
|
|
import { getFailingSources } from "./integrationHealthMonitor.js";
|
|
import { getSettings } from "./settingsStore.js";
|
|
import { sourceLabel } from "./notify.js";
|
|
import { alertIdOfNode, dependencyCover, integrationTypeCause, takenByType } from "./dependencies.js";
|
|
import { endpointConditions } from "./endpointChecks.js";
|
|
import { heartbeatConditions } from "./heartbeats.js";
|
|
import { latestTests, restoreAlerts } from "./restoreTests.js";
|
|
import { kindOfKey, type Alert, type AlertCategory, type AlertSeverity, type SourceFailure } from "./alertTypes.js";
|
|
|
|
export interface AlertsReport {
|
|
alerts: Alert[];
|
|
/** Active problems by severity — those under a maintenance window are counted apart, not in these. */
|
|
counts: { critical: number; warning: number; info: number; silenced: number; /** Held back because another problem explains them (a guest down because its host is). */ grouped: number };
|
|
/** Things that couldn't be checked, and which checks that leaves blind. */
|
|
couldntCheck: { name: string; error: string; affects: string[] }[];
|
|
/** Context worth knowing about how complete the picture is. */
|
|
notes: string[];
|
|
generatedAt: string;
|
|
}
|
|
|
|
/** Where each kind of source lives in the app, and what to call it. */
|
|
const SOURCES: Record<string, { label: string; link: string }> = {
|
|
server: { label: "Server", link: "/servers" },
|
|
proxmox: { label: "Proxmox", link: "/proxmox" },
|
|
synology: { label: "Synology", link: "/synology" },
|
|
semaphore: { label: "Semaphore", link: "/semaphore" },
|
|
gitea: { label: "Gitea", link: "/gitea" },
|
|
dockhand: { label: "Docker", link: "/docker" },
|
|
tailscale: { label: "Tailscale", link: "/tailscale" },
|
|
homelab: { label: "Homelab Manager", link: "/settings/data" },
|
|
pbs: { label: "Proxmox Backup", link: "/backups?view=pbs" },
|
|
uptimekuma: { label: "Uptime Kuma", link: "/uptime-kuma" },
|
|
osticket: { label: "osTicket", link: "/osticket" },
|
|
unifi: { label: "UniFi", link: "/network/devices" },
|
|
npm: { label: "Nginx Proxy Manager", link: "/proxy-hosts" },
|
|
homeassistant: { label: "Home Assistant", link: "/home-assistant" },
|
|
ups: { label: "UPS", link: "/ups" },
|
|
slo: { label: "Service level", link: "/alert-settings?view=levels" },
|
|
duplicati: { label: "Duplicati", link: "/duplicati" },
|
|
jellyfin: { label: "Jellyfin", link: "/jellyfin" },
|
|
authentik: { label: "Authentik", link: "/authentik" },
|
|
prometheus: { label: "Prometheus", link: "/prometheus" },
|
|
immich: { label: "Immich", link: "/immich" },
|
|
paperless: { label: "Paperless-ngx", link: "/paperless" },
|
|
n8n: { label: "n8n", link: "/n8n" },
|
|
netbox: { label: "NetBox", link: "/netbox" },
|
|
nextcloud: { label: "Nextcloud", link: "/nextcloud" },
|
|
hetzner: { label: "Hetzner", link: "/hetzner" },
|
|
speedtest: { label: "Speedtest", link: "/speedtest" },
|
|
fragnet: { label: "Fragnet", link: "/game-servers" },
|
|
sonarr: { label: "Sonarr", link: "/media" },
|
|
radarr: { label: "Radarr", link: "/media" },
|
|
lidarr: { label: "Lidarr", link: "/media" },
|
|
readarr: { label: "Readarr", link: "/media" },
|
|
prowlarr: { label: "Prowlarr", link: "/media" },
|
|
hetznerrobot: { label: "Hetzner Robot", link: "/hetzner-robot" },
|
|
qbittorrent: { label: "qBittorrent", link: "/downloads" },
|
|
sabnzbd: { label: "SABnzbd", link: "/downloads" },
|
|
adguard: { label: "AdGuard Home", link: "/adguard" },
|
|
technitium: { label: "Technitium DNS", link: "/technitium" },
|
|
versions: { label: "Versions", link: "/versions" },
|
|
grafana: { label: "Grafana", link: "/grafana" },
|
|
secrets: { label: "Secrets", link: "/secrets" },
|
|
domains: { label: "Domains", link: "/dns?view=domains" },
|
|
endpoint: { label: "Endpoint", link: "/endpoints" },
|
|
heartbeat: { label: "Heartbeat", link: "/heartbeats" },
|
|
};
|
|
|
|
const SOURCE_TIMEOUT_MS = 20_000;
|
|
/** How long a result is reused. */
|
|
const CACHE_MS = 60_000;
|
|
/** Pressing Refresh over and over shouldn't hammer every integration — a result younger than this is reused even then. */
|
|
const MIN_REFRESH_MS = 10_000;
|
|
|
|
const SEVERITY_ORDER: Record<AlertSeverity, number> = { critical: 0, warning: 1, info: 2 };
|
|
|
|
/** Which kind of source, and which one, a health/automation condition key belongs to. */
|
|
function originOfConditionKey(key: string): { type: string; id: number | null; category: AlertCategory } {
|
|
const device = /^offline:device:(\d+)$/.exec(key);
|
|
if (device) return { type: "unifi", id: Number(device[1]), category: "offline" };
|
|
let m = /^rule:\d+:server:(\d+)/.exec(key);
|
|
if (m) return { type: "server", id: Number(m[1]), category: "monitoring" };
|
|
m = /^baseline:(?:cpu|mem):server:(\d+)$/.exec(key);
|
|
if (m) return { type: "server", id: Number(m[1]), category: "monitoring" };
|
|
m = /^(?:offline|disk|failed-units|smart|zfs|new-port):server:(\d+)/.exec(key);
|
|
if (m) return { type: "server", id: Number(m[1]), category: key.startsWith("offline") ? "offline" : key.startsWith("failed-units") ? "services" : key.startsWith("new-port") ? "monitoring" : "disk" };
|
|
m = /^disk:proxmox:(\d+):/.exec(key);
|
|
if (m) return { type: "proxmox", id: Number(m[1]), category: "disk" };
|
|
m = /^(?:synology-volume|synology-disk|disk:synology):(\d+):/.exec(key);
|
|
if (m) return { type: "synology", id: Number(m[1]), category: "disk" };
|
|
m = /^automation:(semaphore|gitea):(\d+):/.exec(key);
|
|
if (m) return { type: m[1], id: Number(m[2]), category: "automation" };
|
|
m = /^ups-(?:battery|low|replace|load):(\d+)$/.exec(key);
|
|
if (m) return { type: "ups", id: Number(m[1]), category: "monitoring" };
|
|
m = /^slo:(\d+)$/.exec(key);
|
|
if (m) return { type: "slo", id: Number(m[1]), category: "monitoring" };
|
|
m = /^container-(?:restarting|unhealthy|stopped|created|host):(\d+):/.exec(key);
|
|
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
|
|
m = /^docker-(?:vuln(?:-high)?|image-age):(\d+):/.exec(key);
|
|
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
|
|
m = /^speedtest-(?:slow|failing):(\d+)/.exec(key);
|
|
if (m) return { type: "speedtest", id: Number(m[1]), category: "monitoring" };
|
|
m = /^hetzner-(off|stuck|rescue|traffic|forecast|backup|lb|cert|fip):(\d+):/.exec(key);
|
|
if (m) return { type: "hetzner", id: Number(m[2]), category: m[1] === "traffic" || m[1] === "forecast" ? "monitoring" : m[1] === "backup" ? "backup" : "services" };
|
|
m = /^version-behind:(\d+)$/.exec(key);
|
|
if (m) return { type: "versions", id: Number(m[1]), category: "updates" };
|
|
m = /^adguard-(stopped|dns|protection|filters):(\d+)/.exec(key);
|
|
if (m) return { type: "adguard", id: Number(m[2]), category: m[1] === "stopped" || m[1] === "dns" ? "services" : "monitoring" };
|
|
m = /^technitium-(dns|blocking|lists|zones|node|config):(\d+)/.exec(key);
|
|
if (m) return { type: "technitium", id: Number(m[2]), category: m[1] === "dns" || m[1] === "zones" || m[1] === "node" ? "services" : "monitoring" };
|
|
m = /^grafana-(firing|rule-error|ds):(\d+):/.exec(key);
|
|
if (m) return { type: "grafana", id: Number(m[2]), category: m[1] === "ds" ? "integration" : "monitoring" };
|
|
m = /^(sonarr|radarr|prowlarr|lidarr|readarr)-(health|queue|disk):(\d+)/.exec(key);
|
|
if (m) return { type: m[1], id: Number(m[3]), category: m[2] === "disk" ? "disk" : "services" };
|
|
m = /^hetznerrobot-(ending|rescue|setup|boxlocked|boxending):(\d+):/.exec(key);
|
|
if (m) return { type: "hetznerrobot", id: Number(m[2]), category: "services" };
|
|
m = /^(qbittorrent|sabnzbd)-(stuck|connection|paused|stalled|disk):(\d+)/.exec(key);
|
|
if (m) return { type: m[1], id: Number(m[3]), category: m[2] === "disk" ? "disk" : "services" };
|
|
m = /^fragnet-(disk|memory|backup):(\d+):/.exec(key);
|
|
if (m) return { type: "fragnet", id: Number(m[2]), category: m[1] === "disk" ? "disk" : m[1] === "backup" ? "backup" : "monitoring" };
|
|
m = /^duplicati-(?:failed|stale|warnings|streak|growth):(\d+)/.exec(key);
|
|
if (m) return { type: "duplicati", id: Number(m[1]), category: "backup" };
|
|
m = /^nextcloud-(?:maintenance|upgrade|disk|cron|checks):(\d+)/.exec(key);
|
|
if (m) return { type: "nextcloud", id: Number(m[1]), category: key.startsWith("nextcloud-disk") ? "disk" : "services" };
|
|
m = /^netbox-(?:workers):(\d+)/.exec(key);
|
|
if (m) return { type: "netbox", id: Number(m[1]), category: "services" };
|
|
m = /^n8n-(?:failed):(\d+)/.exec(key);
|
|
if (m) return { type: "n8n", id: Number(m[1]), category: "automation" };
|
|
m = /^paperless-(?:component|failed|disk):(\d+)/.exec(key);
|
|
if (m) return { type: "paperless", id: Number(m[1]), category: key.startsWith("paperless-disk") ? "disk" : "services" };
|
|
m = /^immich-(?:disk|jobs):(\d+)/.exec(key);
|
|
if (m) return { type: "immich", id: Number(m[1]), category: key.startsWith("immich-disk") ? "disk" : "services" };
|
|
m = /^prometheus-(?:target|alert):(\d+)/.exec(key);
|
|
if (m) return { type: "prometheus", id: Number(m[1]), category: "monitoring" };
|
|
m = /^authentik-(?:outpost|outpost-version|cert|token|worker|logins|errors):(\d+)/.exec(key);
|
|
if (m) return { type: "authentik", id: Number(m[1]), category: /^authentik-(?:cert|token):/.test(key) ? "expiry" : /^authentik-(?:outpost|worker)/.test(key) ? "services" : "monitoring" };
|
|
m = /^jellyfin-(?:restart|task|logins):(\d+)/.exec(key);
|
|
if (m) return { type: "jellyfin", id: Number(m[1]), category: key.startsWith("jellyfin-task") ? "automation" : "monitoring" };
|
|
return { type: "server", id: null, category: "disk" };
|
|
}
|
|
|
|
function withTimeout<T>(work: Promise<T>): Promise<T> {
|
|
let timer: ReturnType<typeof setTimeout>;
|
|
const limit = new Promise<never>((_, reject) => {
|
|
timer = setTimeout(() => reject(new Error(`timed out after ${SOURCE_TIMEOUT_MS / 1000}s`)), SOURCE_TIMEOUT_MS);
|
|
});
|
|
return Promise.race([work, limit]).finally(() => clearTimeout(timer));
|
|
}
|
|
|
|
const errorText = (err: unknown) => (err instanceof Error ? err.message : String(err));
|
|
const plural = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
|
|
|
|
export async function collectAlerts(): Promise<AlertsReport> {
|
|
const alerts: Alert[] = [];
|
|
const notes: string[] = [];
|
|
const blind = new Map<string, { error: string; affects: Set<string> }>();
|
|
const subjects = await activeSubjects();
|
|
const intRows = await db.select({ id: integrations.id, name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations);
|
|
const intById = new Map(intRows.map((r) => [r.id, r]));
|
|
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((r) => [r.id, r.name]));
|
|
const deviceNames = new Map((await db.select({ id: networkDevices.id, name: networkDevices.name }).from(networkDevices)).map((r) => [r.id, r.name]));
|
|
|
|
/** The thing a maintenance window would go on to silence an alert about `subject` ("server:3", "integration:1"). */
|
|
const targetOf = (type: "server" | "integration" | "dns_provider" | "device", id: number): Alert["target"] => ({
|
|
type,
|
|
id,
|
|
name: (type === "server" ? serverNames.get(id) : type === "device" ? deviceNames.get(id) : type === "integration" ? intById.get(id)?.name : undefined) ?? `${type} #${id}`,
|
|
});
|
|
const targetOfSubject = (subject: string | null): Alert["target"] => {
|
|
const m = /^(server|integration|dns_provider|device):(\d+)$/.exec(subject ?? "");
|
|
return m ? targetOf(m[1] as "server" | "integration" | "dns_provider" | "device", Number(m[2])) : null;
|
|
};
|
|
const integrationTarget = (id: number): Alert["target"] => targetOf("integration", id);
|
|
/** Integration-down alerts are per kind of service; a window on any integration of that kind silences them, so offer the first. */
|
|
const firstIntegrationOfType = (type: string): Alert["target"] => {
|
|
const row = intRows.find((r) => r.type === type && r.enabled);
|
|
return row ? integrationTarget(row.id) : null;
|
|
};
|
|
|
|
function push(a: { category: AlertCategory; severity: AlertSeverity; type: string; message: string; key: string; link?: string | null; silenced?: boolean; target?: Alert["target"]; causedBy?: Alert["causedBy"]; affects?: string[] }) {
|
|
const meta = SOURCES[a.type];
|
|
alerts.push({
|
|
id: `${a.category}:${a.key}`,
|
|
kind: kindOfKey(a.key),
|
|
type: a.type,
|
|
severity: a.severity,
|
|
category: a.category,
|
|
source: meta?.label ?? sourceLabel(a.type),
|
|
message: a.message,
|
|
link: a.link === undefined ? (meta?.link ?? null) : a.link,
|
|
silenced: !!a.silenced || !!a.causedBy,
|
|
target: a.target ?? null,
|
|
...(a.causedBy ? { causedBy: a.causedBy } : {}),
|
|
...(a.affects?.length ? { affects: a.affects } : {}),
|
|
});
|
|
}
|
|
|
|
function cannotCheck(name: string, error: string, check: string) {
|
|
const entry = blind.get(name) ?? { error, affects: new Set<string>() };
|
|
entry.affects.add(check);
|
|
blind.set(name, entry);
|
|
}
|
|
const cannotCheckIntegration = (f: SourceFailure, check: string) => {
|
|
const row = intById.get(f.integrationId);
|
|
cannotCheck(`${row ? (SOURCES[row.type]?.label ?? row.type) : "Integration"} “${f.integrationName}”`, f.message, check);
|
|
};
|
|
const silencedIntegration = (id: number) => subjects.has(`integration:${id}`);
|
|
|
|
// One entry per check. A check that blows up, or hangs, only costs its own section of the page.
|
|
async function check(name: string, work: () => Promise<void>) {
|
|
try {
|
|
await withTimeout(work());
|
|
} catch (err) {
|
|
cannotCheck(name, errorText(err), name);
|
|
}
|
|
}
|
|
|
|
// The servers found to be offline, for the integration failures that one may explain: the server check fills it in (and it is empty if that check fails).
|
|
let offlineKnown!: (ids: Set<number>) => void;
|
|
const offlineServers = new Promise<Set<number>>((resolve) => (offlineKnown = resolve));
|
|
// The same for network devices, which can explain a service that has stopped answering.
|
|
let devicesKnown!: (ids: Set<number>) => void;
|
|
const offlineDeviceIds = new Promise<Set<number>>((resolve) => (devicesKnown = resolve));
|
|
|
|
await Promise.all([
|
|
check("Server and storage health", async () => {
|
|
const { healthChecks } = await getSettings();
|
|
const { snapshot, held } = await collectSnapshot();
|
|
const now = Date.now();
|
|
const grace = startupGraceRemainingMs(now);
|
|
if (grace > 0) {
|
|
notes.push(
|
|
`Server-offline and server-disk checks are paused for another ${Math.ceil(grace / 60_000)} min after the app restarted, so agents get a chance to report before any server is judged.`,
|
|
);
|
|
}
|
|
const conditions = evaluateHealth(snapshot, healthChecks, now, { skipServers: grace > 0 });
|
|
// A server that is down because the one it runs on is down is folded under that one.
|
|
const offlineIds = new Set<number>();
|
|
const offlineDevices = new Set<number>();
|
|
for (const c of conditions) {
|
|
const m = /^offline:(server|device):(\d+)$/.exec(c.key);
|
|
if (m) (m[1] === "server" ? offlineIds : offlineDevices).add(Number(m[2]));
|
|
}
|
|
// (Or because of a service it needs that is failing, or a server it needs that needs one.)
|
|
offlineKnown(offlineIds);
|
|
devicesKnown(offlineDevices);
|
|
const cover = await dependencyCover(offlineIds, getFailingSources().map((f) => f.source), offlineDevices);
|
|
for (const c of conditions) {
|
|
const origin = originOfConditionKey(c.key);
|
|
const subject = subjectOfConditionKey(c.key);
|
|
const offlineMatch = /^offline:(server|device):(\d+)$/.exec(c.key);
|
|
const offlineNode = offlineMatch ? `${offlineMatch[1]}:${offlineMatch[2]}` : undefined;
|
|
const parentOf = offlineNode ? cover.cause.get(offlineNode) : undefined;
|
|
const upsOnBattery = /^ups-battery:(\d+)$/.exec(c.key);
|
|
push({
|
|
causedBy: parentOf !== undefined ? { name: cover.graph.names.get(parentOf) ?? parentOf, alertId: alertIdOfNode(parentOf, cover.graph) ?? undefined } : undefined,
|
|
affects: offlineNode ? cover.taken.get(offlineNode)?.sort((a, b) => a.localeCompare(b)) : upsOnBattery ? cover.taken.get(`integration:${upsOnBattery[1]}`)?.sort((a, b) => a.localeCompare(b)) : undefined,
|
|
category: origin.category,
|
|
severity: c.severity ?? "warning",
|
|
type: origin.type,
|
|
message: c.message,
|
|
key: c.key,
|
|
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : origin.type === "nextcloud" ? "/nextcloud" : origin.type === "netbox" ? "/netbox" : origin.type === "n8n" ? "/n8n" : origin.type === "hetzner" ? "/hetzner" : origin.type === "speedtest" ? "/speedtest" : origin.type === "fragnet" ? "/game-servers" : origin.type === "sonarr" || origin.type === "radarr" || origin.type === "prowlarr" || origin.type === "lidarr" || origin.type === "readarr" ? "/media" : origin.type === "adguard" ? "/adguard" : origin.type === "grafana" ? "/grafana" : undefined,
|
|
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
|
|
});
|
|
}
|
|
// Integrations the check couldn't read this time. (A whole-integration entry covers its nodes, so skip those.)
|
|
if (held.has("server:rules")) cannotCheck("Alert rules", "couldn't be judged — see the server log", "Server and storage health");
|
|
if (held.has("duplicati")) cannotCheck("Duplicati backups", "couldn't be judged — see the server log", "Server and storage health");
|
|
if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health");
|
|
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
|
|
for (const h of held) {
|
|
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner|hetznerrobot|speedtest|fragnet|sonarr|radarr|prowlarr|lidarr|readarr|qbittorrent|sabnzbd|adguard|technitium|grafana):(\d+)(?::(.+))?$/.exec(h);
|
|
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
|
|
const row = intById.get(Number(m[2]));
|
|
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] === "vuln" ? " (vulnerability scans)" : m[3] === "images" ? " (age of the images)" : m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
|
|
}
|
|
}).finally(() => {
|
|
offlineKnown(new Set());
|
|
devicesKnown(new Set());
|
|
}),
|
|
|
|
check("Automation runs", async () => {
|
|
const { items, held } = await collectAutomation();
|
|
for (const c of evaluateAutomation(items).conditions) {
|
|
const origin = originOfConditionKey(c.key);
|
|
const subject = subjectOfConditionKey(c.key);
|
|
push({ category: "automation", severity: "warning", type: origin.type, message: c.message, key: c.key, silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject) });
|
|
}
|
|
for (const h of held) {
|
|
const m = /^(semaphore|gitea):(\d+)$/.exec(h);
|
|
if (!m) continue;
|
|
const row = intById.get(Number(m[2]));
|
|
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”`, "couldn't be read — see the Diagnostic Log", "Automation runs");
|
|
}
|
|
}),
|
|
|
|
check("Proxmox backups", async () => {
|
|
const { failures, uncovered, sourceFailures } = await collectProxmoxBackupProblems({ skipSilenced: false });
|
|
for (const f of failures) {
|
|
push({
|
|
category: "backup",
|
|
severity: "critical",
|
|
type: "proxmox",
|
|
message: `Latest backup on ${f.node}${f.guestId ? ` (guest ${f.guestId})` : ""} didn't succeed [${f.integrationName}]: ${f.status}`,
|
|
key: `proxmox-backup:${f.integrationId}:${f.node}`,
|
|
silenced: f.silenced,
|
|
target: integrationTarget(f.integrationId),
|
|
});
|
|
}
|
|
for (const u of uncovered) {
|
|
push({
|
|
category: "backup",
|
|
severity: "warning",
|
|
type: "proxmox",
|
|
message: `${u.guestName} (#${u.vmid}) on ${u.node} isn't covered by any backup job [${u.integrationName}]`,
|
|
key: `proxmox-uncovered:${u.integrationId}:${u.vmid}`,
|
|
silenced: u.silenced,
|
|
target: integrationTarget(u.integrationId),
|
|
});
|
|
}
|
|
sourceFailures.forEach((f) => cannotCheckIntegration(f, "Proxmox backups"));
|
|
}),
|
|
|
|
// A guest whose newest backup is older than its limit — which Proxmox's own "latest run failed" alert can't see, because that only
|
|
// looks at the most recent run per node.
|
|
check("Backup coverage", async () => {
|
|
const now = new Date();
|
|
const report = await getBackupCoverage({ now, rules: true });
|
|
for (const s of staleBackups(report, now, subjects)) {
|
|
push({
|
|
category: "backup",
|
|
severity: s.severity,
|
|
type: "proxmox",
|
|
message: s.message,
|
|
key: s.key,
|
|
link: "/backups",
|
|
silenced: s.silenced,
|
|
target: s.guest.server ? targetOf("server", s.guest.server.id) : integrationTarget(s.guest.integrationId),
|
|
});
|
|
}
|
|
// A restore that failed, or hasn't been tried in too long (the limit is in Settings; a failure is always raised).
|
|
const { healthChecks } = await getSettings();
|
|
for (const r of restoreAlerts(report, await latestTests(), healthChecks.restoreTestDays, now)) {
|
|
const g = r.guest;
|
|
push({
|
|
category: "backup",
|
|
severity: r.severity,
|
|
type: g ? "proxmox" : "homelab",
|
|
message: r.message,
|
|
key: r.key,
|
|
link: "/backups",
|
|
silenced: g ? subjects.has(`integration:${g.integrationId}`) || (g.server !== null && subjects.has(`server:${g.server.id}`)) : false,
|
|
target: g ? (g.server ? targetOf("server", g.server.id) : integrationTarget(g.integrationId)) : null,
|
|
});
|
|
}
|
|
}),
|
|
|
|
// What the endpoint checks have found: an address, port or certificate that has stopped being fine.
|
|
check("Endpoint checks", async () => {
|
|
for (const c of endpointConditions(await db.select().from(endpointChecks), new Date())) {
|
|
push({
|
|
category: c.category,
|
|
severity: c.severity,
|
|
type: "endpoint",
|
|
message: c.message,
|
|
key: c.key,
|
|
link: "/endpoints",
|
|
silenced: c.serverId !== null && subjects.has(`server:${c.serverId}`),
|
|
target: c.serverId !== null ? targetOf("server", c.serverId) : null,
|
|
});
|
|
}
|
|
}),
|
|
|
|
// Jobs that report in: one that has not done so in time, or said it failed.
|
|
check("Heartbeats", async () => {
|
|
for (const c of heartbeatConditions(await db.select().from(heartbeats), new Date())) {
|
|
push({
|
|
category: c.category,
|
|
severity: c.severity,
|
|
type: "heartbeat",
|
|
message: c.message,
|
|
key: c.key,
|
|
link: "/heartbeats",
|
|
silenced: c.serverId !== null && subjects.has(`server:${c.serverId}`),
|
|
target: c.serverId !== null ? targetOf("server", c.serverId) : null,
|
|
});
|
|
}
|
|
}),
|
|
|
|
// Homelab Manager's own backups: if they are switched on, there should be a recent one. This is what turns a backup that has quietly
|
|
// stopped working — a full disk, a folder that went read-only — into something somebody hears about before they need it.
|
|
check("Homelab Manager's own backups", async () => {
|
|
const { selfBackup } = await getSettings();
|
|
if (!selfBackup.enabled) return;
|
|
const newest = newestSnapshot();
|
|
const last = await lastSelfBackup();
|
|
const failing = last && !last.ok ? ` The last attempt failed: ${last.error}` : "";
|
|
if (!newest) {
|
|
if (failing) push({ category: "backup", severity: "warning", type: "homelab", message: `Homelab Manager has no backup of its own yet.${failing}`, key: "self-backup" });
|
|
return;
|
|
}
|
|
const ageHours = (Date.now() - Date.parse(newest.at)) / 3600_000;
|
|
if (ageHours <= SELF_BACKUP_STALE_HOURS) return;
|
|
push({
|
|
category: "backup",
|
|
severity: ageHours >= 24 * 7 ? "critical" : "warning",
|
|
type: "homelab",
|
|
message: `The newest backup of Homelab Manager's own database is ${ageHours < 72 ? `${Math.round(ageHours)} h` : `${Math.floor(ageHours / 24)} days`} old.${failing}`,
|
|
key: "self-backup",
|
|
});
|
|
}),
|
|
|
|
check("Backup verification", async () => {
|
|
const { problems, sourceFailures } = await collectPbsProblems({ skipSilenced: false });
|
|
for (const p of problems) {
|
|
push({
|
|
category: "backup",
|
|
severity: p.error ? "warning" : "critical",
|
|
type: "pbs",
|
|
message: p.error
|
|
? `Datastore "${p.datastore}" couldn't be read [${p.integrationName}]: ${p.error}`
|
|
: `Datastore "${p.datastore}" has ${plural(p.failedCount, "snapshot")} that failed verification [${p.integrationName}]`,
|
|
key: `pbs:${p.integrationId}:${p.datastore}`,
|
|
silenced: p.silenced,
|
|
target: integrationTarget(p.integrationId),
|
|
});
|
|
}
|
|
sourceFailures.forEach((f) => cannotCheckIntegration(f, "Backup verification"));
|
|
}),
|
|
|
|
check("Image updates", async () => {
|
|
const { items, failures } = await collectDockerUpdates();
|
|
for (const u of items) {
|
|
push({
|
|
category: "updates",
|
|
severity: "info",
|
|
type: "dockhand",
|
|
message: `${u.containerName} [${u.environmentName}, ${u.integrationName}] has an image update available${u.newerVersion ? ` → ${u.newerVersion}` : ""}`,
|
|
key: `docker:${u.integrationId}:${u.environmentName}:${u.containerName}`,
|
|
silenced: silencedIntegration(u.integrationId),
|
|
target: integrationTarget(u.integrationId),
|
|
});
|
|
}
|
|
failures.forEach((f) => cannotCheckIntegration(f, "Image updates"));
|
|
}),
|
|
|
|
check("Tailscale keys", async () => {
|
|
const { items, failures } = await collectTailscaleKeyExpiries();
|
|
for (const k of items) {
|
|
push({
|
|
category: "expiry",
|
|
severity: k.daysLeft < 0 ? "critical" : "warning",
|
|
type: "tailscale",
|
|
message: k.daysLeft < 0 ? `Key for ${k.deviceLabel} [${k.integrationName}] has expired` : `Key for ${k.deviceLabel} [${k.integrationName}] expires in ${plural(k.daysLeft, "day")}`,
|
|
key: `tailscale-key:${k.integrationId}:${k.deviceLabel}`,
|
|
silenced: silencedIntegration(k.integrationId),
|
|
target: integrationTarget(k.integrationId),
|
|
});
|
|
}
|
|
failures.forEach((f) => cannotCheckIntegration(f, "Tailscale keys"));
|
|
}),
|
|
|
|
check("Uptime Kuma", async () => {
|
|
for (const row of intRows.filter((r) => r.type === "uptimekuma" && r.enabled)) {
|
|
try {
|
|
const loaded = await loadIntegrationConfig(row.id);
|
|
if (!loaded) continue;
|
|
for (const m of await createUptimeKumaAdapter(loaded.config as any).listMonitors()) {
|
|
if (m.status !== "down") continue;
|
|
push({
|
|
category: "monitoring",
|
|
severity: "critical",
|
|
type: "uptimekuma",
|
|
message: `Monitor "${m.name}" is down${m.target ? ` (${m.target}${m.port ? `:${m.port}` : ""})` : ""} [${row.name}]`,
|
|
key: `kuma:${row.id}:${m.id}`,
|
|
silenced: silencedIntegration(row.id),
|
|
target: integrationTarget(row.id),
|
|
});
|
|
}
|
|
} catch (err) {
|
|
cannotCheckIntegration({ integrationId: row.id, integrationName: row.name, message: errorText(err) }, "Uptime Kuma");
|
|
}
|
|
}
|
|
}),
|
|
|
|
check("osTicket", async () => {
|
|
for (const row of intRows.filter((r) => r.type === "osticket" && r.enabled)) {
|
|
try {
|
|
const loaded = await loadIntegrationConfig(row.id);
|
|
if (!loaded) continue;
|
|
const overdue = (await createOsTicketAdapter(loaded.config as any).listOpenTickets()).filter((t) => t.isOverdue).length;
|
|
if (overdue > 0) {
|
|
push({
|
|
category: "tickets",
|
|
severity: "warning",
|
|
type: "osticket",
|
|
message: `${plural(overdue, "open ticket")} ${overdue === 1 ? "is" : "are"} overdue [${row.name}]`,
|
|
key: `osticket:${row.id}`,
|
|
silenced: silencedIntegration(row.id),
|
|
target: integrationTarget(row.id),
|
|
});
|
|
}
|
|
} catch (err) {
|
|
cannotCheckIntegration({ integrationId: row.id, integrationName: row.name, message: errorText(err) }, "osTicket");
|
|
}
|
|
}
|
|
}),
|
|
|
|
check("Secrets", async () => {
|
|
for (const s of await db.select().from(secrets)) {
|
|
const status = computeSecretStatus(s.expiryDate, s.warnDays);
|
|
if (status.status === "expired") {
|
|
push({ category: "expiry", severity: "critical", type: "secrets", message: `Secret "${s.name}" expired ${plural(Math.abs(status.daysLeft), "day")} ago (${s.expiryDate})`, key: `secret:${s.id}` });
|
|
} else if (status.status === "expiring") {
|
|
push({ category: "expiry", severity: "warning", type: "secrets", message: `Secret "${s.name}" expires in ${plural(status.daysLeft, "day")} (${s.expiryDate})`, key: `secret:${s.id}` });
|
|
}
|
|
if (s.checkHost && s.lastCheckError) {
|
|
push({
|
|
category: "expiry",
|
|
severity: "warning",
|
|
type: "secrets",
|
|
message: `Couldn't read the live certificate for "${s.name}" (${s.checkHost}:${s.checkPort ?? 443}) — the expiry shown may be stale: ${s.lastCheckError}`,
|
|
key: `secret-check:${s.id}`,
|
|
});
|
|
}
|
|
}
|
|
}),
|
|
|
|
check("Domains", async () => {
|
|
const { expiring, staleChecks } = await collectDomainAlerts();
|
|
for (const d of expiring) {
|
|
push({
|
|
category: "expiry",
|
|
severity: d.status === "expired" ? "critical" : "warning",
|
|
type: "domains",
|
|
message: d.status === "expired" ? `Domain ${d.name} expired on ${d.expiresAt}` : `Domain ${d.name} expires in ${plural(d.daysLeft, "day")} (${d.expiresAt})`,
|
|
key: `domain:${d.name}`,
|
|
});
|
|
}
|
|
for (const s of staleChecks) {
|
|
push({ category: "expiry", severity: "info", type: "domains", message: `Couldn't refresh the registration for ${s.name} — the expiry shown may be stale: ${s.error}`, key: `domain-stale:${s.name}` });
|
|
}
|
|
}),
|
|
|
|
// Not a notification: a disk that is on course to fill is something to see coming, so it is shown here, from the server's own history.
|
|
check("Disk forecasts", async () => {
|
|
const { healthChecks } = await getSettings();
|
|
for (const s of await db.select({ id: servers.id, name: servers.name, thresholds: servers.thresholds }).from(servers)) {
|
|
// A server may have its own forecast window (0 turns it off for that server), its own limit, and disks it is told to leave alone.
|
|
const own = parseThresholds(s.thresholds);
|
|
const forecastDays = effectiveThresholds(healthChecks, own).diskForecastDays;
|
|
if (!(forecastDays > 0)) continue;
|
|
const series = await getForecastSeries(s.id);
|
|
for (const f of disksFillingSoon(series, forecastDays, (mount) => diskLimit(own, mount, healthChecks.diskUsagePercent))) {
|
|
push({
|
|
category: "disk",
|
|
severity: f.daysToFull <= 3 ? "critical" : "warning",
|
|
type: "server",
|
|
message: `Disk ${f.mount} on ${s.name} is filling by about ${f.perDay.toFixed(1)} points a day — full in about ${plural(f.daysToFull, "day")} (${Math.round(f.latest)}% used now)`,
|
|
key: `disk-forecast:${s.id}:${f.mount}`,
|
|
link: `/servers/${s.id}`,
|
|
silenced: subjects.has(`server:${s.id}`),
|
|
target: targetOf("server", s.id),
|
|
});
|
|
}
|
|
}
|
|
}),
|
|
|
|
// The same for the pools the integrations report (a Proxmox storage, a Synology volume), from the history kept on each health pass.
|
|
check("Storage forecasts", async () => {
|
|
const { healthChecks } = await getSettings();
|
|
if (!(healthChecks.diskForecastDays > 0)) return;
|
|
const nowMs = Date.now();
|
|
const rows = (await storageOverview()).filter((r) => nowMs - Date.parse(r.updatedAt) < 2 * 86_400_000 && intById.get(r.integrationId)?.enabled);
|
|
for (const f of storageFillingSoon(rows, healthChecks.diskForecastDays, healthChecks.diskUsagePercent)) {
|
|
const row = intById.get(f.integrationId)!;
|
|
push({
|
|
category: "disk",
|
|
severity: f.daysToFull <= 3 ? "critical" : "warning",
|
|
type: row.type,
|
|
message: `${f.label} is filling by about ${f.perDay.toFixed(1)} points a day — full in about ${plural(f.daysToFull, "day")} (${Math.round(f.latest)}% used now)`,
|
|
key: `storage-forecast:${f.integrationId}:${f.key}`,
|
|
silenced: silencedIntegration(f.integrationId),
|
|
target: integrationTarget(f.integrationId),
|
|
});
|
|
}
|
|
}),
|
|
|
|
// Certificates in use by Nginx Proxy Manager that are running out, and hosts it could not set up.
|
|
check("Nginx Proxy Manager", async () => {
|
|
for (const i of await npmIntegrationIds()) {
|
|
const read = await readNpm(i.id);
|
|
if (!read.snapshot) {
|
|
cannotCheck(`Nginx Proxy Manager “${i.name}”`, read.error ?? "couldn't be read", "Nginx Proxy Manager");
|
|
continue;
|
|
}
|
|
for (const p of npmProblems(i.id, i.name, read.snapshot, new Date())) {
|
|
push({ category: p.kind === "certificate" ? "expiry" : "monitoring", severity: p.severity, type: "npm", message: p.message, key: p.key, link: "/proxy-hosts", silenced: silencedIntegration(i.id), target: integrationTarget(i.id) });
|
|
}
|
|
}
|
|
}),
|
|
|
|
// What was chosen to watch in Home Assistant: an entity that has said nothing for a while, or a number past its limit.
|
|
check("Home Assistant", async () => {
|
|
const statuses = await haSyncStatuses();
|
|
const nowMs = Date.now();
|
|
const watched = await db.select().from(haWatched);
|
|
if (watched.length === 0) return;
|
|
const entities = await db.select().from(haEntities);
|
|
for (const i of intRows.filter((r) => r.type === "homeassistant" && r.enabled)) {
|
|
const mine = watched.filter((w) => w.integrationId === i.id);
|
|
if (mine.length === 0) continue;
|
|
if (!isHaFresh(statuses[i.id], nowMs)) {
|
|
cannotCheck(`Home Assistant “${i.name}”`, statuses[i.id]?.error ?? "hasn't been read yet", "Home Assistant");
|
|
continue;
|
|
}
|
|
const rows = new Map(entities.filter((e) => e.integrationId === i.id).map((e) => [e.entityId, e]));
|
|
for (const p of haProblems(i.id, i.name, mine, rows, new Date(nowMs))) {
|
|
push({ category: "monitoring", severity: p.severity, type: "homeassistant", message: p.message, key: p.key, link: "/home-assistant", silenced: silencedIntegration(i.id), target: integrationTarget(i.id) });
|
|
}
|
|
}
|
|
}),
|
|
|
|
// Not a notification either: an operating system whose support has ended (or is about to) is seen here, with the date, from what the agent says it runs.
|
|
check("Operating system support", async () => {
|
|
const { healthChecks } = await getSettings();
|
|
if (!(healthChecks.osSupportWarnDays > 0)) return;
|
|
const now = new Date();
|
|
for (const s of await db.select({ id: servers.id, name: servers.name, osId: servers.osId, osName: servers.osName, osVersion: servers.osVersion }).from(servers)) {
|
|
if (!s.osId) continue;
|
|
const life = lifecycleOf({ id: s.osId, name: s.osName, version: s.osVersion });
|
|
const left = life ? daysOfSupportLeft(life.endsOn, now) : null;
|
|
if (!life || life.endsOn === null || left === null || left > healthChecks.osSupportWarnDays) continue;
|
|
const ended = left < 0;
|
|
push({
|
|
category: "updates",
|
|
severity: ended ? "warning" : "info",
|
|
type: "server",
|
|
message: ended
|
|
? `${s.name} runs ${life.product}, whose support ended on ${life.endsOn} — it no longer gets security fixes`
|
|
: `${s.name} runs ${life.product}, whose support ends on ${life.endsOn} (in ${plural(left, "day")})`,
|
|
key: `os-support:${s.id}`,
|
|
link: `/servers/${s.id}`,
|
|
silenced: subjects.has(`server:${s.id}`),
|
|
target: targetOf("server", s.id),
|
|
});
|
|
}
|
|
}),
|
|
|
|
check("Integration failures", async () => {
|
|
const { notifications } = await getSettings();
|
|
const failing = getFailingSources();
|
|
// A failure explained by something it needs that is down is folded under that, like a guest under its host.
|
|
const cover = failing.length > 0 ? await dependencyCover(await offlineServers, failing.map((f) => f.source), await offlineDeviceIds).catch(() => null) : null;
|
|
for (const f of failing) {
|
|
const cause = cover ? integrationTypeCause(f.source, cover) : null;
|
|
const affects = cover ? takenByType(f.source, cover) : [];
|
|
push({
|
|
causedBy: cause ? { name: cause.name, alertId: (cover && alertIdOfNode(cause.key, cover.graph)) || undefined } : undefined,
|
|
affects: affects.length > 0 ? affects.sort((a, b) => a.localeCompare(b)) : undefined,
|
|
category: "integration",
|
|
severity: f.alerted ? "critical" : "warning",
|
|
type: f.source,
|
|
message: `${sourceLabel(f.source)} has failed its last ${plural(f.consecutiveFailures, "call")} in a row${f.alerted ? "" : ` (a notification goes out after ${notifications.integrationFailureThreshold})`} — see the Diagnostic Log`,
|
|
key: `failing:${f.source}`,
|
|
link: null,
|
|
silenced: await isSourceInMaintenance(f.source),
|
|
target: firstIntegrationOfType(f.source),
|
|
});
|
|
}
|
|
}),
|
|
]);
|
|
|
|
alerts.sort(
|
|
(a, b) =>
|
|
Number(a.silenced) - Number(b.silenced) ||
|
|
SEVERITY_ORDER[a.severity] - SEVERITY_ORDER[b.severity] ||
|
|
a.source.localeCompare(b.source) ||
|
|
a.message.localeCompare(b.message),
|
|
);
|
|
|
|
// Those that keep coming and going are marked, so nobody mistakes the quiet of their held messages for the problem having gone.
|
|
const flapping = await flappingKeys().catch(() => new Set<string>());
|
|
if (flapping.size > 0) for (const a of alerts) if (flapping.has(keyOfAlertId(a.id, a.category))) a.flapping = true;
|
|
|
|
// Those somebody has snoozed are held back like those under maintenance, with when they come back.
|
|
const snoozes = await loadSnoozes().catch(() => new Map<string, { until: string; snoozedBy: string; snoozedAt: string; closed: boolean; note: string | null }>());
|
|
if (snoozes.size > 0) {
|
|
for (const a of alerts) {
|
|
const sn = snoozes.get(a.id);
|
|
if (sn) {
|
|
if (sn.closed) a.closed = { by: sn.snoozedBy, at: sn.snoozedAt, note: sn.note };
|
|
else a.snoozed = { until: sn.until, by: sn.snoozedBy };
|
|
a.silenced = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
const active = alerts.filter((a) => !a.silenced);
|
|
const grouped = alerts.filter((a) => a.causedBy).length;
|
|
return {
|
|
alerts,
|
|
counts: {
|
|
critical: active.filter((a) => a.severity === "critical").length,
|
|
warning: active.filter((a) => a.severity === "warning").length,
|
|
info: active.filter((a) => a.severity === "info").length,
|
|
silenced: alerts.length - active.length - grouped,
|
|
grouped,
|
|
},
|
|
couldntCheck: [...blind.entries()].map(([name, v]) => ({ name, error: v.error, affects: [...v.affects] })),
|
|
notes,
|
|
generatedAt: new Date().toISOString(),
|
|
};
|
|
}
|
|
|
|
let cache: { at: number; report: AlertsReport } | null = null;
|
|
|
|
/** Forget the stored result, so the next look is a fresh one — after something changes what it would say (a maintenance window starting or ending). */
|
|
export function invalidateAlertsCache(): void {
|
|
cache = null;
|
|
}
|
|
let inFlight: Promise<AlertsReport> | null = null;
|
|
|
|
/** The current alerts, reusing a recent result unless `force` asks for a fresh one (and even then not more than once every few seconds). */
|
|
export async function getAlerts(force: boolean): Promise<AlertsReport & { cached: boolean }> {
|
|
const age = cache ? Date.now() - cache.at : Infinity;
|
|
if (cache && age < (force ? MIN_REFRESH_MS : CACHE_MS)) return { ...cache.report, cached: true };
|
|
inFlight ??= collectAlerts()
|
|
.then((report) => {
|
|
cache = { at: Date.now(), report };
|
|
// Remembered for the incident record and the acknowledgements — in the background, and never allowed to hold up or break the answer.
|
|
void recordFromReport(report).then(() => reconcileAcks(new Set(report.alerts.map((a) => a.id)))).catch((err) => console.error("[alerts] couldn't update the incident record:", err));
|
|
return report;
|
|
})
|
|
.finally(() => {
|
|
inFlight = null;
|
|
});
|
|
return { ...(await inFlight), cached: false };
|
|
}
|