Heartbeats, Coming up and suggested checks; Operations menu tidied
Heartbeats: a cron job or script calls /api/heartbeat/<token> (and /start, /fail); the app alerts when the call is late or the job says it failed. Coming up: one agenda of everything dated (expiries, warranties, end of support, planned and recurring maintenance, restore tests, waiting updates), with a dashboard card and an iCalendar feed behind a share link. Suggested checks: web ports and certificates the app already knows about, added with a click. Menu: Incidents become Alerts -> History (old /incidents links redirect); new Monitoring group (Endpoints, Heartbeats, Uptime Kuma); osTicket under Automation, Admin Links under Infrastructure, Generator under Administration. Also makes the restore-test metrics test follow the real clock. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c05c847d17
commit
4e0c6927b2
44 files changed
+7092
-75
No files matched your search
@@ -0,0 +1,343 @@
|
||||
/**
|
||||
* "Coming up": everything with a date ahead, in one list — a secret, domain, certificate or Tailscale key that expires, a warranty or end of support,
|
||||
* planned and recurring maintenance, a restore test that is due, and updates that have been waiting. Each already lives on a page of its own and in the
|
||||
* weekly digest; this puts them side by side, oldest first, so what is about to need attention is one look, not a dozen.
|
||||
*
|
||||
* Built in two parts so it can be tested exactly: `loadAgendaData` reads what is stored (and, for Tailscale keys, asks Tailscale, remembered for ten
|
||||
* minutes), and `agendaFromData` turns that into items. `buildIcs` writes the same items as an iCalendar feed for a phone's calendar to subscribe to.
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, domains, endpointChecks, integrations, maintenanceSchedules, secrets, servers } from "../db/schema.js";
|
||||
import { collectTailscaleKeyExpiries } from "./tailscaleKeyExpiryScheduler.js";
|
||||
import { describeTarget, listUpcomingWindows } from "./maintenance.js";
|
||||
import { occurrencesBetween, targetsOf } from "./maintenanceSchedules.js";
|
||||
import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
import { latestTests } from "./restoreTests.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { env } from "../env.js";
|
||||
|
||||
export const AGENDA_KINDS = ["secret", "certificate", "domain", "tailscale", "warranty", "support", "maintenance", "restore", "updates"] as const;
|
||||
export type AgendaKind = (typeof AGENDA_KINDS)[number];
|
||||
|
||||
export const KIND_LABELS: Record<AgendaKind, string> = {
|
||||
secret: "Secrets",
|
||||
certificate: "Certificates",
|
||||
domain: "Domains",
|
||||
tailscale: "Tailscale keys",
|
||||
warranty: "Warranties",
|
||||
support: "End of support",
|
||||
maintenance: "Maintenance",
|
||||
restore: "Restore tests",
|
||||
updates: "Updates waiting",
|
||||
};
|
||||
|
||||
export interface AgendaItem {
|
||||
/** Stable, with the date in it: a secret that is renewed is a different entry, and the old one just goes. */
|
||||
id: string;
|
||||
kind: AgendaKind;
|
||||
/** YYYY-MM-DD for something that happens on a day; an ISO time for maintenance, which has a start and an end. */
|
||||
at: string;
|
||||
endsAt: string | null;
|
||||
allDay: boolean;
|
||||
title: string;
|
||||
detail: string | null;
|
||||
/** The page of the app that has more. */
|
||||
link: string;
|
||||
serverId: number | null;
|
||||
/** Whole days from today to the day it falls on, in the time zone set for notifications; negative once it has passed. */
|
||||
daysLeft: number;
|
||||
}
|
||||
|
||||
export interface Agenda {
|
||||
generatedAt: string;
|
||||
today: string;
|
||||
items: AgendaItem[];
|
||||
/** Sources that could not be read, so the agenda is missing something. */
|
||||
notes: string[];
|
||||
}
|
||||
|
||||
export const DEFAULT_DAYS = 90;
|
||||
export const MAX_DAYS = 400;
|
||||
/** What has already happened is shown for this long (a secret that expired last week is still worth seeing). */
|
||||
export const LOOK_BACK_DAYS = 30;
|
||||
const DAY_MS = 86_400_000;
|
||||
|
||||
// ─── Dates ─────────────────────────────────────────────────────────────────
|
||||
|
||||
const isYmd = (v: unknown): v is string => typeof v === "string" && /^\d{4}-\d{2}-\d{2}$/.test(v) && !Number.isNaN(Date.parse(`${v}T00:00:00Z`)) && new Date(`${v}T00:00:00Z`).toISOString().startsWith(v);
|
||||
|
||||
/** The calendar day an instant falls on, as the clock in `tz` sees it. */
|
||||
export function localDate(when: Date, tz: string): string {
|
||||
return new Intl.DateTimeFormat("en-CA", { timeZone: tz, year: "numeric", month: "2-digit", day: "2-digit" }).format(when);
|
||||
}
|
||||
|
||||
const daysBetween = (fromYmd: string, toYmd: string) => Math.round((Date.parse(`${toYmd}T00:00:00Z`) - Date.parse(`${fromYmd}T00:00:00Z`)) / DAY_MS);
|
||||
|
||||
export const addDays = (ymd: string, n: number) => new Date(Date.parse(`${ymd}T00:00:00Z`) + n * DAY_MS).toISOString().slice(0, 10);
|
||||
|
||||
// ─── What is stored ────────────────────────────────────────────────────────
|
||||
|
||||
export interface AgendaData {
|
||||
secrets: { id: number; name: string; type: string; expiryDate: string | null }[];
|
||||
domains: { id: number; name: string; expiresAt: string | null }[];
|
||||
endpoints: { id: number; name: string; enabled: boolean; certExpiresAt: string | null; serverId: number | null }[];
|
||||
tailscale: { integrationId: number; integrationName: string; deviceLabel: string; daysLeft: number }[];
|
||||
assets: { id: number; name: string; warrantyUntil: string | null; endOfSupport: string | null; serverId: number | null }[];
|
||||
/** Windows that have been set up but not begun, one per target. */
|
||||
windows: { startedAt: string; endsAt: string; reason: string | null; source: string | null; targetName: string }[];
|
||||
schedules: { id: number; name: string; reason: string | null; days: string; startTime: string; durationMinutes: number; timezone: string; targetNames: string[] }[];
|
||||
/** The newest restore test of each subject. */
|
||||
restoreTests: { subjectKey: string; label: string; testedAt: string; result: string }[];
|
||||
restoreLimitDays: number;
|
||||
/** Servers with updates waiting. */
|
||||
updates: { name: string; count: number; security: number | null; since: string | null }[];
|
||||
notes: string[];
|
||||
}
|
||||
|
||||
let tailscaleCache: { at: number; items: AgendaData["tailscale"]; failures: string[] } | null = null;
|
||||
const TAILSCALE_CACHE_MS = 10 * 60_000;
|
||||
export const clearAgendaCacheForTests = () => {
|
||||
tailscaleCache = null;
|
||||
};
|
||||
|
||||
async function tailscaleKeys(): Promise<{ items: AgendaData["tailscale"]; failures: string[] }> {
|
||||
if (tailscaleCache && Date.now() - tailscaleCache.at < TAILSCALE_CACHE_MS) return tailscaleCache;
|
||||
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "tailscale"), eq(integrations.enabled, true)));
|
||||
if (rows.length === 0) return { items: [], failures: [] };
|
||||
const { items, failures } = await collectTailscaleKeyExpiries();
|
||||
tailscaleCache = { at: Date.now(), items, failures: failures.map((f) => `Tailscale (${f.integrationName}): ${f.message}`) };
|
||||
return tailscaleCache;
|
||||
}
|
||||
|
||||
export async function loadAgendaData(now: Date = new Date()): Promise<AgendaData> {
|
||||
const notes: string[] = [];
|
||||
const settings = await getSettings();
|
||||
const [secretRows, domainRows, endpointRows, assetRows, scheduleRows, serverRows, tests] = await Promise.all([
|
||||
db.select({ id: secrets.id, name: secrets.name, type: secrets.type, expiryDate: secrets.expiryDate }).from(secrets),
|
||||
db.select({ id: domains.id, name: domains.name, expiresAt: domains.expiresAt }).from(domains),
|
||||
db.select({ id: endpointChecks.id, name: endpointChecks.name, enabled: endpointChecks.enabled, certExpiresAt: endpointChecks.certExpiresAt, serverId: endpointChecks.serverId }).from(endpointChecks),
|
||||
db.select({ id: assets.id, name: assets.name, warrantyUntil: assets.warrantyUntil, endOfSupport: assets.endOfSupport, serverId: assets.serverId }).from(assets),
|
||||
db.select().from(maintenanceSchedules),
|
||||
db.select({ name: servers.name, pendingUpdates: servers.pendingUpdates, since: servers.updatesPendingSince }).from(servers),
|
||||
latestTests(),
|
||||
]);
|
||||
|
||||
const windows: AgendaData["windows"] = [];
|
||||
for (const w of await listUpcomingWindows(now)) {
|
||||
const target = await describeTarget(w.targetType, w.targetId);
|
||||
if (target) windows.push({ startedAt: w.startedAt, endsAt: w.endsAt, reason: w.reason, source: w.source, targetName: target.name });
|
||||
}
|
||||
|
||||
const schedules: AgendaData["schedules"] = [];
|
||||
for (const s of scheduleRows.filter((r) => r.enabled)) {
|
||||
const names: string[] = [];
|
||||
for (const t of targetsOf(s)) {
|
||||
const d = await describeTarget(t.type, t.id);
|
||||
if (d) names.push(d.name);
|
||||
}
|
||||
schedules.push({ id: s.id, name: s.name, reason: s.reason, days: s.days, startTime: s.startTime, durationMinutes: s.durationMinutes, timezone: s.timezone, targetNames: names });
|
||||
}
|
||||
|
||||
const updates: AgendaData["updates"] = [];
|
||||
for (const s of serverRows) {
|
||||
const u = parsePendingUpdates(s.pendingUpdates);
|
||||
if (u && u.count > 0) updates.push({ name: s.name, count: u.count, security: u.security, since: s.since });
|
||||
}
|
||||
|
||||
let tailscale: AgendaData["tailscale"] = [];
|
||||
try {
|
||||
const t = await tailscaleKeys();
|
||||
tailscale = t.items;
|
||||
notes.push(...t.failures);
|
||||
} catch (err) {
|
||||
notes.push(`Tailscale keys could not be read: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
|
||||
return {
|
||||
secrets: secretRows,
|
||||
domains: domainRows,
|
||||
endpoints: endpointRows,
|
||||
tailscale,
|
||||
assets: assetRows,
|
||||
windows,
|
||||
schedules,
|
||||
restoreTests: [...tests.values()].map((t) => ({ subjectKey: t.subjectKey, label: t.label, testedAt: t.testedAt, result: t.result })),
|
||||
restoreLimitDays: settings.healthChecks.restoreTestDays,
|
||||
updates,
|
||||
notes,
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Making the list ───────────────────────────────────────────────────────
|
||||
|
||||
const plural = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
|
||||
|
||||
/** Pure: the items, from what was read. `days` is how far ahead; what has just passed is kept for LOOK_BACK_DAYS. */
|
||||
export function agendaFromData(data: AgendaData, now: Date, tz: string, days: number = DEFAULT_DAYS): Agenda {
|
||||
const today = localDate(now, tz);
|
||||
const horizon = Math.min(Math.max(Math.round(days), 1), MAX_DAYS);
|
||||
const items: AgendaItem[] = [];
|
||||
|
||||
const day = (id: string, kind: AgendaKind, ymd: string | null, fn: (past: boolean) => { title: string; detail?: string | null; link: string; serverId?: number | null }) => {
|
||||
if (!isYmd(ymd)) return;
|
||||
const daysLeft = daysBetween(today, ymd);
|
||||
if (daysLeft < -LOOK_BACK_DAYS || daysLeft > horizon) return;
|
||||
const { title, detail = null, link, serverId = null } = fn(daysLeft < 0);
|
||||
items.push({ id: `${id}:${ymd}`, kind, at: ymd, endsAt: null, allDay: true, title, detail, link, serverId, daysLeft });
|
||||
};
|
||||
|
||||
for (const s of data.secrets) {
|
||||
const certificate = s.type === "ssl_certificate";
|
||||
day(`secret:${s.id}`, certificate ? "certificate" : "secret", s.expiryDate, (past) => ({ title: `${certificate ? "Certificate" : "Secret"} “${s.name}” ${past ? "expired" : "expires"}`, detail: certificate ? "kept as a secret" : null, link: "/secrets" }));
|
||||
}
|
||||
for (const d of data.domains) day(`domain:${d.id}`, "domain", d.expiresAt, (past) => ({ title: `Domain ${d.name} ${past ? "expired" : "expires"}`, link: "/domains" }));
|
||||
for (const e of data.endpoints) {
|
||||
if (!e.enabled || !e.certExpiresAt) continue;
|
||||
day(`endpoint-cert:${e.id}`, "certificate", e.certExpiresAt.slice(0, 10), (past) => ({ title: `The certificate for ${e.name} ${past ? "expired" : "expires"}`, detail: "seen by an endpoint check", link: "/endpoints", serverId: e.serverId }));
|
||||
}
|
||||
for (const t of data.tailscale) {
|
||||
day(`tailscale:${t.integrationId}:${t.deviceLabel}`, "tailscale", addDays(today, t.daysLeft), (past) => ({ title: `Tailscale key for ${t.deviceLabel} ${past ? "expired" : "expires"}`, detail: t.integrationName, link: "/tailscale" }));
|
||||
}
|
||||
for (const a of data.assets) {
|
||||
day(`warranty:${a.id}`, "warranty", a.warrantyUntil, (past) => ({ title: `Warranty on ${a.name} ${past ? "ended" : "ends"}`, link: `/inventory?asset=${a.id}`, serverId: a.serverId }));
|
||||
day(`support:${a.id}`, "support", a.endOfSupport, (past) => ({ title: `Support for ${a.name} ${past ? "ended" : "ends"}`, detail: "time to plan its replacement", link: `/inventory?asset=${a.id}`, serverId: a.serverId }));
|
||||
}
|
||||
|
||||
// A restore test is due a set time after the last good one — only if a time limit is set (Settings → Notifications).
|
||||
if (data.restoreLimitDays > 0) {
|
||||
for (const t of data.restoreTests) {
|
||||
if ((t.result !== "ok" && t.result !== "partial") || !isYmd(t.testedAt)) continue;
|
||||
day(`restore:${t.subjectKey}`, "restore", addDays(t.testedAt, data.restoreLimitDays), (past) => ({ title: `Restore test ${past ? "was due" : "due"}: ${t.label}`, detail: `last tested ${t.testedAt}`, link: "/backups" }));
|
||||
}
|
||||
}
|
||||
|
||||
// Maintenance has a start and an end. Windows set up by a recurring schedule are left to the schedule (they are only made a week ahead).
|
||||
const lastMs = now.getTime() + horizon * DAY_MS;
|
||||
const groups = new Map<string, { start: string; end: string; reason: string | null; names: string[] }>();
|
||||
for (const w of data.windows) {
|
||||
if (w.source?.startsWith("schedule:") || Date.parse(w.startedAt) > lastMs) continue;
|
||||
const key = `${w.startedAt}|${w.endsAt}|${w.reason ?? ""}`;
|
||||
const g = groups.get(key) ?? { start: w.startedAt, end: w.endsAt, reason: w.reason, names: [] };
|
||||
g.names.push(w.targetName);
|
||||
groups.set(key, g);
|
||||
}
|
||||
const timed = (id: string, start: string, end: string, title: string, detail: string | null) =>
|
||||
items.push({ id, kind: "maintenance", at: start, endsAt: end, allDay: false, title, detail, link: "/maintenance", serverId: null, daysLeft: daysBetween(today, localDate(new Date(start), tz)) });
|
||||
const names = (list: string[]) => {
|
||||
const sorted = [...list].sort((a, b) => a.localeCompare(b));
|
||||
return sorted.length <= 3 ? sorted.join(", ") : `${sorted.slice(0, 3).join(", ")} and ${sorted.length - 3} more`;
|
||||
};
|
||||
for (const g of groups.values()) timed(`maintenance:${g.start}:${g.names.length}`, g.start, g.end, `Maintenance: ${names(g.names)}`, g.reason);
|
||||
for (const s of data.schedules) {
|
||||
for (const occ of occurrencesBetween(s, now, new Date(lastMs)).slice(0, 60)) {
|
||||
timed(`schedule:${s.id}:${occ.date}`, occ.start.toISOString(), occ.end.toISOString(), `Recurring maintenance: ${s.name}`, [s.targetNames.length > 0 ? names(s.targetNames) : null, s.reason].filter(Boolean).join(" — ") || null);
|
||||
}
|
||||
}
|
||||
|
||||
// Updates that have been waiting: one line for all of them, from the longest wait.
|
||||
if (data.updates.length > 0) {
|
||||
const since = data.updates.map((u) => u.since).filter((s): s is string => !!s).map((s) => localDate(new Date(s), tz)).sort()[0] ?? today;
|
||||
const security = data.updates.filter((u) => (u.security ?? 0) > 0).length;
|
||||
const waited = Math.max(0, daysBetween(since, today));
|
||||
items.push({
|
||||
id: `updates:${since}`,
|
||||
kind: "updates",
|
||||
at: since,
|
||||
endsAt: null,
|
||||
allDay: true,
|
||||
title: `Updates waiting on ${plural(data.updates.length, "server")}${security > 0 ? ` (${security} with security fixes)` : ""}`,
|
||||
detail: waited > 0 ? `the longest wait is ${plural(waited, "day")}` : "reported today",
|
||||
link: "/updates",
|
||||
serverId: null,
|
||||
daysLeft: -waited,
|
||||
});
|
||||
}
|
||||
|
||||
items.sort((a, b) => a.at.localeCompare(b.at) || a.title.localeCompare(b.title));
|
||||
return { generatedAt: now.toISOString(), today, items, notes: data.notes };
|
||||
}
|
||||
|
||||
export async function buildAgenda(now: Date = new Date(), days: number = DEFAULT_DAYS): Promise<Agenda> {
|
||||
const settings = await getSettings();
|
||||
return agendaFromData(await loadAgendaData(now), now, settings.notifications.timezone || "UTC", days);
|
||||
}
|
||||
|
||||
// ─── The calendar feed ─────────────────────────────────────────────────────
|
||||
|
||||
/** Escapes a text value for iCalendar: backslash, semicolon, comma and line breaks. */
|
||||
const escapeText = (s: string) => s.replace(/\\/g, "\\\\").replace(/;/g, "\\;").replace(/,/g, "\\,").replace(/\r?\n/g, "\\n");
|
||||
|
||||
/** Lines are folded at 75 bytes, a space starting each continuation, without splitting a character. */
|
||||
export function foldLine(line: string): string {
|
||||
const out: string[] = [];
|
||||
let current = "";
|
||||
let bytes = 0;
|
||||
for (const ch of line) {
|
||||
const n = Buffer.byteLength(ch, "utf8");
|
||||
if (bytes + n > (out.length === 0 ? 75 : 74)) {
|
||||
out.push(current);
|
||||
current = "";
|
||||
bytes = 0;
|
||||
}
|
||||
current += ch;
|
||||
bytes += n;
|
||||
}
|
||||
out.push(current);
|
||||
return out.join("\r\n ");
|
||||
}
|
||||
|
||||
const icsDate = (ymd: string) => ymd.replace(/-/g, "");
|
||||
const icsTime = (iso: string) => new Date(iso).toISOString().replace(/[-:]/g, "").replace(/\.\d{3}/, "");
|
||||
|
||||
/** What an event is called when the link was made to show no names: only what kind of thing it is. */
|
||||
const GENERIC: Record<AgendaKind, string> = {
|
||||
secret: "A secret expires",
|
||||
certificate: "A certificate expires",
|
||||
domain: "A domain name expires",
|
||||
tailscale: "A Tailscale key expires",
|
||||
warranty: "A warranty ends",
|
||||
support: "Support ends for a device",
|
||||
maintenance: "Planned maintenance",
|
||||
restore: "A restore test is due",
|
||||
updates: "Updates are waiting",
|
||||
};
|
||||
|
||||
/**
|
||||
* The agenda as an iCalendar feed. A day is an all-day event; maintenance is an event from its start to its end. With `detail` "summary" an event only
|
||||
* says what kind of thing it is (no names), so a feed that is shared can be shown without telling anyone what is in the homelab. Updates that
|
||||
* are waiting are not events — they have no day — and are left out.
|
||||
*/
|
||||
export function buildIcs(agenda: Agenda, options: { name: string; detail: "summary" | "names"; baseUrl?: string }): string {
|
||||
const lines = [
|
||||
"BEGIN:VCALENDAR",
|
||||
"VERSION:2.0",
|
||||
"PRODID:-//Homelab Manager//Coming up//EN",
|
||||
"CALSCALE:GREGORIAN",
|
||||
"METHOD:PUBLISH",
|
||||
`X-WR-CALNAME:${escapeText(options.name)}`,
|
||||
"REFRESH-INTERVAL;VALUE=DURATION:PT6H",
|
||||
"X-PUBLISHED-TTL:PT6H",
|
||||
];
|
||||
const stamp = icsTime(agenda.generatedAt);
|
||||
for (const item of agenda.items) {
|
||||
if (item.kind === "updates") continue;
|
||||
if (item.daysLeft < -7) continue; // an old event in a subscribed calendar is just clutter
|
||||
const names = options.detail === "names";
|
||||
lines.push("BEGIN:VEVENT", `UID:${createHash("sha256").update(item.id).digest("hex").slice(0, 24)}@homelab-manager`, `DTSTAMP:${stamp}`);
|
||||
if (item.allDay) lines.push(`DTSTART;VALUE=DATE:${icsDate(item.at)}`, `DTEND;VALUE=DATE:${icsDate(addDays(item.at, 1))}`);
|
||||
else lines.push(`DTSTART:${icsTime(item.at)}`, `DTEND:${icsTime(item.endsAt ?? item.at)}`);
|
||||
lines.push(`SUMMARY:${escapeText(names ? item.title : GENERIC[item.kind])}`);
|
||||
if (names) {
|
||||
const description = [item.detail, options.baseUrl ? `${options.baseUrl.replace(/\/+$/, "")}${item.link}` : null].filter(Boolean).join("\n");
|
||||
if (description) lines.push(`DESCRIPTION:${escapeText(description)}`);
|
||||
}
|
||||
lines.push(`CATEGORIES:${escapeText(KIND_LABELS[item.kind])}`, "TRANSP:TRANSPARENT", "STATUS:CONFIRMED", "END:VEVENT");
|
||||
}
|
||||
lines.push("END:VCALENDAR");
|
||||
return `${lines.map(foldLine).join("\r\n")}\r\n`;
|
||||
}
|
||||
|
||||
/** What the feed calls itself. */
|
||||
export const calendarName = () => `Homelab Manager${env.environment?.name ? ` (${env.environment.name})` : ""} — coming up`;
|
||||
@@ -11,7 +11,7 @@
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { endpointChecks, integrations, secrets, servers } from "../db/schema.js";
|
||||
import { endpointChecks, heartbeats, integrations, secrets, servers } from "../db/schema.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
|
||||
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
|
||||
@@ -35,6 +35,7 @@ import { getSettings } from "./settingsStore.js";
|
||||
import { sourceLabel } from "./notify.js";
|
||||
import { alertIdOfNode, dependencyCover, integrationTypeCause, takenByType } from "./dependencies.js";
|
||||
import { endpointConditions } from "./endpointChecks.js";
|
||||
import { heartbeatConditions } from "./heartbeats.js";
|
||||
import { latestTests, restoreAlerts } from "./restoreTests.js";
|
||||
import type { Alert, AlertCategory, AlertSeverity, SourceFailure } from "./alertTypes.js";
|
||||
|
||||
@@ -65,6 +66,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
|
||||
secrets: { label: "Secrets", link: "/secrets" },
|
||||
domains: { label: "Domains", link: "/domains" },
|
||||
endpoint: { label: "Endpoint", link: "/endpoints" },
|
||||
heartbeat: { label: "Heartbeat", link: "/heartbeats" },
|
||||
};
|
||||
|
||||
const SOURCE_TIMEOUT_MS = 20_000;
|
||||
@@ -304,6 +306,22 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
}
|
||||
}),
|
||||
|
||||
// Jobs that report in: one that has not done so in time, or said it failed.
|
||||
check("Heartbeats", async () => {
|
||||
for (const c of heartbeatConditions(await db.select().from(heartbeats), new Date())) {
|
||||
push({
|
||||
category: c.category,
|
||||
severity: c.severity,
|
||||
type: "heartbeat",
|
||||
message: c.message,
|
||||
key: c.key,
|
||||
link: "/heartbeats",
|
||||
silenced: c.serverId !== null && subjects.has(`server:${c.serverId}`),
|
||||
target: c.serverId !== null ? targetOf("server", c.serverId) : null,
|
||||
});
|
||||
}
|
||||
}),
|
||||
|
||||
// Homelab Manager's own backups: if they are switched on, there should be a recent one. This is what turns a backup that has quietly
|
||||
// stopped working — a full disk, a folder that went read-only — into something somebody hears about before they need it.
|
||||
check("Homelab Manager's own backups", async () => {
|
||||
|
||||
@@ -21,7 +21,7 @@ export const KIND_LABELS: Record<ChangeKind, string> = {
|
||||
dependencies: "Dependencies",
|
||||
inventory: "Inventory",
|
||||
backups: "Backups and restore tests",
|
||||
endpoints: "Endpoint checks",
|
||||
endpoints: "Endpoint checks and heartbeats",
|
||||
other: "Other",
|
||||
};
|
||||
|
||||
@@ -53,6 +53,7 @@ const KIND_OF_CATEGORY: Record<string, ChangeKind> = {
|
||||
db_backup: "backups",
|
||||
restore_test: "backups",
|
||||
endpoint: "endpoints",
|
||||
heartbeat: "endpoints",
|
||||
};
|
||||
|
||||
/** Sign-ins and a person's own data export are not changes to the homelab; the diagnostic log has its own page. */
|
||||
@@ -154,6 +155,14 @@ export function describeChange(category: string, action: string, detail: Record<
|
||||
return { title: `Endpoint check changed${name ? `: ${name}` : ""}`, tone: "neutral", detail: [str(detail.kind), str(detail.target)].filter(Boolean).join(" · ") || null };
|
||||
case "endpoint/delete":
|
||||
return { title: `Endpoint check removed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
|
||||
case "heartbeat/create":
|
||||
return { title: `Heartbeat added${name ? `: ${name}` : ""}`, tone: "neutral", detail: typeof detail.periodMinutes === "number" ? `expected every ${detail.periodMinutes} min` : null };
|
||||
case "heartbeat/update":
|
||||
return { title: `Heartbeat changed${name ? `: ${name}` : ""}`, tone: "neutral", detail: typeof detail.periodMinutes === "number" ? `expected every ${detail.periodMinutes} min` : null };
|
||||
case "heartbeat/new_address":
|
||||
return { title: `Heartbeat given a new address${name ? `: ${name}` : ""}`, tone: "neutral", detail: "the old address no longer works" };
|
||||
case "heartbeat/delete":
|
||||
return { title: `Heartbeat removed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
|
||||
case "restore_test/record": {
|
||||
const result = str(detail.result);
|
||||
const tone: ActivityTone = result === "failed" ? "bad" : result === "not_needed" ? "neutral" : "good";
|
||||
@@ -250,7 +259,8 @@ export async function getChangeLog(query: ChangeQuery = {}): Promise<{ entries:
|
||||
} else if (a.targetType === "endpoint" && idNum !== null) {
|
||||
targetName = endpointName.get(idNum) ?? targetName;
|
||||
link = "/endpoints";
|
||||
} else if (a.targetType === "asset" && idNum !== null) link = `/inventory?asset=${idNum}`;
|
||||
} else if (a.targetType === "heartbeat") link = "/heartbeats";
|
||||
else if (a.targetType === "asset" && idNum !== null) link = `/inventory?asset=${idNum}`;
|
||||
else if (a.targetType === "restore_test") link = "/backups";
|
||||
else if (a.targetType === "dependency") link = "/dependencies";
|
||||
else if (a.targetType === "settings") link = "/settings/notifications";
|
||||
@@ -280,7 +290,7 @@ export async function getChangeLog(query: ChangeQuery = {}): Promise<{ entries:
|
||||
.limit(1000);
|
||||
for (const i of incidents) {
|
||||
const serverId = i.targetType === "server" ? i.targetId : null;
|
||||
const base = { kind: "alerts" as const, actor: null, target: i.targetType ? { type: i.targetType, id: i.targetId === null ? null : String(i.targetId), name: i.targetName } : null, link: `/incidents/${i.id}`, serverId };
|
||||
const base = { kind: "alerts" as const, actor: null, target: i.targetType ? { type: i.targetType, id: i.targetId === null ? null : String(i.targetId), name: i.targetName } : null, link: `/alerts/incidents/${i.id}`, serverId };
|
||||
if (i.openedAt >= since.toISOString() && (!until || i.openedAt <= until.toISOString())) {
|
||||
entries.push({ detail: {}, entry: { ...base, id: `incident-open-${i.id}`, at: i.openedAt, tone: i.peakSeverity === "critical" ? "bad" : "info", title: `Alert raised: ${i.message}`, detail: i.causedBy ? `explained by ${i.causedBy}` : null } });
|
||||
}
|
||||
|
||||
@@ -5,9 +5,10 @@
|
||||
* a maintenance window on the server a check is linked to holds its problems back, and they alert after it if they are still there.
|
||||
*/
|
||||
import { db } from "../db/client.js";
|
||||
import { endpointChecks } from "../db/schema.js";
|
||||
import { endpointChecks, heartbeats } from "../db/schema.js";
|
||||
import { diffConditions, type ActiveState, type HealthCondition } from "./healthMonitor.js";
|
||||
import { activeSubjects } from "./maintenance.js";
|
||||
import { heartbeatConditions, prunePings } from "./heartbeats.js";
|
||||
import { endpointConditions, pruneResults, runDueChecks, type CheckRow, type ProbeResult } from "./endpointChecks.js";
|
||||
import { notifyHealthIssues, notifyHealthRecovered } from "./notify.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
@@ -28,7 +29,9 @@ async function loadState(): Promise<ActiveState> {
|
||||
export async function runEndpointPass(now: Date = new Date(), probeFn?: (c: CheckRow) => Promise<ProbeResult>): Promise<{ asked: number; announced: number; recovered: number }> {
|
||||
const asked = await runDueChecks(now, probeFn);
|
||||
const rows = await db.select().from(endpointChecks);
|
||||
const current: HealthCondition[] = endpointConditions(rows, now).map((c) => ({ key: c.key, source: c.source, message: c.message, severity: c.severity }));
|
||||
// The jobs that report in are judged in the same pass, by the same bookkeeping: a call that is overdue is announced once, and again as cleared when it comes.
|
||||
const beats = await db.select().from(heartbeats);
|
||||
const current: HealthCondition[] = [...endpointConditions(rows, now), ...heartbeatConditions(beats, now)].map((c) => ({ key: c.key, source: c.source, message: c.message, severity: c.severity }));
|
||||
const { added, resolved, next } = diffConditions(await loadState(), current, new Set(), await activeSubjects(now));
|
||||
await setInternalFlag(STATE_FLAG, JSON.stringify(next));
|
||||
if (added.length > 0) await notifyHealthIssues(added);
|
||||
@@ -48,6 +51,7 @@ export function initEndpointMonitor(): void {
|
||||
if (Date.now() - lastPrune > 24 * 3600_000) {
|
||||
lastPrune = Date.now();
|
||||
await pruneResults();
|
||||
await prunePings();
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[endpoints] the pass failed:", err);
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
/**
|
||||
* Suggested endpoint checks: things the app already knows enough about to offer a check for, so setting up monitoring is a click instead of typing.
|
||||
*
|
||||
* - A web port a server's agent sees open (80, 443, 8080, 8443), with a name that points at it in DNS if there is one — "git.example.lan answers on
|
||||
* 443 on nas" — as a web check, linked to that server so its maintenance silences the alerts.
|
||||
* - A secret that already has a host to read the certificate from, as a certificate check: the secret only warns about the expiry, this also hears
|
||||
* when the service stops answering.
|
||||
*
|
||||
* It only reads what is stored (no network), leaves out what a check already covers, and remembers what somebody said no to. Uptime Kuma's monitors are
|
||||
* not offered: they are already being watched, and the adapter only keeps a host and port for them, not the address.
|
||||
*/
|
||||
import { db } from "../db/client.js";
|
||||
import { dnsRecordsCache, endpointChecks, secrets, servers } from "../db/schema.js";
|
||||
import { isLoopback } from "./agentPorts.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export interface Suggestion {
|
||||
/** Stable identity, so a dismissed suggestion stays dismissed and "add" can name it. */
|
||||
key: string;
|
||||
kind: "http" | "tls";
|
||||
name: string;
|
||||
target: string;
|
||||
port: number | null;
|
||||
serverId: number | null;
|
||||
serverName: string | null;
|
||||
/** Why it is offered, in a sentence. */
|
||||
reason: string;
|
||||
}
|
||||
|
||||
export interface SuggestionInput {
|
||||
servers: { id: number; name: string; hostname: string | null; ips: string[]; ports: { protocol?: string; port: number; address?: string }[] }[];
|
||||
dns: { name: string; type: string; content: string }[];
|
||||
secrets: { name: string; checkHost: string | null; checkPort: number | null }[];
|
||||
existing: { kind: string; target: string; port: number | null }[];
|
||||
dismissed: Set<string>;
|
||||
}
|
||||
|
||||
/** The web ports worth offering, and what they speak. */
|
||||
const WEB_PORTS: Record<number, "http" | "https"> = { 80: "http", 443: "https", 8080: "http", 8443: "https" };
|
||||
/** The https twin of a plain port, so a server answering on both gets the one that reads the certificate. */
|
||||
const SECURE_TWIN: Record<number, number> = { 80: 443, 8080: 8443 };
|
||||
const MAX_NAMES_PER_PORT = 3;
|
||||
const MAX_SUGGESTIONS = 200;
|
||||
|
||||
const cleanHost = (h: string) => h.trim().toLowerCase().replace(/\.$/, "");
|
||||
const isIPv4 = (s: string) => /^\d{1,3}(\.\d{1,3}){3}$/.test(s);
|
||||
const usableIPv4 = (s: string) => isIPv4(s) && !s.startsWith("127.") && !s.startsWith("169.254.");
|
||||
|
||||
/** Where a check's address is, in the same words for an existing check and a suggestion: "web:host:port" and "tls:host:port" (a web check on https reads the certificate too). */
|
||||
function coverage(existing: SuggestionInput["existing"]): Set<string> {
|
||||
const out = new Set<string>();
|
||||
for (const c of existing) {
|
||||
if (c.kind === "http") {
|
||||
try {
|
||||
const u = new URL(c.target);
|
||||
const port = u.port ? Number(u.port) : u.protocol === "https:" ? 443 : 80;
|
||||
const host = cleanHost(u.hostname);
|
||||
out.add(`web:${host}:${port}`);
|
||||
if (u.protocol === "https:") out.add(`tls:${host}:${port}`);
|
||||
} catch {
|
||||
// an address that isn't one covers nothing
|
||||
}
|
||||
} else if (c.kind === "tls" && c.port !== null) out.add(`tls:${cleanHost(c.target)}:${c.port}`);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Pure: the checks worth offering, from what is known. */
|
||||
export function suggestChecks(input: SuggestionInput): { suggestions: Suggestion[]; dismissed: Suggestion[] } {
|
||||
const covered = coverage(input.existing);
|
||||
const all = new Map<string, Suggestion>();
|
||||
|
||||
for (const s of input.servers) {
|
||||
const ips = new Set(s.ips.map((i) => i.trim().toLowerCase()));
|
||||
const hostname = s.hostname ? cleanHost(s.hostname) : null;
|
||||
// Names that point at this server in DNS: an address record with one of its addresses, and an alias to one of those names or to its hostname.
|
||||
const direct = new Set<string>();
|
||||
for (const r of input.dns) {
|
||||
if ((r.type === "A" || r.type === "AAAA") && ips.has(r.content.trim().toLowerCase())) direct.add(cleanHost(r.name));
|
||||
}
|
||||
const names = new Set(direct);
|
||||
for (const r of input.dns) {
|
||||
if (r.type !== "CNAME") continue;
|
||||
const to = cleanHost(r.content);
|
||||
if (direct.has(to) || (hostname !== null && to === hostname)) names.add(cleanHost(r.name));
|
||||
}
|
||||
const dnsNames = [...names].filter((n) => n.includes(".") && !n.startsWith("*") && !n.endsWith(".arpa") && !/\s/.test(n)).sort((a, b) => a.length - b.length || a.localeCompare(b));
|
||||
|
||||
const listening = new Set(s.ports.filter((p) => (p.protocol ?? "tcp") === "tcp" && !(p.address && isLoopback(p.address))).map((p) => p.port));
|
||||
for (const port of Object.keys(WEB_PORTS).map(Number).filter((p) => listening.has(p))) {
|
||||
// Plain and secure on the same server: the secure one is the real service (the plain one is most likely its redirect), and reads the certificate.
|
||||
if (SECURE_TWIN[port] !== undefined && listening.has(SECURE_TWIN[port])) continue;
|
||||
const scheme = WEB_PORTS[port];
|
||||
const fallback = hostname && hostname.includes(".") ? [hostname] : [s.ips.map((i) => i.trim()).find(usableIPv4)].filter((x): x is string => !!x);
|
||||
const hosts = dnsNames.length > 0 ? dnsNames.slice(0, MAX_NAMES_PER_PORT) : fallback;
|
||||
for (const host of hosts) {
|
||||
const defaultPort = scheme === "https" ? 443 : 80;
|
||||
const target = `${scheme}://${host}${port === defaultPort ? "" : `:${port}`}/`;
|
||||
if (covered.has(`web:${host}:${port}`)) continue;
|
||||
const viaDns = dnsNames.includes(host);
|
||||
all.set(`http|${target}`, {
|
||||
key: `http|${target}`,
|
||||
kind: "http",
|
||||
name: `${host}${port === defaultPort ? "" : `:${port}`}`,
|
||||
target,
|
||||
port: null,
|
||||
serverId: s.id,
|
||||
serverName: s.name,
|
||||
reason: `${s.name} is listening on port ${port}${viaDns ? `, and ${host} points at it in DNS` : ""}.`,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const sec of input.secrets) {
|
||||
if (!sec.checkHost || sec.checkPort === null) continue;
|
||||
const host = cleanHost(sec.checkHost);
|
||||
if (covered.has(`tls:${host}:${sec.checkPort}`)) continue;
|
||||
all.set(`tls|${host}|${sec.checkPort}`, {
|
||||
key: `tls|${host}|${sec.checkPort}`,
|
||||
kind: "tls",
|
||||
name: `Certificate: ${sec.name}`,
|
||||
target: host,
|
||||
port: sec.checkPort,
|
||||
serverId: null,
|
||||
serverName: null,
|
||||
reason: `The secret “${sec.name}” already reads this certificate; a check also tells you when ${host} stops answering.`,
|
||||
});
|
||||
}
|
||||
|
||||
// By server, those about no server last, then by address.
|
||||
const sorted = [...all.values()].sort((a, b) => Number(a.serverName === null) - Number(b.serverName === null) || (a.serverName ?? "").localeCompare(b.serverName ?? "") || a.target.localeCompare(b.target));
|
||||
return {
|
||||
suggestions: sorted.filter((x) => !input.dismissed.has(x.key)).slice(0, MAX_SUGGESTIONS),
|
||||
dismissed: sorted.filter((x) => input.dismissed.has(x.key)),
|
||||
};
|
||||
}
|
||||
|
||||
// ─── What was said no to ───────────────────────────────────────────────────
|
||||
|
||||
const DISMISSED_FLAG = "endpointSuggestionsDismissed";
|
||||
const MAX_DISMISSED = 500;
|
||||
|
||||
export async function loadDismissed(): Promise<Set<string>> {
|
||||
const raw = await getInternalFlag(DISMISSED_FLAG);
|
||||
if (!raw) return new Set();
|
||||
try {
|
||||
const v = JSON.parse(raw);
|
||||
return new Set(Array.isArray(v) ? v.filter((k): k is string => typeof k === "string") : []);
|
||||
} catch {
|
||||
return new Set();
|
||||
}
|
||||
}
|
||||
|
||||
export async function saveDismissed(keys: Set<string>): Promise<void> {
|
||||
await setInternalFlag(DISMISSED_FLAG, JSON.stringify([...keys].slice(-MAX_DISMISSED)));
|
||||
}
|
||||
|
||||
// ─── Reading what is stored ────────────────────────────────────────────────
|
||||
|
||||
function jsonArray<T>(stored: string | null): T[] {
|
||||
if (!stored) return [];
|
||||
try {
|
||||
const v = JSON.parse(stored);
|
||||
return Array.isArray(v) ? (v as T[]) : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export async function loadSuggestions(): Promise<{ suggestions: Suggestion[]; dismissed: Suggestion[] }> {
|
||||
const [serverRows, dnsRows, secretRows, existing, dismissed] = await Promise.all([
|
||||
db.select({ id: servers.id, name: servers.name, hostname: servers.hostname, ipAddresses: servers.ipAddresses, listeningPorts: servers.listeningPorts }).from(servers),
|
||||
db.select({ name: dnsRecordsCache.name, type: dnsRecordsCache.type, content: dnsRecordsCache.content }).from(dnsRecordsCache),
|
||||
db.select({ name: secrets.name, checkHost: secrets.checkHost, checkPort: secrets.checkPort }).from(secrets),
|
||||
db.select({ kind: endpointChecks.kind, target: endpointChecks.target, port: endpointChecks.port }).from(endpointChecks),
|
||||
loadDismissed(),
|
||||
]);
|
||||
return suggestChecks({
|
||||
servers: serverRows.map((s) => ({ id: s.id, name: s.name, hostname: s.hostname, ips: jsonArray<string>(s.ipAddresses).filter((i) => typeof i === "string"), ports: jsonArray<{ protocol?: string; port: number; address?: string }>(s.listeningPorts).filter((p) => p && Number.isInteger(p.port)) })),
|
||||
dns: dnsRows,
|
||||
secrets: secretRows,
|
||||
existing,
|
||||
dismissed,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
/**
|
||||
* Heartbeats: "did my 3 a.m. job actually run?". A cron job or backup script calls an address when it finishes; the app writes down that it did, and
|
||||
* speaks up — an alert and a notification, like any other problem — when the call doesn't come in time, or when the job says it failed.
|
||||
*
|
||||
* Each heartbeat has an address with a long random token in it (`/api/heartbeat/<token>`) and a period: how often the job should report, plus a grace
|
||||
* period for jobs that run a little late. Three calls: `…/<token>` (it worked), `…/<token>/start` (it began, so the next call can say how long it took)
|
||||
* and `…/<token>/fail` (it ran and went wrong — this is an alert at once, without waiting for the time to pass). Anything after the call, in the
|
||||
* body or in `?msg=`, is kept as a short note ("3.2 GB, 41 s").
|
||||
*
|
||||
* The token is the whole of the access a call needs, and all it allows is to say "I'm alive": it can't read anything. It is kept as it is, not hashed
|
||||
* (unlike a share link), because the address has to be shown again when somebody sets the job up on another machine.
|
||||
*/
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { and, asc, desc, eq, inArray, lt } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { heartbeatPings, heartbeats } from "../db/schema.js";
|
||||
import { invalidateAlertsCache } from "./alerts.js";
|
||||
|
||||
export type HeartbeatRow = typeof heartbeats.$inferSelect;
|
||||
export type PingKind = "ok" | "fail" | "start";
|
||||
export const PING_KINDS: readonly PingKind[] = ["ok", "fail", "start"];
|
||||
|
||||
export const KEEP_PING_DAYS = 14;
|
||||
export const MAX_MESSAGE = 200;
|
||||
/** "hb_" and then random letters and digits: what an address is allowed to carry. */
|
||||
export const TOKEN_PATTERN = /^hb_[A-Za-z0-9_-]{20,64}$/;
|
||||
const MIN_PERIOD = 1;
|
||||
export const MAX_PERIOD_MINUTES = 60 * 24 * 30;
|
||||
export const MAX_GRACE_MINUTES = 60 * 24;
|
||||
/** A "started" call that is older than this is not what a later "finished" call is timing. */
|
||||
const MAX_RUN_MS = 7 * 86_400_000;
|
||||
|
||||
export const newToken = () => `hb_${randomBytes(18).toString("base64url")}`;
|
||||
|
||||
/** SQLite's own timestamps ("2026-10-19 03:10:00") are UTC without a "T"; ours are ISO. Both read as the instant they are. */
|
||||
export function parseStamp(value: string): number {
|
||||
return Date.parse(/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/.test(value) ? `${value.replace(" ", "T")}Z` : value);
|
||||
}
|
||||
|
||||
/** A note from a job, made fit to store and show: one line, no control characters, not long. */
|
||||
export function cleanMessage(raw: unknown): string | null {
|
||||
if (typeof raw !== "string") return null;
|
||||
// eslint-disable-next-line no-control-regex
|
||||
const text = raw.replace(/[\u0000-\u001f\u007f]+/g, " ").replace(/\s+/g, " ").trim();
|
||||
return text === "" ? null : text.slice(0, MAX_MESSAGE);
|
||||
}
|
||||
|
||||
// ─── What a heartbeat's state is ───────────────────────────────────────────
|
||||
|
||||
export type HeartbeatState = "off" | "waiting" | "up" | "running" | "late" | "failed";
|
||||
|
||||
/** When the next call is due by, counting the grace: from the last call, or from the day it was made if it has never called. */
|
||||
export function dueBy(h: Pick<HeartbeatRow, "periodMinutes" | "graceMinutes" | "lastPingAt" | "createdAt">): Date {
|
||||
const base = h.lastPingAt ? parseStamp(h.lastPingAt) : parseStamp(h.createdAt);
|
||||
return new Date(base + (h.periodMinutes + h.graceMinutes) * 60_000);
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure: how a heartbeat is doing. A job that said it failed is failed until it next succeeds; otherwise it is late once the time (period and
|
||||
* grace) has passed without a call. A new one that has not called yet is waiting, and is late if its first call is overdue too.
|
||||
*/
|
||||
export function heartbeatState(h: Pick<HeartbeatRow, "enabled" | "periodMinutes" | "graceMinutes" | "lastPingAt" | "lastStatus" | "lastStartAt" | "createdAt">, now: Date): HeartbeatState {
|
||||
if (!h.enabled) return "off";
|
||||
if (h.lastStatus === "fail") return "failed";
|
||||
if (now.getTime() > dueBy(h).getTime()) return "late";
|
||||
if (!h.lastPingAt) return "waiting";
|
||||
if (h.lastStartAt && parseStamp(h.lastStartAt) > parseStamp(h.lastPingAt)) return "running";
|
||||
return "up";
|
||||
}
|
||||
|
||||
// ─── What is wrong ─────────────────────────────────────────────────────────
|
||||
|
||||
export interface HeartbeatCondition {
|
||||
key: string;
|
||||
source: "heartbeat";
|
||||
message: string;
|
||||
severity: "critical" | "warning";
|
||||
category: "automation";
|
||||
heartbeatId: number;
|
||||
serverId: number | null;
|
||||
}
|
||||
|
||||
const stamp = (d: Date) => `${d.toISOString().slice(0, 16).replace("T", " ")} UTC`;
|
||||
|
||||
/**
|
||||
* The problems the heartbeats stand for. A call that didn't come in time is a warning, and says when it was due (a time, not "N minutes ago", so
|
||||
* the message doesn't change on every pass); a job that reported a failure is critical. The key carries the server a heartbeat is linked to, so
|
||||
* maintenance on it silences them.
|
||||
*/
|
||||
export function heartbeatConditions(rows: HeartbeatRow[], now: Date): HeartbeatCondition[] {
|
||||
const out: HeartbeatCondition[] = [];
|
||||
for (const h of rows) {
|
||||
const state = heartbeatState(h, now);
|
||||
const suffix = h.serverId !== null ? `:server:${h.serverId}` : "";
|
||||
if (state === "late") {
|
||||
const heard = h.lastPingAt ? `last heard ${stamp(new Date(parseStamp(h.lastPingAt)))}` : "it has never reported";
|
||||
out.push({ key: `heartbeat:${h.id}${suffix}`, source: "heartbeat", category: "automation", severity: "warning", heartbeatId: h.id, serverId: h.serverId, message: `${h.name} hasn't reported — it was due by ${stamp(dueBy(h))} (${heard})` });
|
||||
} else if (state === "failed") {
|
||||
out.push({ key: `heartbeat-fail:${h.id}${suffix}`, source: "heartbeat", category: "automation", severity: "critical", heartbeatId: h.id, serverId: h.serverId, message: `${h.name} reported a failure${h.lastMessage ? ` — ${h.lastMessage}` : ""}` });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ─── A call coming in ──────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Writes down a call. Returns the heartbeat as it is now, or null when there is no heartbeat with that token. A heartbeat that is switched off
|
||||
* still takes calls (so a script isn't broken by pausing it) and notes them, so that switching it back on starts from a fresh clock.
|
||||
*/
|
||||
export async function recordPing(token: string, kind: PingKind, message: string | null, now: Date = new Date()): Promise<HeartbeatRow | null> {
|
||||
if (!TOKEN_PATTERN.test(token)) return null;
|
||||
const [h] = await db.select().from(heartbeats).where(eq(heartbeats.token, token)).limit(1);
|
||||
if (!h) return null;
|
||||
const iso = now.toISOString();
|
||||
let durationMs: number | null = null;
|
||||
let updated: HeartbeatRow;
|
||||
if (kind === "start") {
|
||||
[updated] = await db.update(heartbeats).set({ lastStartAt: iso }).where(eq(heartbeats.id, h.id)).returning();
|
||||
} else {
|
||||
if (h.lastStartAt) {
|
||||
const ran = now.getTime() - parseStamp(h.lastStartAt);
|
||||
const startedAfterLastEnd = !h.lastPingAt || parseStamp(h.lastStartAt) > parseStamp(h.lastPingAt);
|
||||
if (startedAfterLastEnd && ran >= 0 && ran <= MAX_RUN_MS) durationMs = ran;
|
||||
}
|
||||
[updated] = await db
|
||||
.update(heartbeats)
|
||||
.set({ lastPingAt: iso, lastStatus: kind, lastMessage: message, lastDurationMs: durationMs, pingCount: h.pingCount + 1 })
|
||||
.where(eq(heartbeats.id, h.id))
|
||||
.returning();
|
||||
}
|
||||
await db.insert(heartbeatPings).values({ heartbeatId: h.id, at: iso, kind, durationMs, message });
|
||||
invalidateAlertsCache();
|
||||
return updated;
|
||||
}
|
||||
|
||||
// ─── History ───────────────────────────────────────────────────────────────
|
||||
|
||||
export interface PingSummary {
|
||||
at: string;
|
||||
kind: PingKind;
|
||||
durationMs: number | null;
|
||||
}
|
||||
|
||||
/** The last calls of each heartbeat that ended a run (success or failure), oldest first. */
|
||||
export async function loadRecent(ids: number[], perHeartbeat = 30): Promise<Map<number, PingSummary[]>> {
|
||||
const out = new Map<number, PingSummary[]>();
|
||||
if (ids.length === 0) return out;
|
||||
const rows = await db
|
||||
.select({ heartbeatId: heartbeatPings.heartbeatId, at: heartbeatPings.at, kind: heartbeatPings.kind, durationMs: heartbeatPings.durationMs })
|
||||
.from(heartbeatPings)
|
||||
.where(and(inArray(heartbeatPings.heartbeatId, ids), inArray(heartbeatPings.kind, ["ok", "fail"])))
|
||||
.orderBy(desc(heartbeatPings.at));
|
||||
for (const r of rows) {
|
||||
const list = out.get(r.heartbeatId) ?? [];
|
||||
if (list.length < perHeartbeat) list.push({ at: r.at, kind: r.kind as PingKind, durationMs: r.durationMs });
|
||||
out.set(r.heartbeatId, list);
|
||||
}
|
||||
for (const list of out.values()) list.reverse();
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function prunePings(now: Date = new Date()): Promise<void> {
|
||||
await db.delete(heartbeatPings).where(lt(heartbeatPings.at, new Date(now.getTime() - KEEP_PING_DAYS * 86_400_000).toISOString()));
|
||||
}
|
||||
|
||||
export const periodLimits = { min: MIN_PERIOD, max: MAX_PERIOD_MINUTES };
|
||||
|
||||
/** The table order every list uses. */
|
||||
export const listHeartbeats = () => db.select().from(heartbeats).orderBy(asc(heartbeats.name));
|
||||
@@ -25,7 +25,7 @@ export async function listUpcomingWindows(now: Date = new Date()): Promise<Activ
|
||||
export function subjectOfConditionKey(key: string): string | null {
|
||||
let m = /^(?:offline|disk|failed-units|smart|zfs):server:(\d+)(?::|$)/.exec(key);
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^endpoint(?:-cert)?:\d+:server:(\d+)$/.exec(key);
|
||||
m = /^(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:(\d+)$/.exec(key);
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^disk:proxmox:(\d+):/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
|
||||
@@ -4,8 +4,9 @@
|
||||
* integration on its own, so a scrape every fifteen seconds costs a few database reads (the alerts are the app's own minute-old result).
|
||||
*/
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, endpointChecks, integrations, maintenanceWindows, secrets, servers } from "../db/schema.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers } from "../db/schema.js";
|
||||
import { latestTests } from "./restoreTests.js";
|
||||
import { heartbeatState, parseStamp } from "./heartbeats.js";
|
||||
import { env } from "../env.js";
|
||||
import { getAlerts } from "./alerts.js";
|
||||
import { isAgentOutdated, latestAgentVersions } from "./agentVersions.js";
|
||||
@@ -70,6 +71,18 @@ async function addEndpointMetrics(out: Exposition, now: Date) {
|
||||
}
|
||||
}
|
||||
|
||||
/** Jobs that report in, for /metrics. */
|
||||
async function addHeartbeatMetrics(out: Exposition, now: Date) {
|
||||
for (const h of await db.select().from(heartbeats)) {
|
||||
if (!h.enabled) continue;
|
||||
const labels = { name: h.name };
|
||||
const state = heartbeatState(h, now);
|
||||
out.add("homelab_heartbeat_up", "gauge", "1 if the job has reported in time and not said it failed, else 0.", state === "late" || state === "failed" ? 0 : 1, labels);
|
||||
if (h.lastPingAt) out.add("homelab_heartbeat_last_ping_timestamp_seconds", "gauge", "When the job last reported that it finished (Unix time).", Math.floor(parseStamp(h.lastPingAt) / 1000), labels);
|
||||
if (h.lastDurationMs !== null) out.add("homelab_heartbeat_last_duration_seconds", "gauge", "How long the job said its last run took.", Math.round(h.lastDurationMs / 100) / 10, labels);
|
||||
}
|
||||
}
|
||||
|
||||
export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
const started = Date.now();
|
||||
const out = new Exposition();
|
||||
@@ -195,5 +208,6 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
out.add("homelab_scrape_duration_seconds", "gauge", "How long building this answer took.", (Date.now() - started) / 1000);
|
||||
out.add("homelab_scrape_timestamp_seconds", "gauge", "When this answer was built, as a Unix time.", nowSec);
|
||||
await addEndpointMetrics(out, now);
|
||||
await addHeartbeatMetrics(out, now);
|
||||
return out.toString();
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
/**
|
||||
* The name lists behind Operations → Generator's server-name picker.
|
||||
* The name lists behind Administration → Generator's server-name picker.
|
||||
*
|
||||
* The built-in lists below are hand-picked, not taken from any official source — they're a starting point. Admins can
|
||||
* edit every list under Settings → Names, and can replace a Swedish list with real statistics from Skatteverket.
|
||||
|
||||
@@ -109,8 +109,9 @@ export function metaOfConditionKey(key: string, severity?: string): NotifyMeta {
|
||||
else if (key.startsWith("failed-units:")) category = "services";
|
||||
else if (/^(disk|smart|zfs|synology-volume|synology-disk):/.test(key)) category = "disk";
|
||||
else if (key.startsWith("automation:")) category = "automation";
|
||||
else if (key.startsWith("heartbeat")) category = "automation";
|
||||
else if (key.startsWith("endpoint-cert:")) category = "expiry";
|
||||
else if (key.startsWith("endpoint:")) category = "monitoring";
|
||||
const server = /^(?:(?:offline|disk|failed-units|smart|zfs):server:|endpoint(?:-cert)?:\d+:server:)(\d+)/.exec(key);
|
||||
const server = /^(?:(?:offline|disk|failed-units|smart|zfs):server:|(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:)(\d+)/.exec(key);
|
||||
return { category, severity: severity === "critical" ? "critical" : "warning", serverId: server ? Number(server[1]) : null };
|
||||
}
|
||||
Reference in new issue
Block a user