Fold alerts under their host, recurring maintenance, run playbooks from alerts, and an Updates page
- Servers can say what they run on (or it is worked out from the Proxmox node); when the host is down the guests' offline alerts fold under it: held back, not counted, not notified, still recorded. - Recurring maintenance schedules made into ordinary windows a week ahead (maintenance_schedules). - Run a Semaphore template limited to a server from its page or an alert, and watch the run. - Agents report pending updates (apt, dnf, zypper, Windows Update); an Updates page lists them. - Migrations 0024 and 0025. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
44e393f172
commit
171511c6db
45 files changed
+6772
-44
No files matched your search
@@ -18,6 +18,10 @@ export interface Alert {
|
||||
link: string | null;
|
||||
/** Under an active maintenance window: still a real problem, but notifications for it are held back. */
|
||||
silenced: boolean;
|
||||
/** Set when this problem is explained by another one (a guest down because its host is): it is held back like a silenced one, but the reason is this, not maintenance. */
|
||||
causedBy?: { name: string };
|
||||
/** On a host's alert: the servers on it whose own alerts are folded under this one. */
|
||||
affects?: string[];
|
||||
/** What a maintenance window would have to be on to silence it, when there is such a thing — which is what "Silence" acts on. */
|
||||
target: { type: "server" | "integration" | "dns_provider"; id: number; name: string } | null;
|
||||
}
|
||||
|
||||
@@ -28,12 +28,13 @@ import { computeSecretStatus } from "./secretStatus.js";
|
||||
import { getFailingSources } from "./integrationHealthMonitor.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { sourceLabel } from "./notify.js";
|
||||
import { coveredByParent, loadParentRows, resolveParents } from "./serverParents.js";
|
||||
import type { Alert, AlertCategory, AlertSeverity, SourceFailure } from "./alertTypes.js";
|
||||
|
||||
export interface AlertsReport {
|
||||
alerts: Alert[];
|
||||
/** Active problems by severity — those under a maintenance window are counted apart, not in these. */
|
||||
counts: { critical: number; warning: number; info: number; silenced: number };
|
||||
counts: { critical: number; warning: number; info: number; silenced: number; /** Held back because another problem explains them (a guest down because its host is). */ grouped: number };
|
||||
/** Things that couldn't be checked, and which checks that leaves blind. */
|
||||
couldntCheck: { name: string; error: string; affects: string[] }[];
|
||||
/** Context worth knowing about how complete the picture is. */
|
||||
@@ -115,7 +116,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
return row ? integrationTarget(row.id) : null;
|
||||
};
|
||||
|
||||
function push(a: { category: AlertCategory; severity: AlertSeverity; type: string; message: string; key: string; link?: string | null; silenced?: boolean; target?: Alert["target"] }) {
|
||||
function push(a: { category: AlertCategory; severity: AlertSeverity; type: string; message: string; key: string; link?: string | null; silenced?: boolean; target?: Alert["target"]; causedBy?: Alert["causedBy"]; affects?: string[] }) {
|
||||
const meta = SOURCES[a.type];
|
||||
alerts.push({
|
||||
id: `${a.category}:${a.key}`,
|
||||
@@ -124,8 +125,10 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
source: meta?.label ?? sourceLabel(a.type),
|
||||
message: a.message,
|
||||
link: a.link === undefined ? (meta?.link ?? null) : a.link,
|
||||
silenced: !!a.silenced,
|
||||
silenced: !!a.silenced || !!a.causedBy,
|
||||
target: a.target ?? null,
|
||||
...(a.causedBy ? { causedBy: a.causedBy } : {}),
|
||||
...(a.affects?.length ? { affects: a.affects } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -160,10 +163,24 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
`Server-offline and server-disk checks are paused for another ${Math.ceil(grace / 60_000)} min after the app restarted, so agents get a chance to report before any server is judged.`,
|
||||
);
|
||||
}
|
||||
for (const c of evaluateHealth(snapshot, healthChecks, now, { skipServers: grace > 0 })) {
|
||||
const conditions = evaluateHealth(snapshot, healthChecks, now, { skipServers: grace > 0 });
|
||||
// A server that is down because the one it runs on is down is folded under that one.
|
||||
const offlineIds = new Set<number>();
|
||||
for (const c of conditions) {
|
||||
const m = /^offline:server:(\d+)$/.exec(c.key);
|
||||
if (m) offlineIds.add(Number(m[1]));
|
||||
}
|
||||
const cause = coveredByParent(offlineIds, resolveParents(await loadParentRows()));
|
||||
const taken = new Map<number, string[]>();
|
||||
for (const [child, parent] of cause) taken.set(parent, [...(taken.get(parent) ?? []), serverNames.get(child) ?? `#${child}`]);
|
||||
for (const c of conditions) {
|
||||
const origin = originOfConditionKey(c.key);
|
||||
const subject = subjectOfConditionKey(c.key);
|
||||
const offlineId = /^offline:server:(\d+)$/.exec(c.key)?.[1];
|
||||
const parentOf = offlineId ? cause.get(Number(offlineId)) : undefined;
|
||||
push({
|
||||
causedBy: parentOf !== undefined ? { name: serverNames.get(parentOf) ?? `#${parentOf}` } : undefined,
|
||||
affects: offlineId ? taken.get(Number(offlineId))?.sort((a, b) => a.localeCompare(b)) : undefined,
|
||||
category: origin.category,
|
||||
severity: c.severity ?? "warning",
|
||||
type: origin.type,
|
||||
@@ -403,13 +420,15 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
);
|
||||
|
||||
const active = alerts.filter((a) => !a.silenced);
|
||||
const grouped = alerts.filter((a) => a.causedBy).length;
|
||||
return {
|
||||
alerts,
|
||||
counts: {
|
||||
critical: active.filter((a) => a.severity === "critical").length,
|
||||
warning: active.filter((a) => a.severity === "warning").length,
|
||||
info: active.filter((a) => a.severity === "info").length,
|
||||
silenced: alerts.length - active.length,
|
||||
silenced: alerts.length - active.length - grouped,
|
||||
grouped,
|
||||
},
|
||||
couldntCheck: [...blind.entries()].map(([name, v]) => ({ name, error: v.error, affects: [...v.affects] })),
|
||||
notes,
|
||||
|
||||
@@ -8,10 +8,11 @@ import { notifyHealthIssues, notifyHealthRecovered } from "./notify.js";
|
||||
import { recordAudit } from "./audit.js";
|
||||
import { getInternalFlag, getSettings, setInternalFlag } from "./settingsStore.js";
|
||||
import { activeSubjects, subjectOfConditionKey } from "./maintenance.js";
|
||||
import { coveredByParent, loadParentRows, resolveParents } from "./serverParents.js";
|
||||
|
||||
const STATE_FLAG = "healthActiveConditions";
|
||||
/** After a restart the agents haven't had a chance to report yet (they run every 15 min), so server-derived conditions are held rather than judged. */
|
||||
const STARTUP_GRACE_MS = 20 * 60 * 1000;
|
||||
const STARTUP_GRACE_MS = Number(process.env.HEALTH_STARTUP_GRACE_MS ?? 20 * 60 * 1000); // the environment variable is for tests
|
||||
const PROCESS_START = Date.now();
|
||||
|
||||
// ─── Types ──────────────────────────────────────────────────────────────────
|
||||
@@ -212,7 +213,8 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
|
||||
// ─── State diff (pure) ──────────────────────────────────────────────────────
|
||||
|
||||
export type ActiveState = Record<string, { message: string; source: string }>;
|
||||
/** `covered`: the problem is explained by another that is being reported (a guest down because its host is) — recorded, but not announced on its own. */
|
||||
export type ActiveState = Record<string, { message: string; source: string; covered?: boolean }>;
|
||||
|
||||
/** `held` is a set of hierarchical sources that couldn't be read this run; matches the source itself or anything beneath it. */
|
||||
function isHeld(source: string, held: Set<string>): boolean {
|
||||
@@ -226,6 +228,10 @@ function isHeld(source: string, held: Set<string>): boolean {
|
||||
* now is resolved — except when its source couldn't be read this run, in
|
||||
* which case it's carried over untouched. Without that, one failed poll would
|
||||
* report a recovery and then re-alert the same problem on the next poll.
|
||||
*
|
||||
* `covered` are keys of problems explained by another (a guest that is down because its host is). They are kept in the state and
|
||||
* counted in `newlyDown` / `ended` (what happened, for the record), but are neither announced as new nor as resolved — until the
|
||||
* explanation goes away while they are still there, when they are announced as new: an outage is never swallowed.
|
||||
*/
|
||||
export function diffConditions(
|
||||
previous: ActiveState,
|
||||
@@ -233,7 +239,16 @@ export function diffConditions(
|
||||
held: Set<string> = new Set(),
|
||||
/** Subjects ("server:3", "integration:1") under an active maintenance window. */
|
||||
silenced: Set<string> = new Set(),
|
||||
): { added: HealthCondition[]; resolved: { key: string; message: string }[]; next: ActiveState } {
|
||||
covered: Set<string> = new Set(),
|
||||
): {
|
||||
added: HealthCondition[];
|
||||
resolved: { key: string; message: string }[];
|
||||
next: ActiveState;
|
||||
/** Every problem that began, announced or not. */
|
||||
newlyDown: HealthCondition[];
|
||||
/** Every problem that ended, announced or not. */
|
||||
ended: { key: string; message: string }[];
|
||||
} {
|
||||
const isSilenced = (key: string) => {
|
||||
const subject = subjectOfConditionKey(key);
|
||||
return subject !== null && silenced.has(subject);
|
||||
@@ -243,20 +258,24 @@ export function diffConditions(
|
||||
const visible = current.filter((c) => !isSilenced(c.key));
|
||||
|
||||
const next: ActiveState = {};
|
||||
for (const c of visible) next[c.key] = { message: c.message, source: c.source };
|
||||
for (const c of visible) next[c.key] = covered.has(c.key) ? { message: c.message, source: c.source, covered: true } : { message: c.message, source: c.source };
|
||||
|
||||
const added = visible.filter((c) => !(c.key in previous));
|
||||
const newlyDown = visible.filter((c) => !(c.key in previous));
|
||||
// Announced: new and standing alone, or covered until now and no longer.
|
||||
const added = visible.filter((c) => !covered.has(c.key) && (!(c.key in previous) || previous[c.key].covered));
|
||||
const resolved: { key: string; message: string }[] = [];
|
||||
const ended: { key: string; message: string }[] = [];
|
||||
for (const [key, entry] of Object.entries(previous)) {
|
||||
if (key in next) continue;
|
||||
// Held (source unreadable) or silenced: leave it exactly as it was — neither cleared nor re-alerted.
|
||||
if (isHeld(entry.source, held) || isSilenced(key)) {
|
||||
next[key] = entry;
|
||||
} else {
|
||||
resolved.push({ key, message: entry.message });
|
||||
ended.push({ key, message: entry.message });
|
||||
if (!entry.covered) resolved.push({ key, message: entry.message });
|
||||
}
|
||||
}
|
||||
return { added, resolved, next };
|
||||
return { added, resolved, next, newlyDown, ended };
|
||||
}
|
||||
|
||||
// ─── Collection (I/O) ───────────────────────────────────────────────────────
|
||||
@@ -346,6 +365,14 @@ export async function recordServerTransitions(added: { key: string; message: str
|
||||
await write(resolved, { offline: "back_online", disk: "disk_ok" }, false);
|
||||
}
|
||||
|
||||
/** "a, b and c", "a, b, c, d, e and 3 more". */
|
||||
export function alsoList(names: string[]): string {
|
||||
const shown = names.slice(0, 5);
|
||||
const rest = names.length - shown.length;
|
||||
const head = shown.length > 1 ? `${shown.slice(0, -1).join(", ")} and ${shown[shown.length - 1]}` : shown[0];
|
||||
return rest > 0 ? `${shown.join(", ")} and ${rest} more` : head;
|
||||
}
|
||||
|
||||
export async function runHealthCheck(now: number = Date.now()): Promise<{ added: number; resolved: number }> {
|
||||
const { healthChecks } = await getSettings();
|
||||
const { snapshot, held } = await collectSnapshot();
|
||||
@@ -354,12 +381,30 @@ export async function runHealthCheck(now: number = Date.now()): Promise<{ added:
|
||||
if (inGrace) held.add("server");
|
||||
|
||||
const current = evaluateHealth(snapshot, healthChecks, now, { skipServers: inGrace });
|
||||
const { added, resolved, next } = diffConditions(await loadState(), current, held, await activeSubjects(new Date(now)));
|
||||
// A server that is down because the one it runs on is down is folded under that one: recorded, but not announced on its own.
|
||||
const offlineIds = new Set<number>();
|
||||
for (const c of current) {
|
||||
const m = /^offline:server:(\d+)$/.exec(c.key);
|
||||
if (m) offlineIds.add(Number(m[1]));
|
||||
}
|
||||
const cause = coveredByParent(offlineIds, resolveParents(await loadParentRows()));
|
||||
const coveredKeys = new Set([...cause.keys()].map((id) => `offline:server:${id}`));
|
||||
const { added: announced, resolved, next, newlyDown, ended } = diffConditions(await loadState(), current, held, await activeSubjects(new Date(now)), coveredKeys);
|
||||
await setInternalFlag(STATE_FLAG, JSON.stringify(next));
|
||||
|
||||
// A host's message says what else it takes with it.
|
||||
const names = new Map(snapshot.servers.map((s) => [s.id, s.name]));
|
||||
const taken = new Map<number, string[]>();
|
||||
for (const [child, parent] of cause) taken.set(parent, [...(taken.get(parent) ?? []), names.get(child) ?? `#${child}`]);
|
||||
const added = announced.map((c) => {
|
||||
const m = /^offline:server:(\d+)$/.exec(c.key);
|
||||
const children = m ? taken.get(Number(m[1])) : undefined;
|
||||
return children?.length ? { ...c, message: `${c.message} — also takes down ${alsoList(children)}` } : c;
|
||||
});
|
||||
|
||||
// State is tracked even when the alert toggle is off (the notify functions check it),
|
||||
// so turning alerts back on doesn't dump every long-standing condition at once.
|
||||
await recordServerTransitions(added, resolved);
|
||||
await recordServerTransitions(newlyDown, ended);
|
||||
if (added.length > 0) await notifyHealthIssues(added);
|
||||
if (resolved.length > 0) await notifyHealthRecovered(resolved);
|
||||
return { added: added.length, resolved: resolved.length };
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
import { materializeSchedules } from "./maintenanceSchedules.js";
|
||||
|
||||
// Windows are made a week ahead, so a pass is only needed now and then: to make the next day's, and to pick up a server added to a
|
||||
// schedule's list. Quarter-hourly keeps a restart from leaving a gap.
|
||||
const INTERVAL_MS = 15 * 60 * 1000;
|
||||
const FIRST_PASS_DELAY_MS = 30 * 1000;
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
async function pass(): Promise<void> {
|
||||
try {
|
||||
const made = await materializeSchedules();
|
||||
if (made > 0) console.log(`[maintenanceSchedules] made ${made} maintenance window${made === 1 ? "" : "s"} from recurring schedules`);
|
||||
} catch (err) {
|
||||
console.error("[maintenanceSchedules]", err instanceof Error ? err.message : err);
|
||||
}
|
||||
}
|
||||
|
||||
export async function initMaintenanceScheduleScheduler(): Promise<void> {
|
||||
if (timer) return;
|
||||
timer = setInterval(() => void pass(), INTERVAL_MS);
|
||||
setTimeout(() => void pass(), FIRST_PASS_DELAY_MS);
|
||||
console.log(`[maintenanceSchedules] making windows from recurring schedules a week ahead, checked every ${INTERVAL_MS / 60000} minutes`);
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
/**
|
||||
* Recurring maintenance: "every Sunday 02:00–04:00 on these servers". A schedule only says when; the windows that silence alerts are
|
||||
* ordinary maintenance windows made a week ahead (so they show on the timeline and can be cancelled one at a time, like any other).
|
||||
* Each is marked `schedule:<id>:<date>`, which is how a later pass knows it has already made it — even if someone cancelled it.
|
||||
*/
|
||||
import { and, eq, gt, like } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { maintenanceSchedules, maintenanceWindows, type MaintenanceTargetType } from "../db/schema.js";
|
||||
import { invalidateAlertsCache } from "./alerts.js";
|
||||
import { describeTarget } from "./maintenance.js";
|
||||
|
||||
export const HORIZON_DAYS = 7;
|
||||
const DAY_MS = 24 * 3600_000;
|
||||
|
||||
export type ScheduleRow = typeof maintenanceSchedules.$inferSelect;
|
||||
export interface ScheduleTarget {
|
||||
type: MaintenanceTargetType;
|
||||
id: number;
|
||||
}
|
||||
|
||||
export interface Occurrence {
|
||||
start: Date;
|
||||
end: Date;
|
||||
/** The local calendar date it starts on, "2026-10-18" — unique per schedule. */
|
||||
date: string;
|
||||
}
|
||||
|
||||
/** Whether the system knows this time zone name. */
|
||||
export function isValidTimeZone(tz: string): boolean {
|
||||
try {
|
||||
new Intl.DateTimeFormat("en-US", { timeZone: tz });
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
const parts = (utcMs: number, tz: string) => {
|
||||
const f = new Intl.DateTimeFormat("en-US", { timeZone: tz, hourCycle: "h23", year: "numeric", month: "2-digit", day: "2-digit", hour: "2-digit", minute: "2-digit", second: "2-digit" });
|
||||
const out: Record<string, number> = {};
|
||||
for (const p of f.formatToParts(new Date(utcMs))) if (p.type !== "literal") out[p.type] = Number(p.value);
|
||||
return out;
|
||||
};
|
||||
|
||||
/** How far the clock in `tz` is ahead of UTC at that moment, in ms. */
|
||||
function offsetAt(utcMs: number, tz: string): number {
|
||||
const p = parts(utcMs, tz);
|
||||
return Date.UTC(p.year, p.month - 1, p.day, p.hour, p.minute, p.second) - Math.floor(utcMs / 1000) * 1000;
|
||||
}
|
||||
|
||||
/** The instant a wall-clock time in `tz` happens. Winter/summer time is settled by asking the zone for its offset at the answer. */
|
||||
export function zonedTime(year: number, month: number, day: number, hour: number, minute: number, tz: string): Date {
|
||||
const naive = Date.UTC(year, month - 1, day, hour, minute);
|
||||
let guess = naive - offsetAt(naive, tz);
|
||||
guess = naive - offsetAt(guess, tz);
|
||||
return new Date(guess);
|
||||
}
|
||||
|
||||
const pad = (n: number) => String(n).padStart(2, "0");
|
||||
|
||||
/**
|
||||
* The times a schedule happens that touch [from, to]: those that have not ended by `from` and begin before `to`. A schedule that runs
|
||||
* past midnight belongs to the day it starts on.
|
||||
*/
|
||||
export function occurrencesBetween(schedule: Pick<ScheduleRow, "days" | "startTime" | "durationMinutes" | "timezone">, from: Date, to: Date): Occurrence[] {
|
||||
const days = new Set<number>(JSON.parse(schedule.days) as number[]);
|
||||
const [hh, mm] = schedule.startTime.split(":").map(Number);
|
||||
const out: Occurrence[] = [];
|
||||
// Walk the calendar dates around the range (a day either side covers a schedule that started yesterday, or zone differences).
|
||||
const first = parts(from.getTime() - DAY_MS, schedule.timezone);
|
||||
let cursor = Date.UTC(first.year, first.month - 1, first.day);
|
||||
const last = to.getTime() + DAY_MS;
|
||||
for (; cursor <= last; cursor += DAY_MS) {
|
||||
const d = new Date(cursor);
|
||||
if (!days.has(d.getUTCDay())) continue;
|
||||
const y = d.getUTCFullYear();
|
||||
const m = d.getUTCMonth() + 1;
|
||||
const day = d.getUTCDate();
|
||||
const start = zonedTime(y, m, day, hh, mm, schedule.timezone);
|
||||
const end = new Date(start.getTime() + schedule.durationMinutes * 60_000);
|
||||
if (end > from && start < to) out.push({ start, end, date: `${y}-${pad(m)}-${pad(day)}` });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function targetsOf(schedule: Pick<ScheduleRow, "targets">): ScheduleTarget[] {
|
||||
try {
|
||||
const v = JSON.parse(schedule.targets);
|
||||
return Array.isArray(v) ? v.filter((t): t is ScheduleTarget => !!t && typeof t.id === "number" && typeof t.type === "string") : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export const sourceOf = (scheduleId: number, date: string) => `schedule:${scheduleId}:${date}`;
|
||||
|
||||
/**
|
||||
* Makes the windows for the coming week that don't exist yet. Safe to run as often as you like: a window already made (including one
|
||||
* that was cancelled) is left as it is. Returns how many it made.
|
||||
*/
|
||||
export async function materializeSchedules(now: Date = new Date(), horizonDays: number = HORIZON_DAYS, onlyScheduleId?: number): Promise<number> {
|
||||
const schedules = (await db.select().from(maintenanceSchedules)).filter((s) => s.enabled && (onlyScheduleId === undefined || s.id === onlyScheduleId));
|
||||
const horizon = new Date(now.getTime() + horizonDays * DAY_MS);
|
||||
let made = 0;
|
||||
for (const schedule of schedules) {
|
||||
const occurrences = occurrencesBetween(schedule, now, horizon);
|
||||
if (occurrences.length === 0) continue;
|
||||
const existing = await db.select({ source: maintenanceWindows.source, targetType: maintenanceWindows.targetType, targetId: maintenanceWindows.targetId }).from(maintenanceWindows).where(like(maintenanceWindows.source, `schedule:${schedule.id}:%`));
|
||||
const have = new Set(existing.map((w) => `${w.source}|${w.targetType}:${w.targetId}`));
|
||||
for (const target of targetsOf(schedule)) {
|
||||
if (!(await describeTarget(target.type, target.id))) continue; // since removed: nothing left to silence
|
||||
for (const occ of occurrences) {
|
||||
const source = sourceOf(schedule.id, occ.date);
|
||||
if (have.has(`${source}|${target.type}:${target.id}`)) continue;
|
||||
await db.insert(maintenanceWindows).values({
|
||||
targetType: target.type,
|
||||
targetId: target.id,
|
||||
reason: schedule.reason ? `${schedule.name}: ${schedule.reason}` : schedule.name,
|
||||
startedAt: occ.start.toISOString(),
|
||||
endsAt: occ.end.toISOString(),
|
||||
createdBy: `Recurring: ${schedule.name}`,
|
||||
source,
|
||||
});
|
||||
made++;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (made > 0) invalidateAlertsCache();
|
||||
return made;
|
||||
}
|
||||
|
||||
/** Takes back the windows a schedule made that haven't begun — when it is switched off or removed. Ones under way end on their own. */
|
||||
export async function removeFutureWindows(scheduleId: number, now: Date = new Date()): Promise<number> {
|
||||
const gone = await db
|
||||
.delete(maintenanceWindows)
|
||||
.where(and(like(maintenanceWindows.source, `schedule:${scheduleId}:%`), gt(maintenanceWindows.startedAt, now.toISOString())))
|
||||
.returning({ id: maintenanceWindows.id });
|
||||
if (gone.length > 0) invalidateAlertsCache();
|
||||
return gone.length;
|
||||
}
|
||||
|
||||
export async function deleteSchedule(scheduleId: number): Promise<void> {
|
||||
await removeFutureWindows(scheduleId);
|
||||
await db.delete(maintenanceSchedules).where(eq(maintenanceSchedules.id, scheduleId));
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
/**
|
||||
* What each server says is waiting to be installed: how many updates, how many of them security fixes, whether a restart is due — and
|
||||
* how long updates have been waiting, which the agent can't know and so is worked out here from one report to the next.
|
||||
*/
|
||||
|
||||
/** What an agent reports about updates, as it arrives. */
|
||||
export interface IncomingUpdates {
|
||||
/** apt, dnf, zypper, windows-update … — what it asked. */
|
||||
manager?: string;
|
||||
count: number;
|
||||
/** Null when this package manager can't say which are security fixes. */
|
||||
security?: number | null;
|
||||
reboot_required?: boolean;
|
||||
}
|
||||
|
||||
export interface PendingUpdates {
|
||||
manager: string;
|
||||
count: number;
|
||||
security: number | null;
|
||||
rebootRequired: boolean;
|
||||
/** ISO — when the agent last told us. */
|
||||
reportedAt: string;
|
||||
}
|
||||
|
||||
export function parsePendingUpdates(stored: string | null): PendingUpdates | null {
|
||||
if (!stored) return null;
|
||||
try {
|
||||
const v = JSON.parse(stored);
|
||||
if (!v || typeof v !== "object" || typeof v.count !== "number") return null;
|
||||
return {
|
||||
manager: typeof v.manager === "string" ? v.manager : "",
|
||||
count: v.count,
|
||||
security: typeof v.security === "number" ? v.security : null,
|
||||
rebootRequired: v.rebootRequired === true,
|
||||
reportedAt: typeof v.reportedAt === "string" ? v.reportedAt : "",
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The columns to store after a report. Updates waiting since "since" keep that moment for as long as some are still waiting; when
|
||||
* none are, it is cleared, and the next time some appear it starts again from then.
|
||||
*/
|
||||
export function nextUpdateState(
|
||||
previous: { pendingUpdates: string | null; updatesPendingSince: string | null },
|
||||
incoming: IncomingUpdates,
|
||||
now: string,
|
||||
): { pendingUpdates: string; updatesPendingSince: string | null } {
|
||||
const before = parsePendingUpdates(previous.pendingUpdates);
|
||||
const waiting = incoming.count > 0;
|
||||
const since = waiting ? (before && before.count > 0 && previous.updatesPendingSince ? previous.updatesPendingSince : now) : null;
|
||||
const record: PendingUpdates = {
|
||||
manager: incoming.manager ?? "",
|
||||
count: incoming.count,
|
||||
security: typeof incoming.security === "number" ? incoming.security : null,
|
||||
rebootRequired: incoming.reboot_required === true,
|
||||
reportedAt: now,
|
||||
};
|
||||
return { pendingUpdates: JSON.stringify(record), updatesPendingSince: since };
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
/**
|
||||
* Which server runs on which: a virtual machine on its Proxmox host, a container on a Docker box. When the host stops reporting, the
|
||||
* things on it stop too, and seven alerts about seven guests only hide the one that matters — so those alerts are folded under the
|
||||
* host's. The link is either set by hand ("runs on") or worked out: a server linked to a Proxmox guest runs on the server named after
|
||||
* that guest's Proxmox node, when there is exactly one such server.
|
||||
*/
|
||||
import { db } from "../db/client.js";
|
||||
import { servers } from "../db/schema.js";
|
||||
|
||||
export interface ParentRow {
|
||||
id: number;
|
||||
name: string;
|
||||
hostname: string | null;
|
||||
parentServerId: number | null;
|
||||
proxmoxNode: string | null;
|
||||
}
|
||||
|
||||
export interface ParentLink {
|
||||
parentId: number;
|
||||
/** Worked out from the Proxmox node rather than set by hand. */
|
||||
auto: boolean;
|
||||
}
|
||||
|
||||
const lc = (s: string | null | undefined) => (s ?? "").trim().toLowerCase();
|
||||
|
||||
/** Whether a server is the machine a Proxmox node name refers to: by its name, its hostname, or the first part of a fully qualified hostname. */
|
||||
function isNode(row: ParentRow, node: string): boolean {
|
||||
const n = lc(node);
|
||||
if (!n) return false;
|
||||
const host = lc(row.hostname);
|
||||
return lc(row.name) === n || host === n || host.split(".")[0] === n;
|
||||
}
|
||||
|
||||
/** Each server's parent, where it has one. A link set by hand wins; otherwise the unique server that is its Proxmox node. */
|
||||
export function resolveParents(rows: ParentRow[]): Map<number, ParentLink> {
|
||||
const byId = new Map(rows.map((r) => [r.id, r]));
|
||||
const out = new Map<number, ParentLink>();
|
||||
for (const row of rows) {
|
||||
if (row.parentServerId !== null && row.parentServerId !== row.id && byId.has(row.parentServerId)) {
|
||||
out.set(row.id, { parentId: row.parentServerId, auto: false });
|
||||
continue;
|
||||
}
|
||||
if (row.proxmoxNode) {
|
||||
const candidates = rows.filter((r) => r.id !== row.id && isNode(r, row.proxmoxNode!));
|
||||
if (candidates.length === 1) out.set(row.id, { parentId: candidates[0].id, auto: true });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Of the servers that are down, those whose cause is another server that is down too: each maps to the topmost ancestor that is down —
|
||||
* the root cause. A chain (guest → VM host → the box it hangs off) is followed all the way up; a loop in the links covers nothing.
|
||||
*/
|
||||
export function coveredByParent(offline: Set<number>, parents: Map<number, ParentLink>): Map<number, number> {
|
||||
const covered = new Map<number, number>();
|
||||
for (const id of offline) {
|
||||
const seen = new Set<number>([id]);
|
||||
let root: number | null = null;
|
||||
let looped = false;
|
||||
let link = parents.get(id);
|
||||
while (link && !seen.has(link.parentId)) {
|
||||
if (offline.has(link.parentId)) root = link.parentId;
|
||||
seen.add(link.parentId);
|
||||
link = parents.get(link.parentId);
|
||||
}
|
||||
if (link && seen.has(link.parentId)) looped = true;
|
||||
// A loop explains nothing (each would be waiting for the other), so its members stand alone and are announced.
|
||||
if (root !== null && !looped) covered.set(id, root);
|
||||
}
|
||||
return covered;
|
||||
}
|
||||
|
||||
/** Would making `parentId` the parent of `id` close a loop? True if `id` is `parentId` or already sits above it. */
|
||||
export function wouldLoop(rows: ParentRow[], id: number, parentId: number): boolean {
|
||||
const parents = resolveParents(rows);
|
||||
const seen = new Set<number>();
|
||||
let cur: number | undefined = parentId;
|
||||
while (cur !== undefined && !seen.has(cur)) {
|
||||
if (cur === id) return true;
|
||||
seen.add(cur);
|
||||
cur = parents.get(cur)?.parentId;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function loadParentRows(): Promise<ParentRow[]> {
|
||||
return db
|
||||
.select({ id: servers.id, name: servers.name, hostname: servers.hostname, parentServerId: servers.parentServerId, proxmoxNode: servers.proxmoxNode })
|
||||
.from(servers);
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import { and, eq, notInArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { scheduledTasks, servers } from "../db/schema.js";
|
||||
import { recordServerMetrics } from "./serverMetrics.js";
|
||||
import { nextUpdateState, type IncomingUpdates } from "./pendingUpdates.js";
|
||||
|
||||
export interface IncomingTask {
|
||||
scheduleType: "cron" | "systemd_timer" | "windows_task";
|
||||
@@ -20,6 +21,8 @@ export interface IncomingSystemInfo {
|
||||
memory?: { total_bytes?: number; used_bytes?: number };
|
||||
disks?: { mount: string; size_bytes: number; used_bytes: number }[];
|
||||
listening_ports?: { protocol: "tcp" | "udp"; port: number; address: string; process?: string }[];
|
||||
/** Updates waiting to be installed. Absent from an agent that doesn't report them (or couldn't find out this time). */
|
||||
updates?: IncomingUpdates;
|
||||
}
|
||||
|
||||
export interface AgentReport {
|
||||
@@ -98,10 +101,13 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
|
||||
}
|
||||
|
||||
const system = report.system;
|
||||
const [before] = await db.select({ pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince }).from(servers).where(eq(servers.id, serverId));
|
||||
await db
|
||||
.update(servers)
|
||||
.set({
|
||||
lastSeenAt: now,
|
||||
// Only stored when the agent reported it, so an agent that predates this (or couldn't tell this time) doesn't wipe what it said before.
|
||||
...(system?.updates && before ? nextUpdateState(before, system.updates, now) : {}),
|
||||
...(report.hostname ? { hostname: report.hostname } : {}),
|
||||
// An agent knows what it runs on better than whoever registered the server did.
|
||||
...(report.osType === "linux" || report.osType === "windows" ? { osType: report.osType } : {}),
|
||||
|
||||
Reference in new issue
Block a user