Patch plans, a monthly report and new-port alerts
Patch plans (Updates -> Patch plans): updates on a schedule, one server at a time, each by an ordinary update run, stopping at the first failure by default, silencing what depends on a server while it is updated, with one message at the end. Monthly report (Operations -> Monthly report, and optionally a message on a day of the month): availability, incidents, updates, restore tests, backups and renewals for a calendar month, printable. Backed by daily endpoint figures and a record of renewals. New ports: the first report with ports is the baseline; a TCP port that begins listening later and is not noted on the server's Ports card is a warning, cleared by Mark as expected. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ddd0aa622a
commit
be6c5552bd
46 files changed
+7836
-26
No files matched your search
@@ -9,10 +9,11 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, domains, endpointChecks, integrations, maintenanceSchedules, secrets, servers } from "../db/schema.js";
|
||||
import { assets, domains, endpointChecks, integrations, maintenanceSchedules, patchPlans, secrets, servers } from "../db/schema.js";
|
||||
import { collectTailscaleKeyExpiries } from "./tailscaleKeyExpiryScheduler.js";
|
||||
import { describeTarget, listUpcomingWindows } from "./maintenance.js";
|
||||
import { occurrencesBetween, targetsOf } from "./maintenanceSchedules.js";
|
||||
import { planDays, planOccurrences, planServers } from "./patchPlans.js";
|
||||
import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
import { latestTests } from "./restoreTests.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
@@ -88,6 +89,8 @@ export interface AgendaData {
|
||||
/** Windows that have been set up but not begun, one per target. */
|
||||
windows: { startedAt: string; endsAt: string; reason: string | null; source: string | null; targetName: string }[];
|
||||
schedules: { id: number; name: string; reason: string | null; days: string; startTime: string; durationMinutes: number; timezone: string; targetNames: string[] }[];
|
||||
/** Patch plans that are switched on, with how many servers each covers now. */
|
||||
patchPlans: { id: number; name: string; days: string; startTime: string; timezone: string; servers: number }[];
|
||||
/** The newest restore test of each subject. */
|
||||
restoreTests: { subjectKey: string; label: string; testedAt: string; result: string }[];
|
||||
restoreLimitDays: number;
|
||||
@@ -140,6 +143,10 @@ export async function loadAgendaData(now: Date = new Date()): Promise<AgendaData
|
||||
schedules.push({ id: s.id, name: s.name, reason: s.reason, days: s.days, startTime: s.startTime, durationMinutes: s.durationMinutes, timezone: s.timezone, targetNames: names });
|
||||
}
|
||||
|
||||
const planRows = (await db.select().from(patchPlans)).filter((p) => p.enabled);
|
||||
const allServers = planRows.length > 0 ? await db.select({ id: servers.id, name: servers.name, tags: servers.tags }).from(servers) : [];
|
||||
const plans: AgendaData["patchPlans"] = planRows.map((p) => ({ id: p.id, name: p.name, days: p.days, startTime: p.startTime, timezone: p.timezone, servers: planServers(p, allServers).length }));
|
||||
|
||||
const updates: AgendaData["updates"] = [];
|
||||
for (const s of serverRows) {
|
||||
const u = parsePendingUpdates(s.pendingUpdates);
|
||||
@@ -163,6 +170,7 @@ export async function loadAgendaData(now: Date = new Date()): Promise<AgendaData
|
||||
assets: assetRows,
|
||||
windows,
|
||||
schedules,
|
||||
patchPlans: plans,
|
||||
restoreTests: [...tests.values()].map((t) => ({ subjectKey: t.subjectKey, label: t.label, testedAt: t.testedAt, result: t.result })),
|
||||
restoreLimitDays: settings.healthChecks.restoreTestDays,
|
||||
updates,
|
||||
@@ -223,7 +231,7 @@ export function agendaFromData(data: AgendaData, now: Date, tz: string, days: nu
|
||||
g.names.push(w.targetName);
|
||||
groups.set(key, g);
|
||||
}
|
||||
const timed = (id: string, start: string, end: string, title: string, detail: string | null) =>
|
||||
const timed = (id: string, start: string, end: string | null, title: string, detail: string | null) =>
|
||||
items.push({ id, kind: "maintenance", at: start, endsAt: end, allDay: false, title, detail, link: "/maintenance", serverId: null, daysLeft: daysBetween(today, localDate(new Date(start), tz)) });
|
||||
const names = (list: string[]) => {
|
||||
const sorted = [...list].sort((a, b) => a.localeCompare(b));
|
||||
@@ -236,6 +244,13 @@ export function agendaFromData(data: AgendaData, now: Date, tz: string, days: nu
|
||||
}
|
||||
}
|
||||
|
||||
// A patch plan is maintenance too: when it will start and on how many servers.
|
||||
for (const p of data.patchPlans) {
|
||||
for (const occ of planOccurrences(p, now, new Date(lastMs)).filter((o) => o.start.getTime() > now.getTime()).slice(0, 60)) {
|
||||
timed(`plan:${p.id}:${occ.date}`, occ.start.toISOString(), null, `Patch plan: ${p.name}`, `updates ${plural(p.servers, "server")}, one at a time`);
|
||||
}
|
||||
}
|
||||
|
||||
// Updates that have been waiting: one line for all of them, from the longest wait.
|
||||
if (data.updates.length > 0) {
|
||||
const since = data.updates.map((u) => u.since).filter((s): s is string => !!s).map((s) => localDate(new Date(s), tz)).sort()[0] ?? today;
|
||||
|
||||
@@ -79,8 +79,8 @@ const SEVERITY_ORDER: Record<AlertSeverity, number> = { critical: 0, warning: 1,
|
||||
|
||||
/** Which kind of source, and which one, a health/automation condition key belongs to. */
|
||||
function originOfConditionKey(key: string): { type: string; id: number | null; category: AlertCategory } {
|
||||
let m = /^(?:offline|disk|failed-units|smart|zfs):server:(\d+)/.exec(key);
|
||||
if (m) return { type: "server", id: Number(m[1]), category: key.startsWith("offline") ? "offline" : key.startsWith("failed-units") ? "services" : "disk" };
|
||||
let m = /^(?:offline|disk|failed-units|smart|zfs|new-port):server:(\d+)/.exec(key);
|
||||
if (m) return { type: "server", id: Number(m[1]), category: key.startsWith("offline") ? "offline" : key.startsWith("failed-units") ? "services" : key.startsWith("new-port") ? "monitoring" : "disk" };
|
||||
m = /^disk:proxmox:(\d+):/.exec(key);
|
||||
if (m) return { type: "proxmox", id: Number(m[1]), category: "disk" };
|
||||
m = /^(?:synology-volume|synology-disk|disk:synology):(\d+):/.exec(key);
|
||||
|
||||
@@ -54,6 +54,8 @@ const KIND_OF_CATEGORY: Record<string, ChangeKind> = {
|
||||
restore_test: "backups",
|
||||
endpoint: "endpoints",
|
||||
heartbeat: "endpoints",
|
||||
patch_plan: "servers",
|
||||
report: "other",
|
||||
};
|
||||
|
||||
/** Sign-ins and a person's own data export are not changes to the homelab; the diagnostic log has its own page. */
|
||||
@@ -155,6 +157,22 @@ export function describeChange(category: string, action: string, detail: Record<
|
||||
return { title: `Endpoint check changed${name ? `: ${name}` : ""}`, tone: "neutral", detail: [str(detail.kind), str(detail.target)].filter(Boolean).join(" · ") || null };
|
||||
case "endpoint/delete":
|
||||
return { title: `Endpoint check removed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
|
||||
case "report/send_monthly":
|
||||
return { title: `Monthly report sent${str(detail.month) ? ` (${str(detail.month)})` : ""}`, tone: "neutral", detail: null };
|
||||
case "patch_plan/create":
|
||||
return { title: `Patch plan added${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
|
||||
case "patch_plan/update":
|
||||
return { title: `Patch plan changed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
|
||||
case "patch_plan/delete":
|
||||
return { title: `Patch plan removed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
|
||||
case "patch_plan/run":
|
||||
return { title: `Patch plan started${name ? `: ${name}` : ""}`, tone: "neutral", detail: str(detail.trigger) === "schedule" ? "by its schedule" : str(detail.by) ? `by ${str(detail.by)}` : null };
|
||||
case "patch_plan/finished":
|
||||
return { title: `Patch plan finished${name ? `: ${name}` : ""}`, tone: "good", detail: str(detail.message) };
|
||||
case "patch_plan/failed":
|
||||
return { title: `Patch plan didn't finish cleanly${name ? `: ${name}` : ""}`, tone: "bad", detail: str(detail.message) };
|
||||
case "patch_plan/cancelled":
|
||||
return { title: `Patch plan stopped${name ? `: ${name}` : ""}`, tone: "neutral", detail: str(detail.message) };
|
||||
case "heartbeat/create":
|
||||
return { title: `Heartbeat added${name ? `: ${name}` : ""}`, tone: "neutral", detail: typeof detail.periodMinutes === "number" ? `expected every ${detail.periodMinutes} min` : null };
|
||||
case "heartbeat/update":
|
||||
@@ -260,6 +278,7 @@ export async function getChangeLog(query: ChangeQuery = {}): Promise<{ entries:
|
||||
targetName = endpointName.get(idNum) ?? targetName;
|
||||
link = "/endpoints";
|
||||
} else if (a.targetType === "heartbeat") link = "/heartbeats";
|
||||
else if (a.targetType === "patch_plan") link = "/updates?view=plans";
|
||||
else if (a.targetType === "asset" && idNum !== null) link = `/inventory?asset=${idNum}`;
|
||||
else if (a.targetType === "restore_test") link = "/backups";
|
||||
else if (a.targetType === "dependency") link = "/map?view=dependencies";
|
||||
|
||||
@@ -5,6 +5,7 @@ import { defaultDeps, lookupRegistration, normalizeDomain, resolveRegistration,
|
||||
import { computeSecretStatus } from "./secretStatus.js";
|
||||
import { notifyDomainExpiry } from "./notify.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { noteRenewal } from "./renewals.js";
|
||||
|
||||
export type DomainRow = typeof domains.$inferSelect;
|
||||
|
||||
@@ -48,6 +49,8 @@ export async function checkDomain(id: number, deps: LookupDeps = defaultDeps): P
|
||||
if (!row) return null;
|
||||
const result = await lookupRegistration(row.name, deps);
|
||||
const [updated] = await db.update(domains).set(outcomeFields(result, row, new Date(deps.now()).toISOString())).where(eq(domains.id, id)).returning();
|
||||
// A registration that now runs much longer than it did was renewed.
|
||||
await noteRenewal("domain", row.name, row.expiresAt, updated.expiresAt, new Date(deps.now()));
|
||||
return updated;
|
||||
}
|
||||
|
||||
|
||||
@@ -12,9 +12,10 @@ import https from "node:https";
|
||||
import net from "node:net";
|
||||
import tls from "node:tls";
|
||||
import type { TLSSocket } from "node:tls";
|
||||
import { and, asc, eq, gte, inArray, lt } from "drizzle-orm";
|
||||
import { and, asc, eq, gte, inArray, lt, sql } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { endpointChecks, endpointResults } from "../db/schema.js";
|
||||
import { endpointChecks, endpointDaily, endpointResults } from "../db/schema.js";
|
||||
import { noteRenewal } from "./renewals.js";
|
||||
import { resolveTarget } from "../utils/networkGuard.js";
|
||||
|
||||
export const CHECK_KINDS = ["http", "tcp", "tls"] as const;
|
||||
@@ -244,9 +245,33 @@ export async function runCheck(c: CheckRow, now: Date = new Date(), probeFn: (c:
|
||||
})
|
||||
.where(eq(endpointChecks.id, c.id));
|
||||
await db.insert(endpointResults).values({ checkId: c.id, at: iso, ok: result.ok, ms: result.ms, message: result.message.slice(0, 300) });
|
||||
await addToDaily(c.id, iso, result);
|
||||
// A certificate that is now good for much longer than it was has been renewed.
|
||||
if (result.certExpiresAt) await noteRenewal("certificate", c.name, c.certExpiresAt, result.certExpiresAt, now);
|
||||
return result;
|
||||
}
|
||||
|
||||
/** How long the daily figures are kept: years, where the individual results are kept two weeks. */
|
||||
export const KEEP_DAILY_DAYS = 800;
|
||||
|
||||
/** Adds one result to its day: the monthly report's availability is worked from these, long after the results themselves are gone. */
|
||||
async function addToDaily(checkId: number, atIso: string, result: ProbeResult): Promise<void> {
|
||||
const timed = result.ok && result.ms !== null;
|
||||
const ok = result.ok ? 1 : 0;
|
||||
const ms = timed ? (result.ms as number) : 0;
|
||||
try {
|
||||
await db
|
||||
.insert(endpointDaily)
|
||||
.values({ checkId, day: atIso.slice(0, 10), total: 1, ok, msSum: ms, msCount: timed ? 1 : 0 })
|
||||
.onConflictDoUpdate({
|
||||
target: [endpointDaily.checkId, endpointDaily.day],
|
||||
set: { total: sql`${endpointDaily.total} + 1`, ok: sql`${endpointDaily.ok} + ${ok}`, msSum: sql`${endpointDaily.msSum} + ${ms}`, msCount: sql`${endpointDaily.msCount} + ${timed ? 1 : 0}` },
|
||||
});
|
||||
} catch (err) {
|
||||
console.error("[endpoints] couldn't add to the daily figures:", err);
|
||||
}
|
||||
}
|
||||
|
||||
/** Runs every enabled check that is due, a few at a time. Returns how many it asked. */
|
||||
export async function runDueChecks(now: Date = new Date(), probeFn?: (c: CheckRow) => Promise<ProbeResult>): Promise<number> {
|
||||
const rows = await db.select().from(endpointChecks).where(eq(endpointChecks.enabled, true)).orderBy(asc(endpointChecks.id));
|
||||
@@ -264,6 +289,7 @@ export async function runDueChecks(now: Date = new Date(), probeFn?: (c: CheckRo
|
||||
|
||||
export async function pruneResults(now: Date = new Date()): Promise<void> {
|
||||
await db.delete(endpointResults).where(lt(endpointResults.at, new Date(now.getTime() - KEEP_RESULT_DAYS * 86_400_000).toISOString()));
|
||||
await db.delete(endpointDaily).where(lt(endpointDaily.day, new Date(now.getTime() - KEEP_DAILY_DAYS * 86_400_000).toISOString().slice(0, 10)));
|
||||
}
|
||||
|
||||
// ─── What the history says ─────────────────────────────────────────────────
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, servers } from "../db/schema.js";
|
||||
import { integrations, serverPorts, servers } from "../db/schema.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
|
||||
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
|
||||
@@ -11,6 +11,8 @@ import { activeSubjects, subjectOfConditionKey } from "./maintenance.js";
|
||||
import { dependencyCover, HEALTH_STATE_FLAG } from "./dependencies.js";
|
||||
import { getFailingSources } from "./integrationHealthMonitor.js";
|
||||
import { hostHealthConditions, parseHostHealth, withoutIgnoredServices, type HostHealth } from "./hostHealth.js";
|
||||
import { newListeningPorts, newPortMessage, parseBaseline, portKey, type NewPort } from "./newPorts.js";
|
||||
import { parseJson, type AgentPort } from "./agentPorts.js";
|
||||
import { diskLimit, parseThresholds, type ServerThresholds } from "./serverThresholds.js";
|
||||
|
||||
const STATE_FLAG = HEALTH_STATE_FLAG;
|
||||
@@ -39,6 +41,8 @@ export interface ServerSnapshot {
|
||||
health?: HostHealth | null;
|
||||
/** This server's own thresholds, where it has any: they replace the global ones for it. */
|
||||
thresholds?: ServerThresholds | null;
|
||||
/** TCP ports that began listening after the baseline and that nobody has noted (services/newPorts.ts); empty when the check is off. */
|
||||
newPorts?: NewPort[];
|
||||
}
|
||||
export interface ProxmoxSnapshot {
|
||||
integrationId: number;
|
||||
@@ -123,6 +127,7 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
// A server that never reported has no agent to go quiet; and an offline one's disk figures are stale, so neither is judged on disks.
|
||||
if (s.lastSeenAt && !offline) {
|
||||
for (const c of hostHealthConditions(s.id, s.name, s.health ?? null)) out.push({ key: c.key, source: "server", message: c.message, severity: c.severity });
|
||||
for (const p of s.newPorts ?? []) out.push({ key: `new-port:server:${s.id}:${portKey(p.protocol, p.port)}`, source: "server", message: newPortMessage(s.name, p), severity: "warning" });
|
||||
for (const d of s.disks) {
|
||||
const p = usage(d.usedBytes, d.sizeBytes);
|
||||
// A disk can have its own limit, or be left alone altogether; otherwise the server's, then the global one.
|
||||
@@ -299,6 +304,17 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
const held = new Set<string>();
|
||||
const snapshot: HealthSnapshot = { servers: [], proxmox: [], synology: [] };
|
||||
|
||||
const { healthChecks: checks } = await getSettings();
|
||||
// The ports somebody has said are expected (a label or a comment on the Ports page); a row that only records an open scan says nothing.
|
||||
const expectedByServer = new Map<number, Set<string>>();
|
||||
if (checks.newPortAlerts) {
|
||||
for (const r of await db.select().from(serverPorts)) {
|
||||
if (!r.label && !r.comment) continue;
|
||||
const set = expectedByServer.get(r.serverId) ?? new Set<string>();
|
||||
set.add(portKey(r.protocol, r.port));
|
||||
expectedByServer.set(r.serverId, set);
|
||||
}
|
||||
}
|
||||
for (const s of await db.select().from(servers)) {
|
||||
let disks: ServerSnapshot["disks"] = [];
|
||||
try {
|
||||
@@ -306,7 +322,8 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
} catch {
|
||||
// a malformed report shouldn't stop every other server being checked
|
||||
}
|
||||
snapshot.servers.push({ id: s.id, name: s.name, lastSeenAt: s.lastSeenAt, disks, health: withoutIgnoredServices(parseHostHealth(s.hostHealth), parseThresholds(s.thresholds)?.ignoreServices), thresholds: parseThresholds(s.thresholds) });
|
||||
const newPorts = checks.newPortAlerts ? newListeningPorts(parseJson<AgentPort[] | null>(s.listeningPorts, null), parseBaseline(s.portBaseline), expectedByServer.get(s.id) ?? new Set()) : [];
|
||||
snapshot.servers.push({ id: s.id, name: s.name, lastSeenAt: s.lastSeenAt, disks, health: withoutIgnoredServices(parseHostHealth(s.hostHealth), parseThresholds(s.thresholds)?.ignoreServices), thresholds: parseThresholds(s.thresholds), newPorts });
|
||||
}
|
||||
|
||||
const rows = await db
|
||||
|
||||
@@ -23,7 +23,7 @@ export async function listUpcomingWindows(now: Date = new Date()): Promise<Activ
|
||||
* so conditions persisted by an earlier version still map correctly.
|
||||
*/
|
||||
export function subjectOfConditionKey(key: string): string | null {
|
||||
let m = /^(?:offline|disk|failed-units|smart|zfs):server:(\d+)(?::|$)/.exec(key);
|
||||
let m = /^(?:offline|disk|failed-units|smart|zfs|new-port):server:(\d+)(?::|$)/.exec(key);
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:(\d+)$/.exec(key);
|
||||
if (m) return `server:${m[1]}`;
|
||||
|
||||
@@ -0,0 +1,303 @@
|
||||
/**
|
||||
* The monthly report: how a month went — availability of servers, integrations and endpoints, incidents and how long they took, updates applied (by hand and by
|
||||
* patch plans), restore tests and the state of the backups, and what was renewed. Everything in it the app already knows; this gathers it for one calendar month
|
||||
* (on the clock of the notification time zone) into one structure that the page shows, prints and exports, and into a plain-text version that can be sent like the
|
||||
* weekly digest.
|
||||
*
|
||||
* What it can't say, it says: servers' and integrations' availability only goes back as far as the audit log records it; endpoint availability from the daily
|
||||
* figures (kept since they were added, and started from the last two weeks of results); renewals only from when the app began noticing them; the backups are the
|
||||
* state now, not a history. A month with nothing to report for a part leaves that part out.
|
||||
*/
|
||||
import { and, gte, lt } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { endpointChecks, endpointDaily, integrations, patchPlanRuns, restoreTests, servers, updateRuns } from "../db/schema.js";
|
||||
import { getCachedBackupCoverage } from "./backupCoverage.js";
|
||||
import { listIncidents } from "./incidents.js";
|
||||
import { zonedTime } from "./maintenanceSchedules.js";
|
||||
import { renewalsBetween } from "./renewals.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { getUptime } from "./uptimeHistory.js";
|
||||
import { spanWords } from "./weeklyDigest.js";
|
||||
|
||||
const DAY_MS = 86_400_000;
|
||||
const MAX_LISTED = 12;
|
||||
|
||||
export interface AvailabilityRow {
|
||||
id: number;
|
||||
name: string;
|
||||
/** Share of the known time spent up, in percent; null when nothing is known for the month. */
|
||||
percent: number | null;
|
||||
downMinutes: number;
|
||||
}
|
||||
|
||||
export interface EndpointRow {
|
||||
id: number;
|
||||
name: string;
|
||||
percent: number | null;
|
||||
avgMs: number | null;
|
||||
checks: number;
|
||||
}
|
||||
|
||||
export interface MonthlyReport {
|
||||
/** "2026-09". */
|
||||
month: string;
|
||||
/** "September 2026". */
|
||||
label: string;
|
||||
/** ISO — the start of the month and its end (or now, for a month that is not over). */
|
||||
from: string;
|
||||
to: string;
|
||||
/** The month is not over yet: everything is "so far". */
|
||||
partial: boolean;
|
||||
timezone: string;
|
||||
servers: AvailabilityRow[];
|
||||
integrations: AvailabilityRow[];
|
||||
endpoints: EndpointRow[];
|
||||
incidents: {
|
||||
began: number;
|
||||
resolved: number;
|
||||
stillOpen: number;
|
||||
/** Of those that began and ended in the month. */
|
||||
totalMinutes: number;
|
||||
averageMinutes: number | null;
|
||||
longest: { id: number; message: string; minutes: number } | null;
|
||||
byCategory: { category: string; count: number }[];
|
||||
/** The longest, up to a dozen. */
|
||||
list: { id: number; message: string; where: string; severity: string; openedAt: string; resolvedAt: string | null; minutes: number }[];
|
||||
};
|
||||
updates: {
|
||||
runs: number;
|
||||
done: number;
|
||||
failed: number;
|
||||
/** Updates the runs say they installed (before minus after, where both are known). */
|
||||
installed: number;
|
||||
byServer: { name: string; runs: number; failed: number; installed: number }[];
|
||||
planRuns: { runs: number; done: number; failed: number };
|
||||
};
|
||||
restoreTests: { label: string; testedAt: string; result: string; note: string | null }[];
|
||||
/** The backup state as of now; null when no coverage has been read yet. */
|
||||
backups: { ok: number; stale: number; failed: number; none: number; unknown: number; total: number; asOf: string } | null;
|
||||
renewals: { kind: string; subject: string; at: string; fromDate: string; toDate: string }[];
|
||||
}
|
||||
|
||||
// ─── Months ────────────────────────────────────────────────────────────────
|
||||
|
||||
const pad = (n: number) => String(n).padStart(2, "0");
|
||||
|
||||
export function isMonthId(v: unknown): v is string {
|
||||
return typeof v === "string" && /^\d{4}-(0[1-9]|1[0-2])$/.test(v);
|
||||
}
|
||||
|
||||
/** The year and month `now` is in, on the clock of `tz`. */
|
||||
function monthOf(now: Date, tz: string): { year: number; month: number } {
|
||||
const parts = new Intl.DateTimeFormat("en-CA", { timeZone: tz, year: "numeric", month: "2-digit" }).formatToParts(now);
|
||||
return { year: Number(parts.find((p) => p.type === "year")?.value), month: Number(parts.find((p) => p.type === "month")?.value) };
|
||||
}
|
||||
|
||||
export const monthId = (year: number, month: number) => `${year}-${pad(month)}`;
|
||||
const shift = (year: number, month: number, by: number) => {
|
||||
const index = year * 12 + (month - 1) + by;
|
||||
return { year: Math.floor(index / 12), month: (index % 12) + 1 };
|
||||
};
|
||||
|
||||
export function monthLabel(id: string): string {
|
||||
const [y, m] = id.split("-").map(Number);
|
||||
return new Date(Date.UTC(y, m - 1, 15)).toLocaleDateString("en-GB", { month: "long", year: "numeric", timeZone: "UTC" });
|
||||
}
|
||||
|
||||
/** The month before the one `now` is in: what a report sent at the start of a month is about. */
|
||||
export function lastFullMonth(now: Date, tz: string): string {
|
||||
const cur = monthOf(now, tz);
|
||||
const prev = shift(cur.year, cur.month, -1);
|
||||
return monthId(prev.year, prev.month);
|
||||
}
|
||||
|
||||
/** This month and the eleven before it, newest first. */
|
||||
export function recentMonths(now: Date, tz: string, count = 12): { id: string; label: string }[] {
|
||||
const cur = monthOf(now, tz);
|
||||
return Array.from({ length: count }, (_, i) => {
|
||||
const m = shift(cur.year, cur.month, -i);
|
||||
const id = monthId(m.year, m.month);
|
||||
return { id, label: monthLabel(id) };
|
||||
});
|
||||
}
|
||||
|
||||
/** The instants a month begins and ends at, on the clock of `tz`. */
|
||||
export function monthBounds(id: string, tz: string): { start: Date; end: Date } {
|
||||
const [y, m] = id.split("-").map(Number);
|
||||
const next = shift(y, m, 1);
|
||||
return { start: zonedTime(y, m, 1, 0, 0, tz), end: zonedTime(next.year, next.month, 1, 0, 0, tz) };
|
||||
}
|
||||
|
||||
// ─── Gathering ─────────────────────────────────────────────────────────────
|
||||
|
||||
const round1 = (n: number) => Math.round(n * 10) / 10;
|
||||
|
||||
export async function buildMonthlyReport(id: string, now: Date = new Date()): Promise<MonthlyReport> {
|
||||
const settings = await getSettings();
|
||||
const tz = settings.notifications.timezone || "UTC";
|
||||
const { start, end: fullEnd } = monthBounds(id, tz);
|
||||
const partial = fullEnd.getTime() > now.getTime();
|
||||
const end = partial ? now : fullEnd;
|
||||
const days = Math.max(1, Math.ceil((end.getTime() - start.getTime()) / DAY_MS));
|
||||
const startIso = start.toISOString();
|
||||
const endIso = end.toISOString();
|
||||
|
||||
// Availability. Servers and integrations from the audit log's record of going away and coming back; laid over the month's days ending where the month does.
|
||||
const uptime = await getUptime(end, days);
|
||||
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((s) => [s.id, s.name]));
|
||||
const integrationNames = new Map((await db.select({ id: integrations.id, name: integrations.name }).from(integrations)).map((i) => [i.id, i.name]));
|
||||
const rowsOf = (series: Record<number, { buckets: { k: number; d: number }[]; percent: number }>, names: Map<number, string>): AvailabilityRow[] =>
|
||||
Object.entries(series)
|
||||
.map(([key, s]) => ({ id: Number(key), name: names.get(Number(key)) ?? `#${key}`, percent: s.percent, downMinutes: s.buckets.reduce((n, b) => n + b.d, 0) }))
|
||||
.filter((r) => names.has(r.id))
|
||||
.sort((a, b) => (a.percent ?? 101) - (b.percent ?? 101) || a.name.localeCompare(b.name));
|
||||
|
||||
// Endpoints: from the daily figures, for the days the month covers. The figures are per UTC day, so a day at either end can belong to a neighbouring month
|
||||
// by a few hours (more than the time zone is ahead of UTC); a month that is not over takes in today.
|
||||
const [yy, mm] = id.split("-").map(Number);
|
||||
const following = shift(yy, mm, 1);
|
||||
const firstDay = `${id}-01`;
|
||||
const lastDayExclusive = partial ? new Date(now.getTime() + DAY_MS).toISOString().slice(0, 10) : `${monthId(following.year, following.month)}-01`;
|
||||
const dailyRows = await db.select().from(endpointDaily).where(and(gte(endpointDaily.day, firstDay), lt(endpointDaily.day, lastDayExclusive)));
|
||||
const checkNames = new Map((await db.select({ id: endpointChecks.id, name: endpointChecks.name }).from(endpointChecks)).map((c) => [c.id, c.name]));
|
||||
const perCheck = new Map<number, { total: number; ok: number; msSum: number; msCount: number }>();
|
||||
for (const r of dailyRows) {
|
||||
const t = perCheck.get(r.checkId) ?? { total: 0, ok: 0, msSum: 0, msCount: 0 };
|
||||
t.total += r.total;
|
||||
t.ok += r.ok;
|
||||
t.msSum += r.msSum;
|
||||
t.msCount += r.msCount;
|
||||
perCheck.set(r.checkId, t);
|
||||
}
|
||||
const endpointRows: EndpointRow[] = [...perCheck]
|
||||
.filter(([cid, t]) => checkNames.has(cid) && t.total > 0)
|
||||
.map(([cid, t]) => ({ id: cid, name: checkNames.get(cid)!, percent: round1((t.ok / t.total) * 100), avgMs: t.msCount > 0 ? Math.round(t.msSum / t.msCount) : null, checks: t.total }))
|
||||
.sort((a, b) => (a.percent ?? 101) - (b.percent ?? 101) || a.name.localeCompare(b.name));
|
||||
|
||||
// Incidents.
|
||||
const all = await listIncidents({ now: end, limit: 3000 });
|
||||
const inMonth = (iso: string) => iso >= startIso && iso < endIso;
|
||||
const began = all.filter((i) => inMonth(i.openedAt));
|
||||
const resolved = all.filter((i) => i.resolvedAt !== null && inMonth(i.resolvedAt));
|
||||
const stillOpen = all.filter((i) => i.openedAt < endIso && (i.resolvedAt === null || i.resolvedAt >= endIso)).length;
|
||||
const closedHere = began.filter((i) => i.resolvedAt !== null && inMonth(i.resolvedAt));
|
||||
const minutesOf = (ms: number) => Math.max(1, Math.round(ms / 60_000));
|
||||
const totalMinutes = closedHere.reduce((n, i) => n + minutesOf(i.durationMs), 0);
|
||||
const byDuration = [...began].sort((a, b) => b.durationMs - a.durationMs);
|
||||
const categories = new Map<string, number>();
|
||||
for (const i of began) categories.set(i.category, (categories.get(i.category) ?? 0) + 1);
|
||||
|
||||
// Updates, by hand and by plan.
|
||||
const runRows = await db.select().from(updateRuns).where(and(gte(updateRuns.startedAt, startIso), lt(updateRuns.startedAt, endIso)));
|
||||
const perServer = new Map<number, { runs: number; failed: number; installed: number }>();
|
||||
let installed = 0;
|
||||
for (const r of runRows) {
|
||||
const gained = r.status === "done" && r.updatesBefore !== null && r.updatesAfter !== null ? Math.max(0, r.updatesBefore - r.updatesAfter) : 0;
|
||||
installed += gained;
|
||||
const t = perServer.get(r.serverId) ?? { runs: 0, failed: 0, installed: 0 };
|
||||
t.runs++;
|
||||
if (r.status === "failed") t.failed++;
|
||||
t.installed += gained;
|
||||
perServer.set(r.serverId, t);
|
||||
}
|
||||
const planRows = await db.select().from(patchPlanRuns).where(and(gte(patchPlanRuns.startedAt, startIso), lt(patchPlanRuns.startedAt, endIso)));
|
||||
|
||||
// Restore tests, and the backups as they are now.
|
||||
const tests = await db.select().from(restoreTests).where(and(gte(restoreTests.testedAt, firstDay), lt(restoreTests.testedAt, lastDayExclusive)));
|
||||
const coverage = await getCachedBackupCoverage(now);
|
||||
|
||||
const renewalRows = await renewalsBetween(start, end);
|
||||
|
||||
return {
|
||||
month: id,
|
||||
label: monthLabel(id),
|
||||
from: startIso,
|
||||
to: endIso,
|
||||
partial,
|
||||
timezone: tz,
|
||||
servers: rowsOf(uptime.servers, serverNames),
|
||||
integrations: rowsOf(uptime.integrations, integrationNames),
|
||||
endpoints: endpointRows,
|
||||
incidents: {
|
||||
began: began.length,
|
||||
resolved: resolved.length,
|
||||
stillOpen,
|
||||
totalMinutes,
|
||||
averageMinutes: closedHere.length > 0 ? Math.round(totalMinutes / closedHere.length) : null,
|
||||
longest: byDuration[0] ? { id: byDuration[0].id, message: byDuration[0].message, minutes: minutesOf(byDuration[0].durationMs) } : null,
|
||||
byCategory: [...categories].map(([category, count]) => ({ category, count })).sort((a, b) => b.count - a.count || a.category.localeCompare(b.category)),
|
||||
list: byDuration.slice(0, MAX_LISTED).map((i) => ({ id: i.id, message: i.message, where: i.target?.name ?? i.source, severity: i.peakSeverity, openedAt: i.openedAt, resolvedAt: i.resolvedAt, minutes: minutesOf(i.durationMs) })),
|
||||
},
|
||||
updates: {
|
||||
runs: runRows.length,
|
||||
done: runRows.filter((r) => r.status === "done").length,
|
||||
failed: runRows.filter((r) => r.status === "failed").length,
|
||||
installed,
|
||||
byServer: [...perServer]
|
||||
.map(([sid, t]) => ({ name: serverNames.get(sid) ?? `#${sid}`, ...t }))
|
||||
.sort((a, b) => b.runs - a.runs || a.name.localeCompare(b.name)),
|
||||
planRuns: { runs: planRows.length, done: planRows.filter((r) => r.status === "done").length, failed: planRows.filter((r) => r.status === "failed").length },
|
||||
},
|
||||
restoreTests: tests.map((t) => ({ label: t.label, testedAt: t.testedAt, result: t.result, note: t.note })).sort((a, b) => a.testedAt.localeCompare(b.testedAt)),
|
||||
backups: coverage ? { ok: coverage.counts.ok, stale: coverage.counts.stale, failed: coverage.counts.failed, none: coverage.counts.none, unknown: coverage.counts.unknown, total: coverage.counts.total, asOf: coverage.generatedAt } : null,
|
||||
renewals: renewalRows.map((r) => ({ kind: r.kind, subject: r.subject, at: r.at, fromDate: r.fromDate, toDate: r.toDate })),
|
||||
};
|
||||
}
|
||||
|
||||
// ─── As a message ──────────────────────────────────────────────────────────
|
||||
|
||||
const plural = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
|
||||
const RESULT_WORDS: Record<string, string> = { ok: "passed", partial: "partly worked", failed: "failed", not_needed: "marked as not needed" };
|
||||
|
||||
function average(rows: AvailabilityRow[]): number | null {
|
||||
const known = rows.filter((r) => r.percent !== null);
|
||||
return known.length === 0 ? null : round1(known.reduce((n, r) => n + (r.percent as number), 0) / known.length);
|
||||
}
|
||||
|
||||
/** The report as plain text, for a notification or an email: short, a topic only where there is something to say. */
|
||||
export function monthlyReportText(r: MonthlyReport): { title: string; message: string } {
|
||||
const sections: string[] = [];
|
||||
const lines = (heading: string, body: string[]) => body.length > 0 && sections.push(`${heading.toUpperCase()}\n${body.join("\n")}`);
|
||||
|
||||
const avail: string[] = [];
|
||||
const serverAvg = average(r.servers);
|
||||
if (serverAvg !== null) {
|
||||
avail.push(`Servers: ${serverAvg}% on average (${plural(r.servers.length, "server")}).`);
|
||||
for (const s of r.servers.filter((x) => x.downMinutes > 0).slice(0, 6)) avail.push(`- ${s.name}: ${s.percent}% (${spanWords(s.downMinutes)} down)`);
|
||||
}
|
||||
if (r.endpoints.length > 0) {
|
||||
const avg = round1(r.endpoints.reduce((n, e) => n + (e.percent ?? 0), 0) / r.endpoints.length);
|
||||
avail.push(`Endpoints: ${avg}% on average (${plural(r.endpoints.length, "check")}).`);
|
||||
for (const e of r.endpoints.filter((x) => (x.percent ?? 100) < 100).slice(0, 6)) avail.push(`- ${e.name}: ${e.percent}%${e.avgMs !== null ? `, ${e.avgMs} ms on average` : ""}`);
|
||||
}
|
||||
const integrationAvg = average(r.integrations);
|
||||
if (integrationAvg !== null) avail.push(`Integrations: ${integrationAvg}% on average.`);
|
||||
lines("Availability", avail);
|
||||
|
||||
const inc = r.incidents;
|
||||
if (inc.began > 0 || inc.resolved > 0 || inc.stillOpen > 0) {
|
||||
const body = [`${plural(inc.began, "incident")} began, ${inc.resolved} ended, ${inc.stillOpen} still open.`];
|
||||
if (inc.averageMinutes !== null) body.push(`A resolved incident took ${spanWords(inc.averageMinutes)} on average.`);
|
||||
for (const i of inc.list.slice(0, 3)) body.push(`- ${i.message} — ${spanWords(i.minutes)}${i.resolvedAt ? "" : " and still open"}`);
|
||||
lines("Incidents", body);
|
||||
} else if (serverAvg !== null || r.endpoints.length > 0) {
|
||||
lines("Incidents", ["None."]);
|
||||
}
|
||||
|
||||
const u = r.updates;
|
||||
const upd: string[] = [];
|
||||
if (u.runs > 0) upd.push(`${plural(u.runs, "update run")}: ${u.done} done${u.failed > 0 ? `, ${u.failed} failed` : ""}${u.installed > 0 ? ` — ${plural(u.installed, "update")} installed` : ""}.`);
|
||||
if (u.planRuns.runs > 0) upd.push(`Patch plans ran ${plural(u.planRuns.runs, "time")}${u.planRuns.failed > 0 ? `, ${u.planRuns.failed} not cleanly` : ", all cleanly"}.`);
|
||||
lines("Updates", upd);
|
||||
|
||||
const bk: string[] = [];
|
||||
for (const t of r.restoreTests) bk.push(`- Restore test of ${t.label} (${t.testedAt}) ${RESULT_WORDS[t.result] ?? t.result}`);
|
||||
if (r.restoreTests.length === 0) bk.push("No restore test was recorded.");
|
||||
if (r.backups) bk.push(`Backups now: ${r.backups.ok} of ${plural(r.backups.total, "guest")} recent${r.backups.failed + r.backups.stale + r.backups.none > 0 ? `; ${r.backups.failed} failed, ${r.backups.stale} stale, ${r.backups.none} without a backup` : ""}.`);
|
||||
lines("Backups", bk);
|
||||
|
||||
lines("Renewed", r.renewals.slice(0, 10).map((x) => `- ${x.kind === "domain" ? "Domain" : "Certificate"} ${x.subject}: ${x.fromDate} → ${x.toDate}`));
|
||||
|
||||
const title = `Homelab monthly report — ${r.label}${r.partial ? " (so far)" : ""}`;
|
||||
return { title, message: sections.length > 0 ? sections.join("\n\n") : "Nothing was recorded for this month." };
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import schedule from "node-schedule";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { notify } from "./notify.js";
|
||||
import { buildMonthlyReport, lastFullMonth, monthlyReportText } from "./monthlyReport.js";
|
||||
|
||||
let currentJob: schedule.Job | null = null;
|
||||
|
||||
/** The cron rule for "this day of the month at HH:MM" (1–28, so every month has it). */
|
||||
export function cronForMonthlyReport(day: number, time: string): string {
|
||||
const [h, m] = time.split(":").map(Number);
|
||||
const d = Number.isInteger(day) && day >= 1 && day <= 28 ? day : 1;
|
||||
return `${Number.isFinite(m) ? m : 0} ${Number.isFinite(h) ? h : 8} ${d} * *`;
|
||||
}
|
||||
|
||||
/** Builds last month's report and sends it through the channels (held with other notifications during quiet hours). */
|
||||
export async function sendMonthlyReport(now: Date = new Date()): Promise<{ title: string; message: string }> {
|
||||
const tz = (await getSettings()).notifications.timezone || "UTC";
|
||||
const text = monthlyReportText(await buildMonthlyReport(lastFullMonth(now, tz), now));
|
||||
await notify(text.title, text.message);
|
||||
return text;
|
||||
}
|
||||
|
||||
/** (Re)schedules the monthly report per the current settings. Call again after they change. */
|
||||
export async function scheduleMonthlyReport(): Promise<void> {
|
||||
if (currentJob) {
|
||||
currentJob.cancel();
|
||||
currentJob = null;
|
||||
}
|
||||
const { monthlyReport, notifications } = await getSettings();
|
||||
if (!monthlyReport.enabled) {
|
||||
console.log("[report] monthly report is off, nothing scheduled");
|
||||
return;
|
||||
}
|
||||
const rule = cronForMonthlyReport(monthlyReport.day, monthlyReport.time);
|
||||
currentJob = schedule.scheduleJob({ rule, tz: notifications.timezone }, () => {
|
||||
sendMonthlyReport().catch((err) => console.error("[report] couldn't send the monthly report:", err));
|
||||
});
|
||||
console.log(`Monthly report scheduled: ${rule} (${notifications.timezone})`);
|
||||
}
|
||||
|
||||
export async function initMonthlyReportScheduler(): Promise<void> {
|
||||
await scheduleMonthlyReport();
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
/**
|
||||
* New ports: a server that starts listening on something it did not before is worth a look — a service somebody started, or something that should not be
|
||||
* there. The first report that carries ports is taken as the baseline: what the server was listening on, taken as normal. After that a TCP port that is
|
||||
* listening on an address other than this machine's own, is not in the baseline, and has not been noted on the Ports page, is a warning (the Alerts page,
|
||||
* and a notification) until the port is noted ("this is expected") or stops listening.
|
||||
*
|
||||
* Left out on purpose: UDP (clients and discovery open them all the time), ports in the dynamic range (49152 and up, where programs pick a free one), and
|
||||
* anything bound only to the loopback address, which nothing else on the network can reach.
|
||||
*/
|
||||
import { isLoopback, type AgentPort } from "./agentPorts.js";
|
||||
|
||||
/** From here up, a port is usually one a program picked for itself, not a service. */
|
||||
export const DYNAMIC_PORTS_FROM = 49152;
|
||||
|
||||
export interface NewPort {
|
||||
protocol: "tcp";
|
||||
port: number;
|
||||
/** What is listening, as the agent says; null when it does not. */
|
||||
process: string | null;
|
||||
}
|
||||
|
||||
export const portKey = (protocol: string, port: number) => `${protocol}:${port}`;
|
||||
|
||||
/** What is stored, read back as a set of "tcp:443"; null when there is no baseline yet (or it can't be read). */
|
||||
export function parseBaseline(stored: string | null): Set<string> | null {
|
||||
if (!stored) return null;
|
||||
try {
|
||||
const v = JSON.parse(stored);
|
||||
return Array.isArray(v) ? new Set(v.filter((k): k is string => typeof k === "string")) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** The baseline a first report makes: everything it listens on, in any protocol, so nothing about it is ever "new". */
|
||||
export function baselineFrom(listening: Pick<AgentPort, "protocol" | "port">[]): string {
|
||||
return JSON.stringify([...new Set(listening.map((p) => portKey(p.protocol, p.port)))].sort());
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure: the ports that are new — listening now, reachable from outside this machine, in neither the baseline nor the noted ports. `expected` is the set of
|
||||
* "tcp:8080" somebody gave a label or a comment on the Ports page. With no baseline yet, nothing is new.
|
||||
*/
|
||||
export function newListeningPorts(listening: AgentPort[] | null | undefined, baseline: Set<string> | null, expected: Set<string>): NewPort[] {
|
||||
if (!listening || baseline === null) return [];
|
||||
const found = new Map<number, NewPort>();
|
||||
for (const p of listening) {
|
||||
if (p.protocol !== "tcp" || !Number.isInteger(p.port) || p.port <= 0 || p.port >= DYNAMIC_PORTS_FROM) continue;
|
||||
if (isLoopback(p.address ?? "")) continue;
|
||||
const key = portKey("tcp", p.port);
|
||||
if (baseline.has(key) || expected.has(key)) continue;
|
||||
const seen = found.get(p.port);
|
||||
if (seen) seen.process = seen.process ?? (p.process || null);
|
||||
else found.set(p.port, { protocol: "tcp", port: p.port, process: p.process || null });
|
||||
}
|
||||
return [...found.values()].sort((a, b) => a.port - b.port);
|
||||
}
|
||||
|
||||
export const newPortMessage = (serverName: string, p: NewPort) =>
|
||||
`${serverName}: a new port is listening — ${p.port}/tcp${p.process ? ` (${p.process})` : ""}. If it should be there, note it on the server's Ports card.`;
|
||||
@@ -449,6 +449,16 @@ export async function notifyUpdateRunFinished(run: { serverName: string; templat
|
||||
);
|
||||
}
|
||||
|
||||
/** How a patch plan's run ended: one message, with a line for each server. Held back or sent with the other update-run results. */
|
||||
export async function notifyPatchPlanFinished(run: { planName: string; ok: boolean; message: string; lines: string[] }): Promise<void> {
|
||||
if (!(await eventEnabled("updateRunResults"))) return;
|
||||
await notify(
|
||||
run.ok ? `Homelab Manager — Patch plan "${run.planName}" finished` : `Homelab Manager — Patch plan "${run.planName}" didn't finish cleanly`,
|
||||
`${run.message}${run.lines.length > 0 ? `\n\n${run.lines.join("\n")}` : ""}`,
|
||||
{ category: "updates", severity: run.ok ? "warning" : "critical" },
|
||||
);
|
||||
}
|
||||
|
||||
export async function notifyPbsVerificationFailed(
|
||||
failures: { integrationName: string; datastore: string; failedCount: number; error: string | null }[],
|
||||
): Promise<void> {
|
||||
|
||||
@@ -107,11 +107,12 @@ export function metaOfConditionKey(key: string, severity?: string): NotifyMeta {
|
||||
let category: AlertCategory | null = null;
|
||||
if (key.startsWith("offline:")) category = "offline";
|
||||
else if (key.startsWith("failed-units:")) category = "services";
|
||||
else if (key.startsWith("new-port:")) category = "monitoring";
|
||||
else if (/^(disk|smart|zfs|synology-volume|synology-disk):/.test(key)) category = "disk";
|
||||
else if (key.startsWith("automation:")) category = "automation";
|
||||
else if (key.startsWith("heartbeat")) category = "automation";
|
||||
else if (key.startsWith("endpoint-cert:")) category = "expiry";
|
||||
else if (key.startsWith("endpoint:")) category = "monitoring";
|
||||
const server = /^(?:(?:offline|disk|failed-units|smart|zfs):server:|(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:)(\d+)/.exec(key);
|
||||
const server = /^(?:(?:offline|disk|failed-units|smart|zfs|new-port):server:|(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:)(\d+)/.exec(key);
|
||||
return { category, severity: severity === "critical" ? "critical" : "warning", serverId: server ? Number(server[1]) : null };
|
||||
}
|
||||
@@ -0,0 +1,321 @@
|
||||
/**
|
||||
* Patch plans: "Sundays at 02:00, the servers tagged lab, one at a time, stop at the first failure."
|
||||
*
|
||||
* A plan only says when, to whom and how. When it is due (or somebody presses Run now) it makes a *run*: a list of the servers it matches, and goes down
|
||||
* the list one server at a time. Each server is updated by an ordinary update run (services/updateRuns.ts) — the server is put under maintenance, the
|
||||
* Semaphore playbook is run limited to it, the app waits for it to report again — and the plan waits for that to finish before it starts the next. What
|
||||
* depends on the server is silenced for the same time (it will look down while the server restarts). If a server's update fails and the plan says to stop,
|
||||
* the servers after it are left alone and the run says why; otherwise the plan carries on. A message at the end says how each server went.
|
||||
*
|
||||
* A plan starts once a day at most: the day it last started on is kept, so a restart of this app at 02:10 neither misses it nor repeats it. If the app was
|
||||
* down at the time, it still starts, up to half an hour late; later than that, that day is skipped.
|
||||
*/
|
||||
import { and, desc, eq, inArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { maintenanceWindows, patchPlanRuns, patchPlans, servers, updateRuns } from "../db/schema.js";
|
||||
import { invalidateAlertsCache } from "./alerts.js";
|
||||
import { recordAudit } from "./audit.js";
|
||||
import { impactOf } from "./dependencies.js";
|
||||
import { startOrExtendWindow } from "./maintenance.js";
|
||||
import { occurrencesBetween, type Occurrence } from "./maintenanceSchedules.js";
|
||||
import { notifyPatchPlanFinished } from "./notify.js";
|
||||
import { parseTags } from "./serverTags.js";
|
||||
import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
import { ACTIVE_STATUSES, cancelUpdateRun, startUpdateRun, UpdateRunError } from "./updateRuns.js";
|
||||
|
||||
export type PlanRow = typeof patchPlans.$inferSelect;
|
||||
export type PlanRunRow = typeof patchPlanRuns.$inferSelect;
|
||||
export type ItemStatus = "pending" | "running" | "done" | "failed" | "skipped";
|
||||
export type RunStatus = "running" | "done" | "failed" | "cancelled";
|
||||
|
||||
export interface PlanItem {
|
||||
serverId: number;
|
||||
name: string;
|
||||
status: ItemStatus;
|
||||
updateRunId: number | null;
|
||||
message: string | null;
|
||||
/** Windows made for what depends on this server, which end with it. */
|
||||
windowIds: number[];
|
||||
}
|
||||
|
||||
/** How late a plan may still start, if the app was not running at its time. */
|
||||
export const START_GRACE_MS = 30 * 60_000;
|
||||
export const POLL_INTERVAL_MS = 20_000;
|
||||
export const MAX_PLANS = 30;
|
||||
|
||||
export class PlanError extends Error {
|
||||
constructor(
|
||||
readonly status: number,
|
||||
readonly code: string,
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Reading what is stored ────────────────────────────────────────────────
|
||||
|
||||
function jsonArray<T>(stored: string | null | undefined): T[] {
|
||||
if (!stored) return [];
|
||||
try {
|
||||
const v = JSON.parse(stored);
|
||||
return Array.isArray(v) ? (v as T[]) : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export const parseItems = (stored: string | null | undefined): PlanItem[] => jsonArray<PlanItem>(stored).filter((i) => i && typeof i.serverId === "number");
|
||||
export const planDays = (plan: Pick<PlanRow, "days">): number[] => jsonArray<number>(plan.days).filter((d) => Number.isInteger(d));
|
||||
export const planTags = (plan: Pick<PlanRow, "tags">): string[] => parseTags(plan.tags);
|
||||
export const planServerIds = (plan: Pick<PlanRow, "serverIds">): number[] => jsonArray<number>(plan.serverIds).filter((i) => Number.isInteger(i));
|
||||
|
||||
// ─── Who a plan is for, and when ───────────────────────────────────────────
|
||||
|
||||
/** Pure: the servers a plan covers — those with any of its tags, and those named — by name. */
|
||||
export function planServers(plan: Pick<PlanRow, "tags" | "serverIds">, all: { id: number; name: string; tags: string | null }[]): { id: number; name: string }[] {
|
||||
const tags = new Set(planTags(plan));
|
||||
const ids = new Set(planServerIds(plan));
|
||||
return all
|
||||
.filter((s) => ids.has(s.id) || parseTags(s.tags).some((t) => tags.has(t)))
|
||||
.map((s) => ({ id: s.id, name: s.name }))
|
||||
.sort((a, b) => a.name.localeCompare(b.name));
|
||||
}
|
||||
|
||||
/** The times a plan happens, from `from` to `to`. A plan has no length; the minute it starts is what matters. */
|
||||
export function planOccurrences(plan: Pick<PlanRow, "days" | "startTime" | "timezone">, from: Date, to: Date): Occurrence[] {
|
||||
return occurrencesBetween({ days: plan.days, startTime: plan.startTime, durationMinutes: 1, timezone: plan.timezone }, from, to);
|
||||
}
|
||||
|
||||
/** Pure: the occurrence a plan should start for now — its time has come, it is no more than the grace late, and it has not started on that day. */
|
||||
export function dueOccurrence(plan: Pick<PlanRow, "enabled" | "days" | "startTime" | "timezone" | "lastRunDate">, now: Date, graceMs: number = START_GRACE_MS): Occurrence | null {
|
||||
if (!plan.enabled) return null;
|
||||
const found = planOccurrences(plan, new Date(now.getTime() - graceMs - 60_000), new Date(now.getTime() + 1000)).filter((o) => o.start.getTime() <= now.getTime() && now.getTime() - o.start.getTime() <= graceMs && o.date !== plan.lastRunDate);
|
||||
return found.length > 0 ? found[found.length - 1] : null;
|
||||
}
|
||||
|
||||
// ─── Starting a run ────────────────────────────────────────────────────────
|
||||
|
||||
const ACTIVE_RUN: RunStatus[] = ["running"];
|
||||
|
||||
export async function activeRunOf(planId: number): Promise<PlanRunRow | null> {
|
||||
const [run] = await db.select().from(patchPlanRuns).where(and(eq(patchPlanRuns.planId, planId), inArray(patchPlanRuns.status, ACTIVE_RUN))).limit(1);
|
||||
return run ?? null;
|
||||
}
|
||||
|
||||
export async function startPlanRun(planId: number, options: { trigger: "schedule" | "manual"; actor: string; now?: Date }): Promise<PlanRunRow> {
|
||||
const now = options.now ?? new Date();
|
||||
const [plan] = await db.select().from(patchPlans).where(eq(patchPlans.id, planId)).limit(1);
|
||||
if (!plan) throw new PlanError(404, "not_found", "That plan doesn't exist.");
|
||||
if (await activeRunOf(plan.id)) throw new PlanError(409, "already_running", `"${plan.name}" is already running.`);
|
||||
const matched = planServers(plan, await db.select({ id: servers.id, name: servers.name, tags: servers.tags }).from(servers));
|
||||
if (matched.length === 0) throw new PlanError(400, "no_servers", "No server has this plan's tags or is named in it, so there is nothing to update.");
|
||||
const items: PlanItem[] = matched.map((s) => ({ serverId: s.id, name: s.name, status: "pending", updateRunId: null, message: null, windowIds: [] }));
|
||||
const [run] = await db
|
||||
.insert(patchPlanRuns)
|
||||
.values({ planId: plan.id, planName: plan.name, trigger: options.trigger, startedBy: options.actor, status: "running", message: `Updating ${matched.length} server${matched.length === 1 ? "" : "s"}, one at a time.`, items: JSON.stringify(items), startedAt: now.toISOString() })
|
||||
.returning();
|
||||
await recordAudit({ category: "patch_plan", action: "run", targetType: "patch_plan", targetId: plan.id, detail: { name: plan.name, run: run.id, trigger: options.trigger, servers: matched.length, by: options.actor } });
|
||||
return run;
|
||||
}
|
||||
|
||||
// ─── Following a run ───────────────────────────────────────────────────────
|
||||
|
||||
async function endWindows(ids: number[], now: Date): Promise<void> {
|
||||
if (ids.length === 0) return;
|
||||
await db.update(maintenanceWindows).set({ endsAt: now.toISOString() }).where(inArray(maintenanceWindows.id, ids));
|
||||
invalidateAlertsCache();
|
||||
}
|
||||
|
||||
/** Silences what depends on the server for as long as it is being updated; returns the windows it made (ones it only extended belong to someone else). */
|
||||
async function silenceDependents(plan: PlanRow, serverId: number, serverName: string, actor: string, now: Date): Promise<number[]> {
|
||||
if (!plan.silenceDependents) return [];
|
||||
const made: number[] = [];
|
||||
try {
|
||||
const impact = await impactOf("server", serverId, now);
|
||||
for (const d of impact?.dependents ?? []) {
|
||||
const started = await startOrExtendWindow(d.type, d.id, plan.windowMinutes, `Depends on ${serverName} — patch plan "${plan.name}"`.slice(0, 200), actor, now);
|
||||
if (started && !started.extended) made.push(started.window.id);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[patchPlans] couldn't silence what depends on a server:", err);
|
||||
}
|
||||
return made;
|
||||
}
|
||||
|
||||
const words = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
|
||||
|
||||
/** What a finished run says, in a line. */
|
||||
export function summariseItems(items: PlanItem[]): string {
|
||||
const count = (s: ItemStatus) => items.filter((i) => i.status === s).length;
|
||||
const parts = [`${count("done")} updated`];
|
||||
if (count("failed") > 0) parts.push(`${count("failed")} failed`);
|
||||
if (count("skipped") > 0) parts.push(`${count("skipped")} left alone`);
|
||||
return `${parts.join(", ")} of ${words(items.length, "server")}.`;
|
||||
}
|
||||
|
||||
async function saveRun(run: PlanRunRow, items: PlanItem[], extra: Partial<PlanRunRow> = {}): Promise<void> {
|
||||
await db.update(patchPlanRuns).set({ items: JSON.stringify(items), ...extra }).where(eq(patchPlanRuns.id, run.id));
|
||||
}
|
||||
|
||||
async function finishRun(run: PlanRunRow, plan: PlanRow | null, items: PlanItem[], status: RunStatus, now: Date, why?: string): Promise<void> {
|
||||
const message = [summariseItems(items), why].filter(Boolean).join(" ");
|
||||
await saveRun(run, items, { status, message, finishedAt: now.toISOString() });
|
||||
await recordAudit({ category: "patch_plan", action: status === "done" ? "finished" : status, targetType: "patch_plan", targetId: run.planId, detail: { name: run.planName, run: run.id, message } });
|
||||
if (status !== "cancelled") {
|
||||
const lines = items.map((i) => `${i.status === "done" ? "✓" : i.status === "failed" ? "✕" : "–"} ${i.name}${i.message ? ` — ${i.message}` : ""}`);
|
||||
await notifyPatchPlanFinished({ planName: run.planName, ok: status === "done", message, lines }).catch((err) => console.error("[patchPlans] couldn't send the result:", err));
|
||||
}
|
||||
void plan;
|
||||
}
|
||||
|
||||
/** When a server's turn is over and it failed: stop (leaving the rest alone) or carry on. */
|
||||
function afterFailure(plan: PlanRow, items: PlanItem[]): boolean {
|
||||
if (!plan.stopOnFailure) return false;
|
||||
for (const i of items) if (i.status === "pending") Object.assign(i, { status: "skipped", message: "Not started, because an earlier server failed." });
|
||||
return true;
|
||||
}
|
||||
|
||||
async function advanceRun(run: PlanRunRow, now: Date): Promise<void> {
|
||||
const [plan] = await db.select().from(patchPlans).where(eq(patchPlans.id, run.planId)).limit(1);
|
||||
const items = parseItems(run.items);
|
||||
if (!plan) return; // deleted: the run goes with it
|
||||
|
||||
// The server whose turn it is: has its update run finished?
|
||||
const current = items.find((i) => i.status === "running");
|
||||
if (current) {
|
||||
const [ur] = current.updateRunId !== null ? await db.select().from(updateRuns).where(eq(updateRuns.id, current.updateRunId)).limit(1) : [];
|
||||
if (ur && ACTIVE_STATUSES.includes(ur.status as (typeof ACTIVE_STATUSES)[number])) return;
|
||||
const ok = ur?.status === "done";
|
||||
current.status = ok ? "done" : "failed";
|
||||
current.message = ur?.message ?? "The update run is gone.";
|
||||
await endWindows(current.windowIds, now);
|
||||
if (!ok && afterFailure(plan, items)) {
|
||||
await finishRun(run, plan, items, "failed", now, `Stopped after ${current.name}.`);
|
||||
return;
|
||||
}
|
||||
await saveRun(run, items);
|
||||
}
|
||||
|
||||
// The next one.
|
||||
for (;;) {
|
||||
const next = items.find((i) => i.status === "pending");
|
||||
if (!next) {
|
||||
await finishRun(run, plan, items, items.some((i) => i.status === "failed") ? "failed" : "done", now);
|
||||
return;
|
||||
}
|
||||
const [server] = await db.select().from(servers).where(eq(servers.id, next.serverId)).limit(1);
|
||||
if (!server) {
|
||||
Object.assign(next, { status: "skipped", message: "The server was removed." });
|
||||
continue;
|
||||
}
|
||||
if (plan.onlyWithUpdates) {
|
||||
const updates = parsePendingUpdates(server.pendingUpdates);
|
||||
if (!updates || updates.count === 0) {
|
||||
Object.assign(next, { status: "skipped", message: "No updates were waiting." });
|
||||
continue;
|
||||
}
|
||||
}
|
||||
const actor = `Patch plan "${plan.name}"`;
|
||||
try {
|
||||
const ur = await startUpdateRun(
|
||||
{
|
||||
serverId: server.id,
|
||||
integrationId: plan.integrationId,
|
||||
projectId: plan.projectId,
|
||||
templateId: plan.templateId,
|
||||
limit: plan.limitFrom === "hostname" && server.hostname ? server.hostname : server.name,
|
||||
windowMinutes: plan.windowMinutes,
|
||||
actor,
|
||||
},
|
||||
now,
|
||||
);
|
||||
Object.assign(next, { status: "running", updateRunId: ur.id, message: "The update run has started.", windowIds: await silenceDependents(plan, server.id, server.name, actor, now) });
|
||||
await saveRun(run, items);
|
||||
return; // one at a time: this one is followed until it is over
|
||||
} catch (err) {
|
||||
const message = err instanceof UpdateRunError || err instanceof Error ? err.message : String(err);
|
||||
Object.assign(next, { status: "failed", message });
|
||||
if (afterFailure(plan, items)) {
|
||||
await finishRun(run, plan, items, "failed", now, `Stopped at ${next.name}.`);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** One pass over every plan run that is under way. Returns how many it looked at (for the tests). */
|
||||
export async function advancePlanRuns(now: Date = new Date()): Promise<number> {
|
||||
const active = await db.select().from(patchPlanRuns).where(inArray(patchPlanRuns.status, ACTIVE_RUN));
|
||||
for (const run of active) {
|
||||
try {
|
||||
await advanceRun(run, now);
|
||||
} catch (err) {
|
||||
console.error(`[patchPlans] run ${run.id}:`, err);
|
||||
}
|
||||
}
|
||||
return active.length;
|
||||
}
|
||||
|
||||
/** Starts the plans whose time has come. Returns how many it started. */
|
||||
export async function startDuePlans(now: Date = new Date()): Promise<number> {
|
||||
let started = 0;
|
||||
for (const plan of await db.select().from(patchPlans)) {
|
||||
const due = dueOccurrence(plan, now);
|
||||
if (!due) continue;
|
||||
if (await activeRunOf(plan.id)) continue; // still on the last one: today is skipped rather than overlapping
|
||||
// Written down first, so that whatever happens next it is not tried again every twenty seconds for half an hour.
|
||||
await db.update(patchPlans).set({ lastRunDate: due.date }).where(eq(patchPlans.id, plan.id));
|
||||
try {
|
||||
await startPlanRun(plan.id, { trigger: "schedule", actor: "the schedule", now });
|
||||
started++;
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
await db.insert(patchPlanRuns).values({ planId: plan.id, planName: plan.name, trigger: "schedule", startedBy: "the schedule", status: "failed", message: `Didn't start: ${message}`, items: "[]", startedAt: now.toISOString(), finishedAt: now.toISOString() });
|
||||
}
|
||||
}
|
||||
return started;
|
||||
}
|
||||
|
||||
export async function cancelPlanRun(id: number, actor: string, now: Date = new Date()): Promise<PlanRunRow | null> {
|
||||
const [run] = await db.select().from(patchPlanRuns).where(eq(patchPlanRuns.id, id)).limit(1);
|
||||
if (!run) return null;
|
||||
if (run.status !== "running") throw new PlanError(409, "not_active", "That run is already over.");
|
||||
const items = parseItems(run.items);
|
||||
for (const i of items) {
|
||||
if (i.status === "pending") Object.assign(i, { status: "skipped", message: `Not started: stopped by ${actor}.` });
|
||||
else if (i.status === "running") {
|
||||
if (i.updateRunId !== null) await cancelUpdateRun(i.updateRunId, actor, now).catch(() => undefined);
|
||||
await endWindows(i.windowIds, now);
|
||||
Object.assign(i, { status: "failed", message: `Stopped being followed by ${actor}. A playbook already started keeps running in Semaphore.` });
|
||||
}
|
||||
}
|
||||
const [plan] = await db.select().from(patchPlans).where(eq(patchPlans.id, run.planId)).limit(1);
|
||||
await finishRun(run, plan ?? null, items, "cancelled", now, `Stopped by ${actor}.`);
|
||||
const [after] = await db.select().from(patchPlanRuns).where(eq(patchPlanRuns.id, id)).limit(1);
|
||||
return after;
|
||||
}
|
||||
|
||||
/** Recent runs, those still going first. */
|
||||
export async function listPlanRuns(options: { planId?: number; limit?: number } = {}): Promise<PlanRunRow[]> {
|
||||
const rows = await db
|
||||
.select()
|
||||
.from(patchPlanRuns)
|
||||
.where(options.planId !== undefined ? eq(patchPlanRuns.planId, options.planId) : undefined)
|
||||
.orderBy(desc(patchPlanRuns.startedAt))
|
||||
.limit(options.limit ?? 20);
|
||||
return [...rows.filter((r) => r.status === "running"), ...rows.filter((r) => r.status !== "running")];
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
/** Every twenty seconds: start what is due, follow what is going. A run that was under way when the app last stopped is picked up where it was. */
|
||||
export function initPatchPlans(): void {
|
||||
if (timer) return;
|
||||
timer = setInterval(() => {
|
||||
void startDuePlans()
|
||||
.then(() => advancePlanRuns())
|
||||
.catch((err) => console.error("[patchPlans]", err));
|
||||
}, POLL_INTERVAL_MS);
|
||||
timer.unref();
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
/**
|
||||
* Renewals: a certificate, domain registration or secret whose expiry moved later was renewed. The app sees it happen when it re-reads the date — an endpoint
|
||||
* check seeing a new certificate, the registry saying a domain now runs longer, a secret's live certificate being replaced — and writes it down, so the monthly
|
||||
* report can say "certificates renewed: 3". It only goes back as far as the version that began watching; nothing is guessed about earlier.
|
||||
*/
|
||||
import { and, eq, gte, lt } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { renewals } from "../db/schema.js";
|
||||
|
||||
/** An expiry has to move at least this far for it to be a renewal (a registry rounding a date by a day is not one). */
|
||||
export const MIN_RENEWAL_DAYS = 20;
|
||||
const DAY_MS = 86_400_000;
|
||||
|
||||
export type RenewalKind = "certificate" | "domain" | "secret";
|
||||
|
||||
const dayOf = (value: string | null | undefined): string | null => (value && /^\d{4}-\d{2}-\d{2}/.test(value) ? value.slice(0, 10) : null);
|
||||
|
||||
/** Pure: whether the expiry went from one date to a clearly later one. Either may be a full ISO time; only the day counts. */
|
||||
export function isRenewal(from: string | null | undefined, to: string | null | undefined): boolean {
|
||||
const a = dayOf(from);
|
||||
const b = dayOf(to);
|
||||
if (!a || !b) return false;
|
||||
return (Date.parse(`${b}T00:00:00Z`) - Date.parse(`${a}T00:00:00Z`)) / DAY_MS >= MIN_RENEWAL_DAYS;
|
||||
}
|
||||
|
||||
/** Writes down a renewal if the dates say it was one (and the same one is not already written down). Never throws: it must not hold up the check that saw it. */
|
||||
export async function noteRenewal(kind: RenewalKind, subject: string, from: string | null | undefined, to: string | null | undefined, now: Date = new Date()): Promise<boolean> {
|
||||
try {
|
||||
if (!isRenewal(from, to)) return false;
|
||||
const fromDate = dayOf(from)!;
|
||||
const toDate = dayOf(to)!;
|
||||
const [same] = await db.select({ id: renewals.id }).from(renewals).where(and(eq(renewals.kind, kind), eq(renewals.subject, subject), eq(renewals.toDate, toDate))).limit(1);
|
||||
if (same) return false;
|
||||
await db.insert(renewals).values({ kind, subject, at: now.toISOString(), fromDate, toDate });
|
||||
return true;
|
||||
} catch (err) {
|
||||
console.error("[renewals] couldn't record a renewal:", err);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function renewalsBetween(from: Date, to: Date) {
|
||||
return db.select().from(renewals).where(and(gte(renewals.at, from.toISOString()), lt(renewals.at, to.toISOString()))).orderBy(renewals.at);
|
||||
}
|
||||
@@ -113,6 +113,17 @@ export interface HealthCheckSettings {
|
||||
backupStaleHours: number;
|
||||
/** A restore test older than this many days (or none at all) is a problem for a guest that has backups; 0 = never (a failed test always is). */
|
||||
restoreTestDays: number;
|
||||
/** A server that starts listening on a TCP port it did not at first, and that is not noted on its Ports card, is a warning (services/newPorts.ts). */
|
||||
newPortAlerts: boolean;
|
||||
}
|
||||
|
||||
/** The monthly report: last month's availability, incidents, updates and renewals, in one message on a day of the month. Off until switched on. */
|
||||
export interface MonthlyReportSettings {
|
||||
enabled: boolean;
|
||||
/** Day of the month, 1–28 (every month has it), in notifications.timezone. */
|
||||
day: number;
|
||||
/** "HH:MM", in notifications.timezone. */
|
||||
time: string;
|
||||
}
|
||||
|
||||
/** The weekly digest: one message a week through the notification channels. Off until an admin switches it on under Settings → Notifications. */
|
||||
@@ -225,6 +236,7 @@ export interface AppSettings {
|
||||
quietHours: QuietHoursSettings;
|
||||
healthChecks: HealthCheckSettings;
|
||||
digest: DigestSettings;
|
||||
monthlyReport: MonthlyReportSettings;
|
||||
signIn: SignInSettings;
|
||||
profilePictures: ProfilePictureSettings;
|
||||
inventory: InventorySettings;
|
||||
@@ -267,12 +279,13 @@ const DEFAULTS: AppSettings = {
|
||||
logRetention: { enabled: false, retentionDays: 90, intervalHours: 24 },
|
||||
quietHours: { enabled: false, start: "22:00", end: "07:00" },
|
||||
digest: { enabled: false, day: 1, time: "08:00" },
|
||||
monthlyReport: { enabled: false, day: 1, time: "08:00" },
|
||||
inventory: { currency: "", pricePerKwh: null },
|
||||
agents: { autoUpdate: false },
|
||||
selfBackup: { enabled: true, time: "03:30", keep: 14 },
|
||||
profilePictures: { gravatar: false, identityProvider: false },
|
||||
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
|
||||
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0 },
|
||||
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true },
|
||||
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
|
||||
// Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN.
|
||||
consistency: { excludedRanges: ["172.16.0.0/12"] },
|
||||
|
||||
@@ -5,6 +5,7 @@ import { recordServerMetrics } from "./serverMetrics.js";
|
||||
import { nextUpdateState, type IncomingUpdates } from "./pendingUpdates.js";
|
||||
import { nextHostHealth, type IncomingHostHealth } from "./hostHealth.js";
|
||||
import { recordAudit } from "./audit.js";
|
||||
import { baselineFrom } from "./newPorts.js";
|
||||
import { fillInventoryFromServer } from "./inventoryAutofill.js";
|
||||
import { nextDiagnostics, type AgentDiagnostic } from "./agentDiagnostics.js";
|
||||
|
||||
@@ -113,7 +114,7 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
|
||||
}
|
||||
|
||||
const system = report.system;
|
||||
const [before] = await db.select({ pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince, hostHealth: servers.hostHealth, agentVersion: servers.agentVersion }).from(servers).where(eq(servers.id, serverId));
|
||||
const [before] = await db.select({ pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince, hostHealth: servers.hostHealth, agentVersion: servers.agentVersion, portBaseline: servers.portBaseline }).from(servers).where(eq(servers.id, serverId));
|
||||
await db
|
||||
.update(servers)
|
||||
.set({
|
||||
@@ -144,6 +145,8 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
|
||||
: {}),
|
||||
// Only stored when the agent reported it, so an agent that predates this doesn't wipe the field.
|
||||
...(system?.listening_ports ? { listeningPorts: JSON.stringify(system.listening_ports) } : {}),
|
||||
// The first report that carries ports says what is normal for this server; a port that appears later is "new" (services/newPorts.ts).
|
||||
...(system?.listening_ports && before && before.portBaseline === null ? { portBaseline: baselineFrom(system.listening_ports) } : {}),
|
||||
})
|
||||
.where(eq(servers.id, serverId));
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import { eq, isNotNull } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { secrets } from "../db/schema.js";
|
||||
import { resolveTarget } from "../utils/networkGuard.js";
|
||||
import { noteRenewal } from "./renewals.js";
|
||||
|
||||
/**
|
||||
* Opens a real TLS connection and returns the server certificate's expiry as
|
||||
@@ -79,6 +80,8 @@ export async function refreshTlsSecret(id: number): Promise<TlsCheckResult | nul
|
||||
...(expiryDate !== row.expiryDate ? { updatedAt: now } : {}),
|
||||
})
|
||||
.where(eq(secrets.id, id));
|
||||
// A certificate that is now good for much longer than it was has been replaced.
|
||||
await noteRenewal("certificate", row.name, row.expiryDate, expiryDate);
|
||||
return { id, name: row.name, host: row.checkHost, port, ok: true, expiryDate };
|
||||
} catch (err) {
|
||||
const error = err instanceof Error ? err.message : String(err);
|
||||
|
||||
Reference in new issue
Block a user