Stale-backup alerts, per-server alert limits, and update runs from the Updates page

- Backups: a guest whose newest backup is older than its limit is an alert (backup kind),
  in the daily backup message, the digest and /metrics (homelab_backup_guests, ..._last_age_seconds)
- Per-server thresholds (servers.thresholds, migration 0030): offline minutes, disk percent,
  forecast window, backup age, and per-disk limits or "leave alone"; applied everywhere the
  global ones were; saving clears the alerts cache
- Update runs (update_runs, migration 0031): Update... on the Updates page puts the server under
  maintenance, runs a Semaphore playbook limited to it, waits for the host to report, and
  reports what is left and whether a restart is due; poller survives restarts
- Notification toggle for update-run results; tests and docs

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 01:25:09 +02:00
1 parent 5dc6cea69c
commit 7a5b623444
42 files changed
+7561 -57

No files matched your search

+9 -5
View File
@@ -17,6 +17,7 @@ import { notify, notifyNow } from "./notify.js";
import { parsePendingUpdates } from "./pendingUpdates.js";
import { computeSecretStatus } from "./secretStatus.js";
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
import { getSettings } from "./settingsStore.js";
import { getUptime } from "./uptimeHistory.js";
@@ -68,7 +69,7 @@ export async function buildDigest(now: Date = new Date()): Promise<Digest> {
const sections: Section[] = [];
const serverRows = await db
.select({ id: servers.id, name: servers.name, lastSeenAt: servers.lastSeenAt, pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince })
.select({ id: servers.id, name: servers.name, lastSeenAt: servers.lastSeenAt, pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince, thresholds: servers.thresholds })
.from(servers);
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
@@ -134,11 +135,14 @@ export async function buildDigest(now: Date = new Date()): Promise<Digest> {
// ── Disks heading for full ──
try {
const { diskForecastDays, diskUsagePercent } = settings.healthChecks;
if (diskForecastDays > 0) {
const { diskUsagePercent } = settings.healthChecks;
{
const lines: string[] = [];
for (const s of serverRows) {
for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), diskForecastDays, diskUsagePercent)) {
const own = parseThresholds(s.thresholds);
const forecastDays = effectiveThresholds(settings.healthChecks, own).diskForecastDays;
if (!(forecastDays > 0)) continue;
for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), forecastDays, (mount) => diskLimit(own, mount, diskUsagePercent))) {
lines.push(`- ${s.name}: ${f.mount} is ${Math.round(f.latest)}% full and filling by about ${f.perDay.toFixed(1)} points a day — full in about ${plural(f.daysToFull, "day")}`);
}
}
@@ -196,7 +200,7 @@ export async function buildDigest(now: Date = new Date()): Promise<Digest> {
// ── Backups ──
try {
const coverage = await getBackupCoverage({ now });
const coverage = await getBackupCoverage({ now, rules: true });
const order = { failed: 0, none: 1, stale: 2 } as const;
const bad = coverage.guests.filter((g): g is typeof g & { status: keyof typeof order } => g.status === "failed" || g.status === "none" || g.status === "stale").sort((a, b) => order[a.status] - order[b.status] || a.name.localeCompare(b.name));
if (bad.length > 0) {