Stale-backup alerts, per-server alert limits, and update runs from the Updates page
- Backups: a guest whose newest backup is older than its limit is an alert (backup kind), in the daily backup message, the digest and /metrics (homelab_backup_guests, ..._last_age_seconds) - Per-server thresholds (servers.thresholds, migration 0030): offline minutes, disk percent, forecast window, backup age, and per-disk limits or "leave alone"; applied everywhere the global ones were; saving clears the alerts cache - Update runs (update_runs, migration 0031): Update... on the Updates page puts the server under maintenance, runs a Semaphore playbook limited to it, waits for the host to report, and reports what is left and whether a restart is due; poller survives restarts - Notification toggle for update-run results; tests and docs Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5dc6cea69c
commit
7a5b623444
42 files changed
+7561
-57
No files matched your search
@@ -17,6 +17,7 @@ import { notify, notifyNow } from "./notify.js";
|
||||
import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
import { computeSecretStatus } from "./secretStatus.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { getUptime } from "./uptimeHistory.js";
|
||||
|
||||
@@ -68,7 +69,7 @@ export async function buildDigest(now: Date = new Date()): Promise<Digest> {
|
||||
const sections: Section[] = [];
|
||||
|
||||
const serverRows = await db
|
||||
.select({ id: servers.id, name: servers.name, lastSeenAt: servers.lastSeenAt, pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince })
|
||||
.select({ id: servers.id, name: servers.name, lastSeenAt: servers.lastSeenAt, pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince, thresholds: servers.thresholds })
|
||||
.from(servers);
|
||||
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
|
||||
|
||||
@@ -134,11 +135,14 @@ export async function buildDigest(now: Date = new Date()): Promise<Digest> {
|
||||
|
||||
// ── Disks heading for full ──
|
||||
try {
|
||||
const { diskForecastDays, diskUsagePercent } = settings.healthChecks;
|
||||
if (diskForecastDays > 0) {
|
||||
const { diskUsagePercent } = settings.healthChecks;
|
||||
{
|
||||
const lines: string[] = [];
|
||||
for (const s of serverRows) {
|
||||
for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), diskForecastDays, diskUsagePercent)) {
|
||||
const own = parseThresholds(s.thresholds);
|
||||
const forecastDays = effectiveThresholds(settings.healthChecks, own).diskForecastDays;
|
||||
if (!(forecastDays > 0)) continue;
|
||||
for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), forecastDays, (mount) => diskLimit(own, mount, diskUsagePercent))) {
|
||||
lines.push(`- ${s.name}: ${f.mount} is ${Math.round(f.latest)}% full and filling by about ${f.perDay.toFixed(1)} points a day — full in about ${plural(f.daysToFull, "day")}`);
|
||||
}
|
||||
}
|
||||
@@ -196,7 +200,7 @@ export async function buildDigest(now: Date = new Date()): Promise<Digest> {
|
||||
|
||||
// ── Backups ──
|
||||
try {
|
||||
const coverage = await getBackupCoverage({ now });
|
||||
const coverage = await getBackupCoverage({ now, rules: true });
|
||||
const order = { failed: 0, none: 1, stale: 2 } as const;
|
||||
const bad = coverage.guests.filter((g): g is typeof g & { status: keyof typeof order } => g.status === "failed" || g.status === "none" || g.status === "stale").sort((a, b) => order[a.status] - order[b.status] || a.name.localeCompare(b.name));
|
||||
if (bad.length > 0) {
|
||||
|
||||
Reference in new issue
Block a user