Stale-backup alerts, per-server alert limits, and update runs from the Updates page
- Backups: a guest whose newest backup is older than its limit is an alert (backup kind), in the daily backup message, the digest and /metrics (homelab_backup_guests, ..._last_age_seconds) - Per-server thresholds (servers.thresholds, migration 0030): offline minutes, disk percent, forecast window, backup age, and per-disk limits or "leave alone"; applied everywhere the global ones were; saving clears the alerts cache - Update runs (update_runs, migration 0031): Update... on the Updates page puts the server under maintenance, runs a Semaphore playbook limited to it, waits for the host to report, and reports what is left and whether a restart is due; poller survives restarts - Notification toggle for update-run results; tests and docs Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5dc6cea69c
commit
7a5b623444
42 files changed
+7561
-57
No files matched your search
@@ -13,6 +13,8 @@ import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, secrets, servers } from "../db/schema.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
|
||||
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createUptimeKumaAdapter } from "../integrations/uptimekuma/adapter.js";
|
||||
import { createOsTicketAdapter } from "../integrations/osticket/adapter.js";
|
||||
@@ -241,6 +243,25 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
sourceFailures.forEach((f) => cannotCheckIntegration(f, "Proxmox backups"));
|
||||
}),
|
||||
|
||||
// A guest whose newest backup is older than its limit — which Proxmox's own "latest run failed" alert can't see, because that only
|
||||
// looks at the most recent run per node.
|
||||
check("Backup coverage", async () => {
|
||||
const now = new Date();
|
||||
const report = await getBackupCoverage({ now, rules: true });
|
||||
for (const s of staleBackups(report, now, subjects)) {
|
||||
push({
|
||||
category: "backup",
|
||||
severity: s.severity,
|
||||
type: "proxmox",
|
||||
message: s.message,
|
||||
key: s.key,
|
||||
link: "/backups",
|
||||
silenced: s.silenced,
|
||||
target: s.guest.server ? targetOf("server", s.guest.server.id) : integrationTarget(s.guest.integrationId),
|
||||
});
|
||||
}
|
||||
}),
|
||||
|
||||
check("Backup verification", async () => {
|
||||
const { problems, sourceFailures } = await collectPbsProblems({ skipSilenced: false });
|
||||
for (const p of problems) {
|
||||
@@ -376,10 +397,13 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
// Not a notification: a disk that is on course to fill is something to see coming, so it is shown here, from the server's own history.
|
||||
check("Disk forecasts", async () => {
|
||||
const { healthChecks } = await getSettings();
|
||||
if (!(healthChecks.diskForecastDays > 0)) return;
|
||||
for (const s of await db.select({ id: servers.id, name: servers.name }).from(servers)) {
|
||||
for (const s of await db.select({ id: servers.id, name: servers.name, thresholds: servers.thresholds }).from(servers)) {
|
||||
// A server may have its own forecast window (0 turns it off for that server), its own limit, and disks it is told to leave alone.
|
||||
const own = parseThresholds(s.thresholds);
|
||||
const forecastDays = effectiveThresholds(healthChecks, own).diskForecastDays;
|
||||
if (!(forecastDays > 0)) continue;
|
||||
const series = await getServerMetrics(s.id, 14);
|
||||
for (const f of disksFillingSoon(series, healthChecks.diskForecastDays, healthChecks.diskUsagePercent)) {
|
||||
for (const f of disksFillingSoon(series, forecastDays, (mount) => diskLimit(own, mount, healthChecks.diskUsagePercent))) {
|
||||
push({
|
||||
category: "disk",
|
||||
severity: f.daysToFull <= 3 ? "critical" : "warning",
|
||||
|
||||
Reference in new issue
Block a user