Duplicati: know when a backup failed or has gone quiet

Duplicati reports each operation to a secret address (its own send-http-url option), so
nothing calls Duplicati and no password for it is kept. Each machine is a source; each job
is learned by name and shows its last result, duration, size, changes and a history.
A failed run is a warning (critical from the second in a row); one that has not succeeded
for longer than expected (per job, 2 days by default) is a warning, critical at three
times that. Maintenance on the machine's server silences them; /metrics series added.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-24 21:36:13 +02:00
1 parent 12f2662465
commit 2f082ca5c2
23 files changed
+8047 -4

No files matched your search

+15
View File
@@ -8,6 +8,7 @@ import { db } from "../db/client.js";
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState } from "../db/schema.js";
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
import { levelStatuses } from "./serviceLevels.js";
import { backupState, listAll as listDuplicati } from "./duplicati.js";
import { latestTests } from "./restoreTests.js";
import { heartbeatState, parseStamp } from "./heartbeats.js";
import { env } from "../env.js";
@@ -59,6 +60,19 @@ class Exposition {
}
}
/** Each Duplicati backup, for /metrics: whether it is in order, how long since it last succeeded, and how big what it backs up is. */
async function addDuplicatiMetrics(out: Exposition, now: Date) {
const { sources, backups } = await listDuplicati();
const names = new Map(sources.map((s) => [s.id, s.name]));
for (const b of backups) {
const labels = { backup: b.name, source: names.get(b.sourceId) ?? "" };
const state = backupState(b, now.getTime());
out.add("homelab_duplicati_backup_ok", "gauge", "1 if the Duplicati backup's last run worked and it is not overdue, else 0.", state === "ok" || state === "warning" ? 1 : 0, labels);
if (b.lastSuccessAt) out.add("homelab_duplicati_last_success_age_seconds", "gauge", "Seconds since the Duplicati backup last succeeded (a run with warnings counts).", Math.max(0, Math.round((now.getTime() - Date.parse(b.lastSuccessAt)) / 1000)), labels);
if (b.sizeBytes !== null) out.add("homelab_duplicati_source_size_bytes", "gauge", "The size of what the Duplicati backup examined, as its last report said.", b.sizeBytes, labels);
}
}
/** Each service-level target, for /metrics: its target, how it is doing this month and how much error budget is left. */
async function addServiceLevelMetrics(out: Exposition, now: Date) {
for (const s of await levelStatuses(now, { history: false })) {
@@ -252,6 +266,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
await addEndpointMetrics(out, now);
await addUpsMetrics(out, now);
await addServiceLevelMetrics(out, now);
await addDuplicatiMetrics(out, now);
await addHeartbeatMetrics(out, now);
return out.toString();
}