Duplicati: know when a backup failed or has gone quiet
Duplicati reports each operation to a secret address (its own send-http-url option), so nothing calls Duplicati and no password for it is kept. Each machine is a source; each job is learned by name and shows its last result, duration, size, changes and a history. A failed run is a warning (critical from the second in a row); one that has not succeeded for longer than expected (per job, 2 days by default) is a warning, critical at three times that. Maintenance on the machine's server silences them; /metrics series added. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
12f2662465
commit
2f082ca5c2
23 files changed
+8047
-4
No files matched your search
@@ -8,6 +8,7 @@ import { db } from "../db/client.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState } from "../db/schema.js";
|
||||
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
|
||||
import { levelStatuses } from "./serviceLevels.js";
|
||||
import { backupState, listAll as listDuplicati } from "./duplicati.js";
|
||||
import { latestTests } from "./restoreTests.js";
|
||||
import { heartbeatState, parseStamp } from "./heartbeats.js";
|
||||
import { env } from "../env.js";
|
||||
@@ -59,6 +60,19 @@ class Exposition {
|
||||
}
|
||||
}
|
||||
|
||||
/** Each Duplicati backup, for /metrics: whether it is in order, how long since it last succeeded, and how big what it backs up is. */
|
||||
async function addDuplicatiMetrics(out: Exposition, now: Date) {
|
||||
const { sources, backups } = await listDuplicati();
|
||||
const names = new Map(sources.map((s) => [s.id, s.name]));
|
||||
for (const b of backups) {
|
||||
const labels = { backup: b.name, source: names.get(b.sourceId) ?? "" };
|
||||
const state = backupState(b, now.getTime());
|
||||
out.add("homelab_duplicati_backup_ok", "gauge", "1 if the Duplicati backup's last run worked and it is not overdue, else 0.", state === "ok" || state === "warning" ? 1 : 0, labels);
|
||||
if (b.lastSuccessAt) out.add("homelab_duplicati_last_success_age_seconds", "gauge", "Seconds since the Duplicati backup last succeeded (a run with warnings counts).", Math.max(0, Math.round((now.getTime() - Date.parse(b.lastSuccessAt)) / 1000)), labels);
|
||||
if (b.sizeBytes !== null) out.add("homelab_duplicati_source_size_bytes", "gauge", "The size of what the Duplicati backup examined, as its last report said.", b.sizeBytes, labels);
|
||||
}
|
||||
}
|
||||
|
||||
/** Each service-level target, for /metrics: its target, how it is doing this month and how much error budget is left. */
|
||||
async function addServiceLevelMetrics(out: Exposition, now: Date) {
|
||||
for (const s of await levelStatuses(now, { history: false })) {
|
||||
@@ -252,6 +266,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
await addEndpointMetrics(out, now);
|
||||
await addUpsMetrics(out, now);
|
||||
await addServiceLevelMetrics(out, now);
|
||||
await addDuplicatiMetrics(out, now);
|
||||
await addHeartbeatMetrics(out, now);
|
||||
return out.toString();
|
||||
}
|
||||
Reference in new issue
Block a user