Duplicati: know when a backup failed or has gone quiet

Duplicati reports each operation to a secret address (its own send-http-url option), so
nothing calls Duplicati and no password for it is kept. Each machine is a source; each job
is learned by name and shows its last result, duration, size, changes and a history.
A failed run is a warning (critical from the second in a row); one that has not succeeded
for longer than expected (per job, 2 days by default) is a warning, critical at three
times that. Maintenance on the machine's server silences them; /metrics series added.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-24 21:36:13 +02:00
1 parent 12f2662465
commit 2f082ca5c2
23 files changed
+8047 -4

No files matched your search

+125
View File
@@ -0,0 +1,125 @@
import express, { Router } from "express";
import { desc, eq } from "drizzle-orm";
import { z } from "zod";
import { db } from "../db/client.js";
import { duplicatiBackups, duplicatiRuns, duplicatiSources, servers } from "../db/schema.js";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { invalidateAlertsCache } from "../services/alerts.js";
import { backupState, createSource, DEFAULT_EXPECTED_HOURS, duplicatiProblems, listAll, MAX_SOURCES, publicSource, recordReport } from "../services/duplicati.js";
import { asyncHandler } from "../utils/asyncHandler.js";
import { rateLimit } from "../utils/rateLimit.js";
/**
* The address Duplicati reports to: /api/duplicati-report/<token>. No session and no sign-in — the token is the whole of the access, and all it allows is to say how a backup
* went. Mounted ahead of the browser-only middleware (Duplicati is not a browser). A token that does not exist is answered like any other missing page, and the address is limited.
*/
export const duplicatiReportRouter = Router();
const limiter = rateLimit({ windowMs: 60_000, max: 120, message: "Too many requests. Wait a minute." });
// A report with its messages can be long; whatever it is, it is read as text.
const body = express.text({ type: () => true, limit: "2mb" });
duplicatiReportRouter.all("/:token", limiter, body, asyncHandler(async (req, res) => {
res.setHeader("Cache-Control", "no-store");
res.setHeader("X-Robots-Tag", "noindex, nofollow");
if (!["POST", "PUT"].includes(req.method)) return res.status(405).type("text/plain").send("Method not allowed");
const outcome = await recordReport(req.params.token, typeof req.body === "string" ? req.body : "");
if (!outcome.recorded && outcome.reason === "unknown_token") return res.status(404).type("text/plain").send("Not found");
invalidateAlertsCache();
res.type("text/plain").send(outcome.recorded ? "OK" : outcome.reason === "not_a_backup" ? "OK (not a backup: set aside)" : "OK (not a report this app can read)");
}));
export const duplicatiRouter = Router();
duplicatiRouter.use(requireAuth);
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
/** Every machine reporting, every backup job and where it stands: for everyone signed in, like the backup pages. */
duplicatiRouter.get("/", asyncHandler(async (_req, res) => {
const { sources, backups } = await listAll();
const now = Date.now();
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((s) => [s.id, s.name]));
const problems = duplicatiProblems(sources, backups, now);
res.json({
sources: sources.map((s) => ({ ...publicSource(s), serverName: s.serverId !== null ? (serverNames.get(s.serverId) ?? null) : null, backups: backups.filter((b) => b.sourceId === s.id).length })),
backups: backups.map((b) => ({ ...b, state: backupState(b, now), sourceName: sources.find((s) => s.id === b.sourceId)?.name ?? "", problem: problems.find((p) => new RegExp(`^duplicati-(failed|stale):${b.id}(:|$)`).test(p.key))?.message ?? null })),
servers: [...serverNames].map(([id, name]) => ({ id, name })).sort((a, b) => a.name.localeCompare(b.name)),
defaultExpectedHours: DEFAULT_EXPECTED_HOURS,
});
}));
duplicatiRouter.get("/backups/:id/runs", asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [b] = await db.select({ id: duplicatiBackups.id, name: duplicatiBackups.name }).from(duplicatiBackups).where(eq(duplicatiBackups.id, id)).limit(1);
if (!b) return res.status(404).json({ error: "not_found" });
const asked = Number(req.query.limit);
const limit = Number.isFinite(asked) ? Math.min(400, Math.max(1, Math.round(asked))) : 60;
res.json({ backup: b, runs: await db.select().from(duplicatiRuns).where(eq(duplicatiRuns.backupId, id)).orderBy(desc(duplicatiRuns.at), desc(duplicatiRuns.id)).limit(limit) });
}));
const sourceSchema = z.object({ name: z.string().trim().min(1).max(100), serverId: z.number().int().positive().nullable().optional() });
// Setting one up is an administrator's: it makes an address that can write to this app.
duplicatiRouter.post("/sources", requireRole("admin"), asyncHandler(async (req, res) => {
const parsed = sourceSchema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Give it a name." });
if ((await db.select({ id: duplicatiSources.id }).from(duplicatiSources)).length >= MAX_SOURCES) return res.status(409).json({ error: "too_many", message: `At most ${MAX_SOURCES} machines.` });
if (parsed.data.serverId != null && (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, parsed.data.serverId))).length === 0) return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
const { row, token } = await createSource({ name: parsed.data.name, serverId: parsed.data.serverId ?? null, createdBy: req.currentUser!.name ?? req.currentUser!.email ?? "someone" });
await recordAudit({ actor: req.currentUser!, category: "duplicati", action: "create_source", targetType: "duplicati_source", targetId: row.id, detail: { name: row.name } });
// The secret address is shown now and never again.
res.status(201).json({ source: publicSource(row), token, path: `/api/duplicati-report/${token}` });
}));
duplicatiRouter.patch("/sources/:id", requireRole("admin"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
const parsed = sourceSchema.safeParse(req.body);
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body" });
const [before] = await db.select().from(duplicatiSources).where(eq(duplicatiSources.id, id)).limit(1);
if (!before) return res.status(404).json({ error: "not_found" });
if (parsed.data.serverId != null && (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, parsed.data.serverId))).length === 0) return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
await db.update(duplicatiSources).set({ name: parsed.data.name, serverId: parsed.data.serverId ?? null }).where(eq(duplicatiSources.id, id));
await recordAudit({ actor: req.currentUser!, category: "duplicati", action: "update_source", targetType: "duplicati_source", targetId: id, detail: { name: parsed.data.name } });
invalidateAlertsCache();
res.json({ ok: true });
}));
duplicatiRouter.delete("/sources/:id", requireRole("admin"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [before] = await db.select().from(duplicatiSources).where(eq(duplicatiSources.id, id)).limit(1);
if (!before) return res.status(404).json({ error: "not_found" });
await db.delete(duplicatiSources).where(eq(duplicatiSources.id, id));
await recordAudit({ actor: req.currentUser!, category: "duplicati", action: "delete_source", targetType: "duplicati_source", targetId: id, detail: { name: before.name } });
invalidateAlertsCache();
res.status(204).end();
}));
const backupSchema = z.object({ expectedHours: z.number().int().min(1).max(24 * 90).optional(), alert: z.boolean().optional() });
// How often a backup is expected, and whether it alerts: an operator's call, like the other backup settings.
duplicatiRouter.patch("/backups/:id", requireRole("operator"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
const parsed = backupSchema.safeParse(req.body);
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", message: "Hours must be between 1 and 2160." });
const [before] = await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.id, id)).limit(1);
if (!before) return res.status(404).json({ error: "not_found" });
await db.update(duplicatiBackups).set({ ...(parsed.data.expectedHours !== undefined ? { expectedHours: parsed.data.expectedHours } : {}), ...(parsed.data.alert !== undefined ? { alert: parsed.data.alert } : {}) }).where(eq(duplicatiBackups.id, id));
await recordAudit({ actor: req.currentUser!, category: "duplicati", action: "update_backup", targetType: "duplicati_backup", targetId: id, detail: { name: before.name, ...parsed.data } });
invalidateAlertsCache();
res.json({ ok: true });
}));
// A job that no longer exists: forgotten, with its history. (If it reports again it is learned again.)
duplicatiRouter.delete("/backups/:id", requireRole("operator"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [before] = await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.id, id)).limit(1);
if (!before) return res.status(404).json({ error: "not_found" });
await db.delete(duplicatiBackups).where(eq(duplicatiBackups.id, id));
await recordAudit({ actor: req.currentUser!, category: "duplicati", action: "forget_backup", targetType: "duplicati_backup", targetId: id, detail: { name: before.name } });
invalidateAlertsCache();
res.status(204).end();
}));