Duplicati: know when a backup failed or has gone quiet
Duplicati reports each operation to a secret address (its own send-http-url option), so nothing calls Duplicati and no password for it is kept. Each machine is a source; each job is learned by name and shows its last result, duration, size, changes and a history. A failed run is a warning (critical from the second in a row); one that has not succeeded for longer than expected (per job, 2 days by default) is a warning, critical at three times that. Maintenance on the machine's server silences them; /metrics series added. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
12f2662465
commit
2f082ca5c2
23 files changed
+8047
-4
No files matched your search
@@ -0,0 +1,125 @@
|
||||
import express, { Router } from "express";
|
||||
import { desc, eq } from "drizzle-orm";
|
||||
import { z } from "zod";
|
||||
import { db } from "../db/client.js";
|
||||
import { duplicatiBackups, duplicatiRuns, duplicatiSources, servers } from "../db/schema.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { backupState, createSource, DEFAULT_EXPECTED_HOURS, duplicatiProblems, listAll, MAX_SOURCES, publicSource, recordReport } from "../services/duplicati.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
import { rateLimit } from "../utils/rateLimit.js";
|
||||
|
||||
/**
|
||||
* The address Duplicati reports to: /api/duplicati-report/<token>. No session and no sign-in — the token is the whole of the access, and all it allows is to say how a backup
|
||||
* went. Mounted ahead of the browser-only middleware (Duplicati is not a browser). A token that does not exist is answered like any other missing page, and the address is limited.
|
||||
*/
|
||||
export const duplicatiReportRouter = Router();
|
||||
|
||||
const limiter = rateLimit({ windowMs: 60_000, max: 120, message: "Too many requests. Wait a minute." });
|
||||
// A report with its messages can be long; whatever it is, it is read as text.
|
||||
const body = express.text({ type: () => true, limit: "2mb" });
|
||||
|
||||
duplicatiReportRouter.all("/:token", limiter, body, asyncHandler(async (req, res) => {
|
||||
res.setHeader("Cache-Control", "no-store");
|
||||
res.setHeader("X-Robots-Tag", "noindex, nofollow");
|
||||
if (!["POST", "PUT"].includes(req.method)) return res.status(405).type("text/plain").send("Method not allowed");
|
||||
const outcome = await recordReport(req.params.token, typeof req.body === "string" ? req.body : "");
|
||||
if (!outcome.recorded && outcome.reason === "unknown_token") return res.status(404).type("text/plain").send("Not found");
|
||||
invalidateAlertsCache();
|
||||
res.type("text/plain").send(outcome.recorded ? "OK" : outcome.reason === "not_a_backup" ? "OK (not a backup: set aside)" : "OK (not a report this app can read)");
|
||||
}));
|
||||
|
||||
export const duplicatiRouter = Router();
|
||||
duplicatiRouter.use(requireAuth);
|
||||
|
||||
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
|
||||
|
||||
/** Every machine reporting, every backup job and where it stands: for everyone signed in, like the backup pages. */
|
||||
duplicatiRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const { sources, backups } = await listAll();
|
||||
const now = Date.now();
|
||||
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((s) => [s.id, s.name]));
|
||||
const problems = duplicatiProblems(sources, backups, now);
|
||||
res.json({
|
||||
sources: sources.map((s) => ({ ...publicSource(s), serverName: s.serverId !== null ? (serverNames.get(s.serverId) ?? null) : null, backups: backups.filter((b) => b.sourceId === s.id).length })),
|
||||
backups: backups.map((b) => ({ ...b, state: backupState(b, now), sourceName: sources.find((s) => s.id === b.sourceId)?.name ?? "", problem: problems.find((p) => new RegExp(`^duplicati-(failed|stale):${b.id}(:|$)`).test(p.key))?.message ?? null })),
|
||||
servers: [...serverNames].map(([id, name]) => ({ id, name })).sort((a, b) => a.name.localeCompare(b.name)),
|
||||
defaultExpectedHours: DEFAULT_EXPECTED_HOURS,
|
||||
});
|
||||
}));
|
||||
|
||||
duplicatiRouter.get("/backups/:id/runs", asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [b] = await db.select({ id: duplicatiBackups.id, name: duplicatiBackups.name }).from(duplicatiBackups).where(eq(duplicatiBackups.id, id)).limit(1);
|
||||
if (!b) return res.status(404).json({ error: "not_found" });
|
||||
const asked = Number(req.query.limit);
|
||||
const limit = Number.isFinite(asked) ? Math.min(400, Math.max(1, Math.round(asked))) : 60;
|
||||
res.json({ backup: b, runs: await db.select().from(duplicatiRuns).where(eq(duplicatiRuns.backupId, id)).orderBy(desc(duplicatiRuns.at), desc(duplicatiRuns.id)).limit(limit) });
|
||||
}));
|
||||
|
||||
const sourceSchema = z.object({ name: z.string().trim().min(1).max(100), serverId: z.number().int().positive().nullable().optional() });
|
||||
|
||||
// Setting one up is an administrator's: it makes an address that can write to this app.
|
||||
duplicatiRouter.post("/sources", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const parsed = sourceSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Give it a name." });
|
||||
if ((await db.select({ id: duplicatiSources.id }).from(duplicatiSources)).length >= MAX_SOURCES) return res.status(409).json({ error: "too_many", message: `At most ${MAX_SOURCES} machines.` });
|
||||
if (parsed.data.serverId != null && (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, parsed.data.serverId))).length === 0) return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
|
||||
const { row, token } = await createSource({ name: parsed.data.name, serverId: parsed.data.serverId ?? null, createdBy: req.currentUser!.name ?? req.currentUser!.email ?? "someone" });
|
||||
await recordAudit({ actor: req.currentUser!, category: "duplicati", action: "create_source", targetType: "duplicati_source", targetId: row.id, detail: { name: row.name } });
|
||||
// The secret address is shown now and never again.
|
||||
res.status(201).json({ source: publicSource(row), token, path: `/api/duplicati-report/${token}` });
|
||||
}));
|
||||
|
||||
duplicatiRouter.patch("/sources/:id", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
const parsed = sourceSchema.safeParse(req.body);
|
||||
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body" });
|
||||
const [before] = await db.select().from(duplicatiSources).where(eq(duplicatiSources.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
if (parsed.data.serverId != null && (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, parsed.data.serverId))).length === 0) return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
|
||||
await db.update(duplicatiSources).set({ name: parsed.data.name, serverId: parsed.data.serverId ?? null }).where(eq(duplicatiSources.id, id));
|
||||
await recordAudit({ actor: req.currentUser!, category: "duplicati", action: "update_source", targetType: "duplicati_source", targetId: id, detail: { name: parsed.data.name } });
|
||||
invalidateAlertsCache();
|
||||
res.json({ ok: true });
|
||||
}));
|
||||
|
||||
duplicatiRouter.delete("/sources/:id", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [before] = await db.select().from(duplicatiSources).where(eq(duplicatiSources.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
await db.delete(duplicatiSources).where(eq(duplicatiSources.id, id));
|
||||
await recordAudit({ actor: req.currentUser!, category: "duplicati", action: "delete_source", targetType: "duplicati_source", targetId: id, detail: { name: before.name } });
|
||||
invalidateAlertsCache();
|
||||
res.status(204).end();
|
||||
}));
|
||||
|
||||
const backupSchema = z.object({ expectedHours: z.number().int().min(1).max(24 * 90).optional(), alert: z.boolean().optional() });
|
||||
|
||||
// How often a backup is expected, and whether it alerts: an operator's call, like the other backup settings.
|
||||
duplicatiRouter.patch("/backups/:id", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
const parsed = backupSchema.safeParse(req.body);
|
||||
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", message: "Hours must be between 1 and 2160." });
|
||||
const [before] = await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
await db.update(duplicatiBackups).set({ ...(parsed.data.expectedHours !== undefined ? { expectedHours: parsed.data.expectedHours } : {}), ...(parsed.data.alert !== undefined ? { alert: parsed.data.alert } : {}) }).where(eq(duplicatiBackups.id, id));
|
||||
await recordAudit({ actor: req.currentUser!, category: "duplicati", action: "update_backup", targetType: "duplicati_backup", targetId: id, detail: { name: before.name, ...parsed.data } });
|
||||
invalidateAlertsCache();
|
||||
res.json({ ok: true });
|
||||
}));
|
||||
|
||||
// A job that no longer exists: forgotten, with its history. (If it reports again it is learned again.)
|
||||
duplicatiRouter.delete("/backups/:id", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [before] = await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
await db.delete(duplicatiBackups).where(eq(duplicatiBackups.id, id));
|
||||
await recordAudit({ actor: req.currentUser!, category: "duplicati", action: "forget_backup", targetType: "duplicati_backup", targetId: id, detail: { name: before.name } });
|
||||
invalidateAlertsCache();
|
||||
res.status(204).end();
|
||||
}));
|
||||
Reference in new issue
Block a user