Duplicati: know when a backup failed or has gone quiet

Duplicati reports each operation to a secret address (its own send-http-url option), so
nothing calls Duplicati and no password for it is kept. Each machine is a source; each job
is learned by name and shows its last result, duration, size, changes and a history.
A failed run is a warning (critical from the second in a row); one that has not succeeded
for longer than expected (per job, 2 days by default) is a warning, critical at three
times that. Maintenance on the machine's server silences them; /metrics series added.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-24 21:36:13 +02:00
1 parent 12f2662465
commit 2f082ca5c2
23 files changed
+8047 -4

No files matched your search

+52
View File
@@ -0,0 +1,52 @@
CREATE TABLE `duplicati_backups` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`source_id` integer NOT NULL,
`name` text NOT NULL,
`last_report_at` text,
`last_result` text,
`last_begin` text,
`last_end` text,
`duration_seconds` integer,
`examined_files` integer,
`size_bytes` integer,
`added_files` integer,
`modified_files` integer,
`deleted_files` integer,
`warnings` integer DEFAULT 0 NOT NULL,
`errors` integer DEFAULT 0 NOT NULL,
`message` text,
`last_success_at` text,
`consecutive_failures` integer DEFAULT 0 NOT NULL,
`expected_hours` integer DEFAULT 48 NOT NULL,
`alert` integer DEFAULT true NOT NULL,
`created_at` text DEFAULT (current_timestamp) NOT NULL,
FOREIGN KEY (`source_id`) REFERENCES `duplicati_sources`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE UNIQUE INDEX `duplicati_backups_name_idx` ON `duplicati_backups` (`source_id`,`name`);--> statement-breakpoint
CREATE TABLE `duplicati_runs` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`backup_id` integer NOT NULL,
`at` text NOT NULL,
`result` text NOT NULL,
`duration_seconds` integer,
`size_bytes` integer,
`files_changed` integer,
`warnings` integer DEFAULT 0 NOT NULL,
`errors` integer DEFAULT 0 NOT NULL,
FOREIGN KEY (`backup_id`) REFERENCES `duplicati_backups`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE INDEX `duplicati_runs_idx` ON `duplicati_runs` (`backup_id`,`at`);--> statement-breakpoint
CREATE TABLE `duplicati_sources` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`name` text NOT NULL,
`token_hash` text NOT NULL,
`token_prefix` text NOT NULL,
`server_id` integer,
`created_by` text,
`created_at` text DEFAULT (current_timestamp) NOT NULL,
`last_report_at` text
);
--> statement-breakpoint
CREATE UNIQUE INDEX `duplicati_sources_token_hash_unique` ON `duplicati_sources` (`token_hash`);
File diff suppressed because it is too large. Load diff
+7
View File
@@ -393,6 +393,13 @@
"when": 1792612881951,
"tag": "0055_gorgeous_the_phantom",
"breakpoints": true
},
{
"idx": 56,
"version": "6",
"when": 1792869961357,
"tag": "0056_dizzy_carnage",
"breakpoints": true
}
]
}
+4
View File
@@ -57,6 +57,7 @@ import { alertActionsRouter } from "./routes/alertActions.js";
import { homeAssistantRouter } from "./routes/homeAssistant.js";
import { upsRouter } from "./routes/ups.js";
import { awayRouter } from "./routes/away.js";
import { duplicatiReportRouter, duplicatiRouter } from "./routes/duplicati.js";
import { serviceLevelsRouter } from "./routes/serviceLevels.js";
import { publicApiRouter } from "./routes/publicApi.js";
import { generatorRouter } from "./routes/generator.js";
@@ -95,6 +96,8 @@ export function createApp() {
app.use("/api/agent/report", agentReportRouter);
// A job reporting that it ran: a token in the address, no session, and no browser to be checked for where it came from.
app.use("/api/heartbeat", heartbeatPingRouter);
// Duplicati reporting how a backup went: a token in the address, no session, and no browser to be checked.
app.use("/api/duplicati-report", duplicatiReportRouter);
// The read API: a personal token in a header, GET only, no session and nothing for the same-origin check to protect.
app.use("/api/v1", publicApiRouter);
// What the buttons on a notification lead to: a signed token in the address, for named alerts only, no session.
@@ -189,6 +192,7 @@ export function createApp() {
app.use("/api/home-assistant", homeAssistantRouter);
app.use("/api/ups", upsRouter);
app.use("/api/away", awayRouter);
app.use("/api/duplicati", duplicatiRouter);
app.use("/api/service-levels", serviceLevelsRouter);
app.use("/api/todos", todosRouter);
app.use("/api/notes", notesRouter);
+69
View File
@@ -1159,6 +1159,75 @@ export const haReadings = sqliteTable(
(t) => [index("ha_readings_lookup_idx").on(t.integrationId, t.entityId, t.at)],
);
// ─── Duplicati ──────────────────────────────────────────────────────────────
// A Duplicati that reports to this app: one row per machine running it, with the secret token its reports carry in their address (only the hash is kept). Duplicati pushes a
// report after every operation (its own "send-http-url" option); services/duplicati.ts reads them.
export const duplicatiSources = sqliteTable("duplicati_sources", {
id: integer("id").primaryKey({ autoIncrement: true }),
name: text("name").notNull(),
tokenHash: text("token_hash").notNull().unique(),
tokenPrefix: text("token_prefix").notNull(),
serverId: integer("server_id"), // soft reference: the server it runs on, so that server's maintenance silences its alerts
createdBy: text("created_by"),
createdAt: text("created_at")
.notNull()
.default(sql`(current_timestamp)`),
lastReportAt: text("last_report_at"), // ISO
});
// One backup job of one source, as its last report left it. A job is learned from its first report (by name) and is expected to succeed every `expectedHours`.
export const duplicatiBackups = sqliteTable(
"duplicati_backups",
{
id: integer("id").primaryKey({ autoIncrement: true }),
sourceId: integer("source_id")
.notNull()
.references(() => duplicatiSources.id, { onDelete: "cascade" }),
name: text("name").notNull(),
lastReportAt: text("last_report_at"), // ISO
lastResult: text("last_result"), // Success | Warning | Error | Fatal
lastBegin: text("last_begin"),
lastEnd: text("last_end"),
durationSeconds: integer("duration_seconds"),
examinedFiles: integer("examined_files"),
sizeBytes: integer("size_bytes"), // what was examined: the size of the source
addedFiles: integer("added_files"),
modifiedFiles: integer("modified_files"),
deletedFiles: integer("deleted_files"),
warnings: integer("warnings").notNull().default(0),
errors: integer("errors").notNull().default(0),
message: text("message"), // the first error, or warning
lastSuccessAt: text("last_success_at"), // ISO; a backup with warnings still counts
consecutiveFailures: integer("consecutive_failures").notNull().default(0),
expectedHours: integer("expected_hours").notNull().default(48),
alert: integer("alert", { mode: "boolean" }).notNull().default(true),
createdAt: text("created_at")
.notNull()
.default(sql`(current_timestamp)`),
},
(t) => [uniqueIndex("duplicati_backups_name_idx").on(t.sourceId, t.name)],
);
// Each report of a backup, for its history: kept 400 days and at most 400 per job.
export const duplicatiRuns = sqliteTable(
"duplicati_runs",
{
id: integer("id").primaryKey({ autoIncrement: true }),
backupId: integer("backup_id")
.notNull()
.references(() => duplicatiBackups.id, { onDelete: "cascade" }),
at: text("at").notNull(), // ISO
result: text("result").notNull(),
durationSeconds: integer("duration_seconds"),
sizeBytes: integer("size_bytes"),
filesChanged: integer("files_changed"),
warnings: integer("warnings").notNull().default(0),
errors: integer("errors").notNull().default(0),
},
(t) => [index("duplicati_runs_idx").on(t.backupId, t.at)],
);
// ─── Service-level targets ──────────────────────────────────────────────────
// "99.9 % for Gitea": a target on an endpoint check, a server or an integration, measured over calendar months (services/serviceLevels.ts). The thing measured is a
+125
View File
@@ -0,0 +1,125 @@
import express, { Router } from "express";
import { desc, eq } from "drizzle-orm";
import { z } from "zod";
import { db } from "../db/client.js";
import { duplicatiBackups, duplicatiRuns, duplicatiSources, servers } from "../db/schema.js";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { invalidateAlertsCache } from "../services/alerts.js";
import { backupState, createSource, DEFAULT_EXPECTED_HOURS, duplicatiProblems, listAll, MAX_SOURCES, publicSource, recordReport } from "../services/duplicati.js";
import { asyncHandler } from "../utils/asyncHandler.js";
import { rateLimit } from "../utils/rateLimit.js";
/**
* The address Duplicati reports to: /api/duplicati-report/<token>. No session and no sign-in — the token is the whole of the access, and all it allows is to say how a backup
* went. Mounted ahead of the browser-only middleware (Duplicati is not a browser). A token that does not exist is answered like any other missing page, and the address is limited.
*/
export const duplicatiReportRouter = Router();
const limiter = rateLimit({ windowMs: 60_000, max: 120, message: "Too many requests. Wait a minute." });
// A report with its messages can be long; whatever it is, it is read as text.
const body = express.text({ type: () => true, limit: "2mb" });
duplicatiReportRouter.all("/:token", limiter, body, asyncHandler(async (req, res) => {
res.setHeader("Cache-Control", "no-store");
res.setHeader("X-Robots-Tag", "noindex, nofollow");
if (!["POST", "PUT"].includes(req.method)) return res.status(405).type("text/plain").send("Method not allowed");
const outcome = await recordReport(req.params.token, typeof req.body === "string" ? req.body : "");
if (!outcome.recorded && outcome.reason === "unknown_token") return res.status(404).type("text/plain").send("Not found");
invalidateAlertsCache();
res.type("text/plain").send(outcome.recorded ? "OK" : outcome.reason === "not_a_backup" ? "OK (not a backup: set aside)" : "OK (not a report this app can read)");
}));
export const duplicatiRouter = Router();
duplicatiRouter.use(requireAuth);
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
/** Every machine reporting, every backup job and where it stands: for everyone signed in, like the backup pages. */
duplicatiRouter.get("/", asyncHandler(async (_req, res) => {
const { sources, backups } = await listAll();
const now = Date.now();
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((s) => [s.id, s.name]));
const problems = duplicatiProblems(sources, backups, now);
res.json({
sources: sources.map((s) => ({ ...publicSource(s), serverName: s.serverId !== null ? (serverNames.get(s.serverId) ?? null) : null, backups: backups.filter((b) => b.sourceId === s.id).length })),
backups: backups.map((b) => ({ ...b, state: backupState(b, now), sourceName: sources.find((s) => s.id === b.sourceId)?.name ?? "", problem: problems.find((p) => new RegExp(`^duplicati-(failed|stale):${b.id}(:|$)`).test(p.key))?.message ?? null })),
servers: [...serverNames].map(([id, name]) => ({ id, name })).sort((a, b) => a.name.localeCompare(b.name)),
defaultExpectedHours: DEFAULT_EXPECTED_HOURS,
});
}));
duplicatiRouter.get("/backups/:id/runs", asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [b] = await db.select({ id: duplicatiBackups.id, name: duplicatiBackups.name }).from(duplicatiBackups).where(eq(duplicatiBackups.id, id)).limit(1);
if (!b) return res.status(404).json({ error: "not_found" });
const asked = Number(req.query.limit);
const limit = Number.isFinite(asked) ? Math.min(400, Math.max(1, Math.round(asked))) : 60;
res.json({ backup: b, runs: await db.select().from(duplicatiRuns).where(eq(duplicatiRuns.backupId, id)).orderBy(desc(duplicatiRuns.at), desc(duplicatiRuns.id)).limit(limit) });
}));
const sourceSchema = z.object({ name: z.string().trim().min(1).max(100), serverId: z.number().int().positive().nullable().optional() });
// Setting one up is an administrator's: it makes an address that can write to this app.
duplicatiRouter.post("/sources", requireRole("admin"), asyncHandler(async (req, res) => {
const parsed = sourceSchema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Give it a name." });
if ((await db.select({ id: duplicatiSources.id }).from(duplicatiSources)).length >= MAX_SOURCES) return res.status(409).json({ error: "too_many", message: `At most ${MAX_SOURCES} machines.` });
if (parsed.data.serverId != null && (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, parsed.data.serverId))).length === 0) return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
const { row, token } = await createSource({ name: parsed.data.name, serverId: parsed.data.serverId ?? null, createdBy: req.currentUser!.name ?? req.currentUser!.email ?? "someone" });
await recordAudit({ actor: req.currentUser!, category: "duplicati", action: "create_source", targetType: "duplicati_source", targetId: row.id, detail: { name: row.name } });
// The secret address is shown now and never again.
res.status(201).json({ source: publicSource(row), token, path: `/api/duplicati-report/${token}` });
}));
duplicatiRouter.patch("/sources/:id", requireRole("admin"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
const parsed = sourceSchema.safeParse(req.body);
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body" });
const [before] = await db.select().from(duplicatiSources).where(eq(duplicatiSources.id, id)).limit(1);
if (!before) return res.status(404).json({ error: "not_found" });
if (parsed.data.serverId != null && (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, parsed.data.serverId))).length === 0) return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
await db.update(duplicatiSources).set({ name: parsed.data.name, serverId: parsed.data.serverId ?? null }).where(eq(duplicatiSources.id, id));
await recordAudit({ actor: req.currentUser!, category: "duplicati", action: "update_source", targetType: "duplicati_source", targetId: id, detail: { name: parsed.data.name } });
invalidateAlertsCache();
res.json({ ok: true });
}));
duplicatiRouter.delete("/sources/:id", requireRole("admin"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [before] = await db.select().from(duplicatiSources).where(eq(duplicatiSources.id, id)).limit(1);
if (!before) return res.status(404).json({ error: "not_found" });
await db.delete(duplicatiSources).where(eq(duplicatiSources.id, id));
await recordAudit({ actor: req.currentUser!, category: "duplicati", action: "delete_source", targetType: "duplicati_source", targetId: id, detail: { name: before.name } });
invalidateAlertsCache();
res.status(204).end();
}));
const backupSchema = z.object({ expectedHours: z.number().int().min(1).max(24 * 90).optional(), alert: z.boolean().optional() });
// How often a backup is expected, and whether it alerts: an operator's call, like the other backup settings.
duplicatiRouter.patch("/backups/:id", requireRole("operator"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
const parsed = backupSchema.safeParse(req.body);
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", message: "Hours must be between 1 and 2160." });
const [before] = await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.id, id)).limit(1);
if (!before) return res.status(404).json({ error: "not_found" });
await db.update(duplicatiBackups).set({ ...(parsed.data.expectedHours !== undefined ? { expectedHours: parsed.data.expectedHours } : {}), ...(parsed.data.alert !== undefined ? { alert: parsed.data.alert } : {}) }).where(eq(duplicatiBackups.id, id));
await recordAudit({ actor: req.currentUser!, category: "duplicati", action: "update_backup", targetType: "duplicati_backup", targetId: id, detail: { name: before.name, ...parsed.data } });
invalidateAlertsCache();
res.json({ ok: true });
}));
// A job that no longer exists: forgotten, with its history. (If it reports again it is learned again.)
duplicatiRouter.delete("/backups/:id", requireRole("operator"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [before] = await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.id, id)).limit(1);
if (!before) return res.status(404).json({ error: "not_found" });
await db.delete(duplicatiBackups).where(eq(duplicatiBackups.id, id));
await recordAudit({ actor: req.currentUser!, category: "duplicati", action: "forget_backup", targetType: "duplicati_backup", targetId: id, detail: { name: before.name } });
invalidateAlertsCache();
res.status(204).end();
}));
+5 -1
View File
@@ -74,6 +74,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
homeassistant: { label: "Home Assistant", link: "/home-assistant" },
ups: { label: "UPS", link: "/ups" },
slo: { label: "Service level", link: "/service-levels" },
duplicati: { label: "Duplicati", link: "/duplicati" },
secrets: { label: "Secrets", link: "/secrets" },
domains: { label: "Domains", link: "/dns?view=domains" },
endpoint: { label: "Endpoint", link: "/endpoints" },
@@ -110,6 +111,8 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
if (m) return { type: "slo", id: Number(m[1]), category: "monitoring" };
m = /^container-(?:restarting|unhealthy|stopped|created|host):(\d+):/.exec(key);
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
m = /^duplicati-(?:failed|stale):(\d+)/.exec(key);
if (m) return { type: "duplicati", id: Number(m[1]), category: "backup" };
return { type: "server", id: null, category: "disk" };
}
@@ -232,12 +235,13 @@ export async function collectAlerts(): Promise<AlertsReport> {
type: origin.type,
message: c.message,
key: c.key,
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : undefined,
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : undefined,
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
});
}
// Integrations the check couldn't read this time. (A whole-integration entry covers its nodes, so skip those.)
if (held.has("server:rules")) cannotCheck("Alert rules", "couldn't be judged — see the server log", "Server and storage health");
if (held.has("duplicati")) cannotCheck("Duplicati backups", "couldn't be judged — see the server log", "Server and storage health");
if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health");
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
for (const h of held) {
+11
View File
@@ -43,6 +43,7 @@ const KIND_OF_CATEGORY: Record<string, ChangeKind> = {
alert_rule: "alerts",
service_level: "endpoints",
away: "settings",
duplicati: "backups",
todo: "other",
note: "servers",
webhook: "settings",
@@ -183,6 +184,16 @@ export function describeChange(category: string, action: string, detail: Record<
return { title: detail.change === "undo" ? "A change to several pieces of hardware was undone" : `${typeof detail.changed === "number" ? detail.changed : "Several"} piece${detail.changed === 1 ? "" : "s"} of hardware changed at once`, tone: "neutral", detail: genericSummary(detail, ["ids", "changed", "change"]) };
case "webhook/retry":
return { title: `A webhook message was sent again${name ? `: ${name}` : str(detail.label) ? `: ${str(detail.label)}` : ""}`, tone: detail.ok === false ? "bad" : "neutral", detail: str(detail.event) };
case "duplicati/create_source":
return { title: `Duplicati machine added${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
case "duplicati/update_source":
return { title: `Duplicati machine changed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
case "duplicati/delete_source":
return { title: `Duplicati machine removed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
case "duplicati/update_backup":
return { title: `Duplicati backup changed${name ? `: ${name}` : ""}`, tone: "neutral", detail: genericSummary(detail) };
case "duplicati/forget_backup":
return { title: `Duplicati backup forgotten${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
case "away/start":
return { title: `Away mode started${str(detail.note) ? `: ${str(detail.note)}` : ""}`, tone: "neutral", detail: `until ${str(detail.until)?.slice(0, 16).replace("T", " ") ?? "?"} UTC${detail.criticalThrough ? ", critical alerts still sent" : ", everything held"}` };
case "away/end":
+261
View File
@@ -0,0 +1,261 @@
/**
* Duplicati: whether the backups it makes are being made. Duplicati can tell a URL how each operation went (its own "send-http-url" option, with the report as JSON), so this
* is the other way round from most of the app: nothing here calls Duplicati, no password for it is kept, and it works on every version. Each machine running Duplicati is a
* *source* with a secret address; each report on that address is read, the backup job it is about is learned by name, and what it says — how it ended, how long it took, how big
* the source was, what changed — is kept with a history.
*
* What makes an alert: a backup whose last run **failed** (Error or Fatal; critical from the second failure in a row), and one that has **not succeeded for longer than it is
* expected to** (a day or two by default, set per job; critical at three times that). A run that finished with warnings still counts as a backup. Only runs of the *Backup*
* operation count: a compact, a repair or a verification does not say whether the data is safe.
*/
import { createHash, randomBytes } from "node:crypto";
import { and, asc, desc, eq, lt, notInArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { duplicatiBackups, duplicatiRuns, duplicatiSources } from "../db/schema.js";
export type SourceRow = typeof duplicatiSources.$inferSelect;
export type BackupRow = typeof duplicatiBackups.$inferSelect;
export const MAX_SOURCES = 50;
export const DEFAULT_EXPECTED_HOURS = 48;
export const KEEP_RUNS = 400;
export const KEEP_RUN_DAYS = 400;
const HOUR_MS = 3_600_000;
const DAY_MS = 86_400_000;
const hash = (token: string) => createHash("sha256").update(token).digest("hex");
// ─── Reading a report ───────────────────────────────────────────────────────
export interface ParsedReport {
operation: string;
/** Success, Warning, Error or Fatal (whatever Duplicati said, if it said something else). */
result: string;
backupName: string;
begin: string | null;
end: string | null;
durationSeconds: number | null;
examinedFiles: number | null;
sizeBytes: number | null;
addedFiles: number | null;
modifiedFiles: number | null;
deletedFiles: number | null;
warnings: number;
errors: number;
message: string | null;
}
/** "00:05:12.7" or "1.02:03:04" in seconds. */
export function parseDuration(text: unknown): number | null {
if (typeof text !== "string") return null;
const m = /^(?:(\d+)\.)?(\d+):(\d{2}):(\d{2})(?:\.\d+)?$/.exec(text.trim());
return m ? Number(m[1] ?? 0) * 86_400 + Number(m[2]) * 3600 + Number(m[3]) * 60 + Number(m[4]) : null;
}
const asInt = (v: unknown): number | null => {
const n = typeof v === "string" ? Number(v.replace(/,/g, "")) : v;
return typeof n === "number" && Number.isFinite(n) ? Math.round(n) : null;
};
const asIso = (v: unknown): string | null => {
if (typeof v !== "string") return null;
const t = Date.parse(v);
return Number.isFinite(t) ? new Date(t).toISOString() : null;
};
const obj = (v: unknown): Record<string, unknown> => (v !== null && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {});
const lineOf = (v: unknown): string | null => {
const text = typeof v === "string" ? v : v !== null && typeof v === "object" ? JSON.stringify(v) : null;
return text ? text.replace(/\s+/g, " ").trim().slice(0, 300) : null;
};
/**
* Pure: the report Duplicati sent — JSON (`send-http-result-output-format=Json`: the results at the top, or under `Data`, with the job's name among `Extra`), or the plain text it sends
* by default ("ParsedResult: Success" lines). Null when it is neither. Read defensively; the exact fields have not been checked against every Duplicati version.
*/
export function parseReport(body: unknown): ParsedReport | null {
let payload: Record<string, unknown> | null = null;
if (typeof body === "string") {
const trimmed = body.trim();
if (trimmed.startsWith("{")) {
try {
payload = obj(JSON.parse(trimmed));
} catch {
payload = null;
}
} else return parseText(trimmed);
} else payload = obj(body);
if (!payload || Object.keys(payload).length === 0) return null;
const data = Object.keys(obj(payload.Data)).length > 0 ? obj(payload.Data) : payload;
const extra = { ...obj(payload.Extra), ...obj(data.Extra) };
const result = typeof data.ParsedResult === "string" ? data.ParsedResult : null;
if (!result) return null;
const list = (v: unknown) => (Array.isArray(v) ? v : []);
const warnings = list(data.Warnings);
const errors = list(data.Errors);
const named = [extra["backup-name"], extra.backupName, extra.BackupName, data.BackupName].find((n) => typeof n === "string" && n.trim() !== "");
return {
operation: String(data.MainOperation ?? extra.OperationName ?? extra.operationname ?? extra.operation ?? "Backup"),
result,
backupName: typeof named === "string" ? named.trim().slice(0, 100) : typeof extra["backup-id"] === "string" ? `Backup ${extra["backup-id"]}`.slice(0, 100) : "Unnamed backup",
begin: asIso(data.BeginTime),
end: asIso(data.EndTime),
durationSeconds: parseDuration(data.Duration),
examinedFiles: asInt(data.ExaminedFiles),
sizeBytes: asInt(data.SizeOfExaminedFiles),
addedFiles: asInt(data.AddedFiles),
modifiedFiles: asInt(data.ModifiedFiles),
deletedFiles: asInt(data.DeletedFiles),
warnings: asInt(data.WarningsActualLength) ?? warnings.length,
errors: asInt(data.ErrorsActualLength) ?? errors.length,
message: lineOf(errors[0]) ?? lineOf(warnings[0]),
};
}
function parseText(text: string): ParsedReport | null {
const value = (key: string) => new RegExp(`^\\s*${key}\\s*:\\s*(.+?)\\s*$`, "im").exec(text)?.[1] ?? null;
const result = value("ParsedResult");
if (!result) return null;
const name = value("backup-name");
return {
operation: value("MainOperation") ?? value("OperationName") ?? "Backup",
result,
backupName: name ? name.slice(0, 100) : "Unnamed backup",
begin: null,
end: null,
durationSeconds: parseDuration(value("Duration")),
examinedFiles: asInt(value("ExaminedFiles")?.match(/^[\d,]+/)?.[0]),
sizeBytes: asInt(value("SizeOfExaminedFiles")?.match(/^[\d,]+/)?.[0]),
addedFiles: asInt(value("AddedFiles")?.match(/^[\d,]+/)?.[0]),
modifiedFiles: asInt(value("ModifiedFiles")?.match(/^[\d,]+/)?.[0]),
deletedFiles: asInt(value("DeletedFiles")?.match(/^[\d,]+/)?.[0]),
warnings: asInt(value("WarningsActualLength")?.match(/^[\d,]+/)?.[0]) ?? 0,
errors: asInt(value("ErrorsActualLength")?.match(/^[\d,]+/)?.[0]) ?? 0,
message: null,
};
}
const isGood = (result: string) => /^(success|warning)$/i.test(result);
// ─── What a report does ────────────────────────────────────────────────────
/** Makes a source and its secret address; the secret is shown once, and only its hash is kept. */
export async function createSource(params: { name: string; serverId: number | null; createdBy: string }): Promise<{ row: SourceRow; token: string }> {
const token = `hlm_d_${randomBytes(24).toString("base64url")}`;
const [row] = await db.insert(duplicatiSources).values({ name: params.name, tokenHash: hash(token), tokenPrefix: token.slice(0, 12), serverId: params.serverId, createdBy: params.createdBy }).returning();
return { row, token };
}
export async function sourceForToken(token: string): Promise<SourceRow | null> {
if (typeof token !== "string" || token.length < 20 || token.length > 128) return null;
return (await db.select().from(duplicatiSources).where(eq(duplicatiSources.tokenHash, hash(token))).limit(1))[0] ?? null;
}
export type RecordOutcome = { recorded: true; backupId: number; result: string } | { recorded: false; reason: "unknown_token" | "unreadable" | "not_a_backup" };
/** Reads a report that arrived on a secret address and writes it down. Other operations than a backup are said to have been seen and set aside. */
export async function recordReport(token: string, body: unknown, now: Date = new Date()): Promise<RecordOutcome> {
const source = await sourceForToken(token);
if (!source) return { recorded: false, reason: "unknown_token" };
const nowIso = now.toISOString();
await db.update(duplicatiSources).set({ lastReportAt: nowIso }).where(eq(duplicatiSources.id, source.id));
const report = parseReport(body);
if (!report) return { recorded: false, reason: "unreadable" };
if (!/^backup$/i.test(report.operation)) return { recorded: false, reason: "not_a_backup" };
const [before] = await db.select().from(duplicatiBackups).where(and(eq(duplicatiBackups.sourceId, source.id), eq(duplicatiBackups.name, report.backupName))).limit(1);
const good = isGood(report.result);
const at = report.end ?? nowIso;
const values = {
lastReportAt: nowIso,
lastResult: report.result,
lastBegin: report.begin,
lastEnd: report.end,
durationSeconds: report.durationSeconds,
examinedFiles: report.examinedFiles,
sizeBytes: report.sizeBytes,
addedFiles: report.addedFiles,
modifiedFiles: report.modifiedFiles,
deletedFiles: report.deletedFiles,
warnings: report.warnings,
errors: report.errors,
message: report.message,
lastSuccessAt: good ? at : (before?.lastSuccessAt ?? null),
consecutiveFailures: good ? 0 : (before?.consecutiveFailures ?? 0) + 1,
};
let id: number;
if (before) {
await db.update(duplicatiBackups).set(values).where(eq(duplicatiBackups.id, before.id));
id = before.id;
} else {
id = (await db.insert(duplicatiBackups).values({ sourceId: source.id, name: report.backupName, ...values }).returning({ id: duplicatiBackups.id }))[0].id;
}
const changed = (report.addedFiles ?? 0) + (report.modifiedFiles ?? 0) + (report.deletedFiles ?? 0);
await db.insert(duplicatiRuns).values({ backupId: id, at, result: report.result, durationSeconds: report.durationSeconds, sizeBytes: report.sizeBytes, filesChanged: report.addedFiles === null && report.modifiedFiles === null && report.deletedFiles === null ? null : changed, warnings: report.warnings, errors: report.errors });
await pruneRuns(id, now);
return { recorded: true, backupId: id, result: report.result };
}
async function pruneRuns(backupId: number, now: Date): Promise<void> {
await db.delete(duplicatiRuns).where(and(eq(duplicatiRuns.backupId, backupId), lt(duplicatiRuns.at, new Date(now.getTime() - KEEP_RUN_DAYS * DAY_MS).toISOString())));
const keep = await db.select({ id: duplicatiRuns.id }).from(duplicatiRuns).where(eq(duplicatiRuns.backupId, backupId)).orderBy(desc(duplicatiRuns.at), desc(duplicatiRuns.id)).limit(KEEP_RUNS);
if (keep.length === KEEP_RUNS) await db.delete(duplicatiRuns).where(and(eq(duplicatiRuns.backupId, backupId), notInArray(duplicatiRuns.id, keep.map((k) => k.id))));
}
// ─── Where a backup stands ──────────────────────────────────────────────────
export type BackupState = "ok" | "warning" | "failed" | "stale";
/** Pure: where a backup stands. A failed last run is "failed"; one that has not succeeded for longer than expected is "stale"; one with warnings is "warning". */
export function backupState(b: Pick<BackupRow, "lastResult" | "lastSuccessAt" | "expectedHours" | "createdAt">, nowMs: number): BackupState {
if (b.lastResult && !isGood(b.lastResult)) return "failed";
const since = b.lastSuccessAt ? Date.parse(b.lastSuccessAt) : NaN;
if (Number.isFinite(since) && nowMs - since > b.expectedHours * HOUR_MS) return "stale";
return b.lastResult && /^warning$/i.test(b.lastResult) ? "warning" : "ok";
}
const ago = (ms: number) => {
const h = Math.floor(ms / HOUR_MS);
return h < 1 ? "less than an hour" : h < 48 ? `${h} h` : `${Math.floor(h / 24)} days`;
};
export interface DuplicatiProblem {
key: string;
kind: "failed" | "stale";
severity: "critical" | "warning";
message: string;
}
/** Pure: what is wrong with the backups, for those that are to alert. The key carries the server a source runs on, so maintenance on it silences them. */
export function duplicatiProblems(sources: Pick<SourceRow, "id" | "name" | "serverId">[], backups: BackupRow[], nowMs: number): DuplicatiProblem[] {
const byId = new Map(sources.map((s) => [s.id, s]));
const out: DuplicatiProblem[] = [];
for (const b of backups) {
const source = byId.get(b.sourceId);
if (!source || !b.alert) continue;
const suffix = source.serverId !== null ? `:server:${source.serverId}` : "";
const state = backupState(b, nowMs);
const lastGood = b.lastSuccessAt ? `the last good one was ${ago(nowMs - Date.parse(b.lastSuccessAt))} ago` : "there has been no good one yet";
if (state === "failed") {
out.push({ key: `duplicati-failed:${b.id}${suffix}`, kind: "failed", severity: b.consecutiveFailures >= 2 ? "critical" : "warning", message: `The backup “${b.name}” on ${source.name} failed${b.message ? `: ${b.message}` : ""} (${lastGood})` });
} else if (state === "stale") {
const left = nowMs - Date.parse(b.lastSuccessAt as string);
out.push({ key: `duplicati-stale:${b.id}${suffix}`, kind: "stale", severity: left > 3 * b.expectedHours * HOUR_MS ? "critical" : "warning", message: `The backup “${b.name}” on ${source.name} hasn't succeeded for ${ago(left)} — it should run at least every ${b.expectedHours >= 48 && b.expectedHours % 24 === 0 ? `${b.expectedHours / 24} days` : `${b.expectedHours} h`}` });
}
}
return out;
}
export async function listAll(): Promise<{ sources: SourceRow[]; backups: BackupRow[] }> {
const [sources, backups] = await Promise.all([db.select().from(duplicatiSources).orderBy(asc(duplicatiSources.name)), db.select().from(duplicatiBackups).orderBy(asc(duplicatiBackups.name))]);
return { sources, backups };
}
/** What is wrong with the backups right now, from what is stored. */
export async function currentProblems(nowMs: number = Date.now()): Promise<DuplicatiProblem[]> {
const { sources, backups } = await listAll();
return sources.length === 0 ? [] : duplicatiProblems(sources, backups, nowMs);
}
/** A source as lists show it: everything but the hash. */
export function publicSource(row: SourceRow) {
return { id: row.id, name: row.name, tokenPrefix: row.tokenPrefix, serverId: row.serverId, createdBy: row.createdBy, createdAt: row.createdAt, lastReportAt: row.lastReportAt };
}
+14
View File
@@ -19,6 +19,7 @@ import { computeRuleFindings, findingKey, listRules, type RuleFinding } from "./
import { baselineKey, computeBaselineFindings, type BaselineFinding } from "./baselines.js";
import { levelFindings, levelKey, type LevelFinding } from "./serviceLevels.js";
import { createDockhandAdapter } from "../integrations/dockhand/adapter.js";
import { currentProblems as duplicatiProblemsNow, type DuplicatiProblem } from "./duplicati.js";
import { evaluateContainers, loadContainerMemory, saveContainerMemory, type ContainerMemory, type DockerSnapshot } from "./containerHealth.js";
import { recordStorageSamples } from "./storageHistory.js";
import { isUpsFresh, upsProblems, upsSyncStatuses, type UpsStateRow } from "./upsSync.js";
@@ -86,6 +87,8 @@ export interface HealthSnapshot {
baselines?: BaselineFinding[];
/** Service-level targets whose error budget for the month is mostly or wholly used (services/serviceLevels.ts). */
levels?: LevelFinding[];
/** Duplicati backups that failed or have not succeeded in time (services/duplicati.ts). */
duplicati?: DuplicatiProblem[];
/** Each Docker integration's containers as last read, with what was remembered of them (services/containerHealth.ts). */
docker?: DockerSnapshot[];
dockerMemory?: ContainerMemory;
@@ -195,6 +198,9 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
}
}
// Duplicati backups that failed, or have not succeeded for longer than they should.
for (const p of snapshot.duplicati ?? []) out.push({ key: p.key, source: "duplicati", message: p.message, severity: p.severity });
// Containers that are broken, stopped unexpectedly or never started, and Docker hosts that cannot be reached.
if (snapshot.docker && snapshot.docker.length > 0) {
for (const c of evaluateContainers(snapshot.docker, snapshot.dockerMemory ?? {}, snapshot.dockerNow ?? now, snapshot.dockerIgnore ?? []).conditions) out.push(c);
@@ -422,6 +428,14 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
}
}
// Duplicati: what its reports say, read from what is stored (nothing is asked of Duplicati). A failure here costs only this check.
try {
snapshot.duplicati = await duplicatiProblemsNow(Date.now());
} catch (err) {
console.error("[health] couldn't judge the Duplicati backups:", err instanceof Error ? err.message : err);
held.add("duplicati");
}
// Docker containers, from Dockhand: what is broken, what stopped unexpectedly. A Dockhand that cannot be read holds its containers' alerts rather than clearing them.
if (checks.containerAlerts) {
snapshot.dockerIgnore = checks.containerIgnore;
+2
View File
@@ -31,6 +31,8 @@ export function subjectOfConditionKey(key: string): string | null {
if (m) return `server:${m[1]}`;
m = /^container-(?:restarting|unhealthy|stopped|created|host):(\d+):/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^duplicati-(?:failed|stale):\d+:server:(\d+)$/.exec(key);
if (m) return `server:${m[1]}`;
m = /^(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:(\d+)$/.exec(key);
if (m) return `server:${m[1]}`;
m = /^offline:device:(\d+)$/.exec(key);
+15
View File
@@ -8,6 +8,7 @@ import { db } from "../db/client.js";
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState } from "../db/schema.js";
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
import { levelStatuses } from "./serviceLevels.js";
import { backupState, listAll as listDuplicati } from "./duplicati.js";
import { latestTests } from "./restoreTests.js";
import { heartbeatState, parseStamp } from "./heartbeats.js";
import { env } from "../env.js";
@@ -59,6 +60,19 @@ class Exposition {
}
}
/** Each Duplicati backup, for /metrics: whether it is in order, how long since it last succeeded, and how big what it backs up is. */
async function addDuplicatiMetrics(out: Exposition, now: Date) {
const { sources, backups } = await listDuplicati();
const names = new Map(sources.map((s) => [s.id, s.name]));
for (const b of backups) {
const labels = { backup: b.name, source: names.get(b.sourceId) ?? "" };
const state = backupState(b, now.getTime());
out.add("homelab_duplicati_backup_ok", "gauge", "1 if the Duplicati backup's last run worked and it is not overdue, else 0.", state === "ok" || state === "warning" ? 1 : 0, labels);
if (b.lastSuccessAt) out.add("homelab_duplicati_last_success_age_seconds", "gauge", "Seconds since the Duplicati backup last succeeded (a run with warnings counts).", Math.max(0, Math.round((now.getTime() - Date.parse(b.lastSuccessAt)) / 1000)), labels);
if (b.sizeBytes !== null) out.add("homelab_duplicati_source_size_bytes", "gauge", "The size of what the Duplicati backup examined, as its last report said.", b.sizeBytes, labels);
}
}
/** Each service-level target, for /metrics: its target, how it is doing this month and how much error budget is left. */
async function addServiceLevelMetrics(out: Exposition, now: Date) {
for (const s of await levelStatuses(now, { history: false })) {
@@ -252,6 +266,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
await addEndpointMetrics(out, now);
await addUpsMetrics(out, now);
await addServiceLevelMetrics(out, now);
await addDuplicatiMetrics(out, now);
await addHeartbeatMetrics(out, now);
return out.toString();
}
+343
View File
@@ -0,0 +1,343 @@
/**
* Duplicati's reports: reading them (JSON in both shapes, and the plain text), what a report does to a backup and its history, where a backup stands, what is an alert, and the
* address Duplicati posts to and the pages around it.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { Server } from "node:http";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-duplicati-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, duplicatiSources, duplicatiBackups, duplicatiRuns, auditLog } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const d = await import("../src/services/duplicati.js");
const { duplicatiReportRouter, duplicatiRouter } = await import("../src/routes/duplicati.js");
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
const { subjectOfConditionKey, startOrExtendWindow } = await import("../src/services/maintenance.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { buildMetrics } = await import("../src/services/metrics.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const HOUR = 3_600_000;
const DAY = 86_400_000;
/** A report as Duplicati sends it with the output format set to Json. */
const report = (over: Record<string, unknown> = {}, name = "Documents", extra: Record<string, unknown> = {}) =>
JSON.stringify({
Data: { DeletedFiles: 1, ModifiedFiles: 3, ExaminedFiles: 1520, AddedFiles: 2, SizeOfExaminedFiles: 3221225472, MainOperation: "Backup", ParsedResult: "Success", EndTime: "2026-10-23T03:35:44.1Z", BeginTime: "2026-10-23T03:30:31.4Z", Duration: "00:05:12.7", WarningsActualLength: 0, ErrorsActualLength: 0, Warnings: [], Errors: [], ...over },
Extra: { OperationName: "Backup", "machine-id": "abc", "backup-name": name, "backup-id": "DB-1", ...extra },
});
describe("reading a report", () => {
test("a duration, in either of Duplicati's forms", () => {
assert.equal(d.parseDuration("00:05:12.7"), 312);
assert.equal(d.parseDuration("1.02:03:04"), 86_400 + 2 * 3600 + 3 * 60 + 4);
assert.equal(d.parseDuration("12:00:00"), 43_200);
for (const bad of [null, "", "5 minutes", 312, undefined]) assert.equal(d.parseDuration(bad), null);
});
test("JSON with the results under Data and the job's name among Extra", () => {
const r = d.parseReport(report())!;
assert.deepEqual([r.operation, r.result, r.backupName, r.durationSeconds, r.examinedFiles, r.sizeBytes, r.addedFiles, r.modifiedFiles, r.deletedFiles, r.warnings, r.errors, r.message], ["Backup", "Success", "Documents", 312, 1520, 3221225472, 2, 3, 1, 0, 0, null]);
assert.deepEqual([r.begin, r.end], ["2026-10-23T03:30:31.400Z", "2026-10-23T03:35:44.100Z"]);
});
test("JSON with the results at the top, and a job known only by its id or by nothing", () => {
const top = d.parseReport({ MainOperation: "Backup", ParsedResult: "Warning", WarningsActualLength: 2, Warnings: ["file locked", "another"], Extra: { "backup-name": "Photos" } })!;
assert.deepEqual([top.backupName, top.result, top.warnings, top.message], ["Photos", "Warning", 2, "file locked"]);
assert.equal(d.parseReport(report({}, "", { "backup-name": "", "backup-id": "DB-7" }))!.backupName, "Backup DB-7");
assert.equal(d.parseReport(JSON.stringify({ ParsedResult: "Success" }))!.backupName, "Unnamed backup");
});
test("a failure says why, with the first error", () => {
const r = d.parseReport(report({ ParsedResult: "Fatal", ErrorsActualLength: 1, Errors: ["The backup storage destination is missing.\nIt may be offline"] }))!;
assert.deepEqual([r.result, r.errors, r.message], ["Fatal", 1, "The backup storage destination is missing. It may be offline"]);
});
test("the plain text Duplicati sends by default", () => {
const text = ["DeletedFiles: 0", "ModifiedFiles: 3", "ExaminedFiles: 1,520", "SizeOfExaminedFiles: 3221225472 (3.00 GB)", "MainOperation: Backup", "ParsedResult: Success", "Duration: 00:05:12.7", "WarningsActualLength: 0", "backup-name: Documents"].join("\n");
const r = d.parseReport(text)!;
assert.deepEqual([r.operation, r.result, r.backupName, r.durationSeconds, r.examinedFiles, r.sizeBytes, r.modifiedFiles], ["Backup", "Success", "Documents", 312, 1520, 3221225472, 3]);
});
test("what is not a report is not read", () => {
for (const nothing of ["", "hello", "{}", "{not json", JSON.stringify({ Data: { MainOperation: "Backup" } }), null, 5, []]) assert.equal(d.parseReport(nothing as never), null, String(nothing));
assert.equal(d.parseReport(report({ MainOperation: "Compact" }))?.operation, "Compact");
});
});
describe("what a report does", () => {
let token = "";
let sourceId = 0;
before(async () => {
const made = await d.createSource({ name: "Jackie", serverId: null, createdBy: "Ada" });
token = made.token;
sourceId = made.row.id;
});
const backup = async (name = "Documents") => (await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.name, name)))[0];
test("the secret is shown once and only its hash is kept", async () => {
assert.match(token, /^hlm_d_[\w-]{20,}$/);
const [row] = await db.select().from(duplicatiSources).where(eq(duplicatiSources.id, sourceId));
assert.equal(row.tokenPrefix, token.slice(0, 12));
assert.ok(!JSON.stringify(row).includes(token), "the token itself is nowhere in the row");
assert.equal((await d.sourceForToken(token))?.id, sourceId);
for (const bad of ["", "short", `${token}x`, "x".repeat(200)]) assert.equal(await d.sourceForToken(bad), null);
});
test("a first report teaches the app the job; what it says is kept", async () => {
const t0 = new Date("2026-10-23T04:00:00Z");
assert.deepEqual(await d.recordReport(token, report(), t0), { recorded: true, backupId: (await backup())?.id ?? 1, result: "Success" });
const b = await backup();
assert.deepEqual([b.name, b.lastResult, b.durationSeconds, b.sizeBytes, b.addedFiles, b.modifiedFiles, b.deletedFiles, b.consecutiveFailures, b.expectedHours, b.alert], ["Documents", "Success", 312, 3221225472, 2, 3, 1, 0, 48, true]);
assert.equal(b.lastSuccessAt, "2026-10-23T03:35:44.100Z", "when the backup finished, from its own report");
const [src] = await db.select().from(duplicatiSources).where(eq(duplicatiSources.id, sourceId));
assert.equal(src.lastReportAt, t0.toISOString());
const runs = await db.select().from(duplicatiRuns).where(eq(duplicatiRuns.backupId, b.id));
assert.deepEqual(runs.map((r) => [r.result, r.durationSeconds, r.filesChanged]), [["Success", 312, 6]]);
});
test("failures are counted in a row, the last good one is kept, and a good run (warnings included) ends the run of failures", async () => {
const fail = (end: string) => report({ ParsedResult: "Error", ErrorsActualLength: 1, Errors: ["Unable to connect to the backend"], EndTime: end });
await d.recordReport(token, fail("2026-10-24T03:35:00Z"), new Date("2026-10-24T03:36:00Z"));
let b = await backup();
assert.deepEqual([b.lastResult, b.consecutiveFailures, b.lastSuccessAt, b.message], ["Error", 1, "2026-10-23T03:35:44.100Z", "Unable to connect to the backend"]);
await d.recordReport(token, fail("2026-10-25T03:35:00Z"), new Date("2026-10-25T03:36:00Z"));
assert.equal((await backup()).consecutiveFailures, 2);
await d.recordReport(token, report({ ParsedResult: "Warning", WarningsActualLength: 1, Warnings: ["locked file"], EndTime: "2026-10-26T03:35:00Z" }), new Date("2026-10-26T03:36:00Z"));
b = await backup();
assert.deepEqual([b.lastResult, b.consecutiveFailures, b.lastSuccessAt, b.warnings], ["Warning", 0, "2026-10-26T03:35:00.000Z", 1]);
});
test("a compact, a repair or a test is set aside; an unreadable report and an unknown address are told apart", async () => {
const before = (await db.select().from(duplicatiRuns)).length;
assert.deepEqual(await d.recordReport(token, report({ MainOperation: "Compact" })), { recorded: false, reason: "not_a_backup" });
assert.deepEqual(await d.recordReport(token, "nonsense"), { recorded: false, reason: "unreadable" });
assert.deepEqual(await d.recordReport("hlm_d_not-a-real-token-at-all", report()), { recorded: false, reason: "unknown_token" });
assert.equal((await db.select().from(duplicatiRuns)).length, before, "nothing was written down");
});
test("two jobs on one machine are two backups; the same name on another machine is another", async () => {
await d.recordReport(token, report({}, "Photos"), new Date("2026-10-26T04:00:00Z"));
const other = await d.createSource({ name: "Nova", serverId: null, createdBy: "Ada" });
await d.recordReport(other.token, report({}, "Documents"), new Date("2026-10-26T04:00:00Z"));
const all = await db.select().from(duplicatiBackups);
assert.deepEqual(all.map((b) => `${b.sourceId}:${b.name}`).sort(), [`${sourceId}:Documents`, `${sourceId}:Photos`, `${other.row.id}:Documents`].sort());
});
test("a job's history is kept at most 400 reports long, and none older than 400 days", async () => {
const b = await backup("Photos");
const base = Date.parse("2026-10-26T00:00:00Z");
for (let i = 0; i < 405; i++) await db.insert(duplicatiRuns).values({ backupId: b.id, at: new Date(base - (i + 1) * HOUR).toISOString(), result: "Success", warnings: 0, errors: 0 });
await db.insert(duplicatiRuns).values({ backupId: b.id, at: new Date(base - 500 * DAY).toISOString(), result: "Success", warnings: 0, errors: 0 });
await d.recordReport(token, report({ EndTime: "2026-10-26T05:00:00Z" }, "Photos"), new Date("2026-10-26T05:01:00Z"));
const runs = await db.select().from(duplicatiRuns).where(eq(duplicatiRuns.backupId, b.id));
assert.equal(runs.length, 400);
assert.ok(runs.every((r) => Date.parse(r.at) > base - 400 * DAY));
});
});
describe("where a backup stands, and what is an alert", () => {
const row = (extra: Partial<import("../src/services/duplicati.js").BackupRow> = {}): import("../src/services/duplicati.js").BackupRow => ({
id: 1, sourceId: 1, name: "Documents", lastReportAt: null, lastResult: "Success", lastBegin: null, lastEnd: null, durationSeconds: null, examinedFiles: null, sizeBytes: null, addedFiles: null, modifiedFiles: null, deletedFiles: null,
warnings: 0, errors: 0, message: null, lastSuccessAt: new Date(NOW - 5 * HOUR).toISOString(), consecutiveFailures: 0, expectedHours: 48, alert: true, createdAt: "2026-10-01 00:00:00", ...extra,
});
const NOW = Date.parse("2026-10-24T12:00:00Z");
const iso = (hoursAgo: number) => new Date(NOW - hoursAgo * HOUR).toISOString();
test("ok, warning, failed and stale", () => {
assert.equal(d.backupState(row(), NOW), "ok");
assert.equal(d.backupState(row({ lastResult: "Warning" }), NOW), "warning");
assert.equal(d.backupState(row({ lastResult: "Error", consecutiveFailures: 1 }), NOW), "failed");
assert.equal(d.backupState(row({ lastResult: "Fatal" }), NOW), "failed");
assert.equal(d.backupState(row({ lastSuccessAt: iso(47) }), NOW), "ok", "inside what is expected");
assert.equal(d.backupState(row({ lastSuccessAt: iso(49) }), NOW), "stale");
assert.equal(d.backupState(row({ lastSuccessAt: iso(30), expectedHours: 24 }), NOW), "stale", "it is per job");
assert.equal(d.backupState(row({ lastResult: "Warning", lastSuccessAt: iso(60) }), NOW), "stale", "overdue beats warnings");
});
test("a failure is a warning, critical from the second in a row; a stale one is critical at three times what is expected", () => {
const sources = [{ id: 1, name: "Jackie", serverId: null }];
const p = (b: Partial<import("../src/services/duplicati.js").BackupRow>) => d.duplicatiProblems(sources, [row(b)], NOW);
assert.deepEqual(p({}), []);
const first = p({ lastResult: "Error", consecutiveFailures: 1, message: "Unable to connect to the backend", lastSuccessAt: iso(30) })[0];
assert.deepEqual([first.key, first.kind, first.severity], ["duplicati-failed:1", "failed", "warning"]);
assert.equal(first.message, "The backup “Documents” on Jackie failed: Unable to connect to the backend (the last good one was 30 h ago)");
assert.equal(p({ lastResult: "Error", consecutiveFailures: 2 })[0].severity, "critical");
assert.match(p({ lastResult: "Fatal", consecutiveFailures: 1, lastSuccessAt: null })[0].message, /there has been no good one yet/);
const stale = p({ lastSuccessAt: iso(72) })[0];
assert.deepEqual([stale.key, stale.severity], ["duplicati-stale:1", "warning"]);
assert.equal(stale.message, "The backup “Documents” on Jackie hasn't succeeded for 3 days — it should run at least every 2 days");
assert.equal(p({ lastSuccessAt: iso(150) })[0].severity, "critical");
assert.match(p({ lastSuccessAt: iso(30), expectedHours: 24 })[0].message, /every 24 h/);
});
test("one with its alert switched off says nothing; the server a machine runs on is in the key, so its maintenance can silence it", () => {
assert.deepEqual(d.duplicatiProblems([{ id: 1, name: "Jackie", serverId: null }], [row({ lastResult: "Error", alert: false })], NOW), []);
const [p] = d.duplicatiProblems([{ id: 1, name: "Jackie", serverId: 9 }], [row({ lastResult: "Error", consecutiveFailures: 1 })], NOW);
assert.equal(p.key, "duplicati-failed:1:server:9");
assert.equal(subjectOfConditionKey(p.key), "server:9");
assert.equal(subjectOfConditionKey("duplicati-stale:4:server:2"), "server:2");
assert.equal(subjectOfConditionKey("duplicati-failed:1"), null, "a machine that is not tied to a server is not silenced by one");
});
});
describe("the address Duplicati posts to, the pages, the alerts and /metrics", () => {
let app: Server;
let base = "";
let who = "admin";
let token = "";
let serverId = 0;
before(async () => {
// what the earlier groups made is not this group's: a clean start (a machine takes its backups and their history with it)
await db.delete(duplicatiSources);
await db.insert(users).values([
{ oidcSub: "admin", name: "Ada", role: "admin" },
{ oidcSub: "operator", name: "Olga", role: "operator" },
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
]);
serverId = (await db.insert(servers).values({ name: "Jackie", apiTokenHash: "j", apiTokenPrefix: "j", lastSeenAt: new Date().toISOString() }).returning({ id: servers.id }))[0].id;
const a = express();
// the address Duplicati posts to has no session and is mounted first, as in the app
a.use("/api/duplicati-report", duplicatiReportRouter);
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/duplicati", duplicatiRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
const call = (method: string, path: string, body?: unknown, headers: Record<string, string> = {}) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json", connection: "close", ...headers }, body: body === undefined ? undefined : typeof body === "string" ? body : JSON.stringify(body) });
test("an administrator makes a machine and is shown its secret address once; nobody else can, and it is in the audit log", async () => {
who = "operator";
assert.equal((await call("POST", "/duplicati/sources", { name: "Nope" })).status, 403);
who = "admin";
for (const bad of [{}, { name: " " }, { name: "x", serverId: 99999 }]) assert.equal((await call("POST", "/duplicati/sources", bad)).status, 400, JSON.stringify(bad));
const res = await call("POST", "/duplicati/sources", { name: "Jackie", serverId });
assert.equal(res.status, 201);
const made = (await res.json()) as { source: Record<string, unknown>; token: string; path: string };
token = made.token;
assert.equal(made.path, `/api/duplicati-report/${token}`);
assert.ok(!("tokenHash" in made.source));
const list = JSON.stringify(await (await call("GET", "/duplicati")).json());
assert.ok(!list.includes(token) && !list.includes("tokenHash"), "never shown again");
assert.equal((await db.select().from(auditLog).where(eq(auditLog.category, "duplicati"))).length, 1);
});
test("Duplicati posts to the address with no sign-in: JSON or text; a wrong address is not found, and a GET is refused", async () => {
const url = `/duplicati-report/${token}`;
const good = await call("POST", url, report({ EndTime: new Date().toISOString() }, "Documents"), { "content-type": "text/plain" });
assert.deepEqual([good.status, await good.text()], [200, "OK"]);
const text = await call("POST", url, ["MainOperation: Backup", "ParsedResult: Success", "Duration: 00:01:00", "backup-name: Photos"].join("\n"), { "content-type": "text/plain" });
assert.equal(text.status, 200);
const compact = await call("POST", url, report({ MainOperation: "Compact" }), { "content-type": "text/plain" });
assert.match(await compact.text(), /not a backup/);
assert.match(await (await call("POST", url, "rubbish", { "content-type": "text/plain" })).text(), /not a report this app can read/);
assert.equal((await call("POST", "/duplicati-report/hlm_d_this-token-does-not-exist-1234", report())).status, 404);
assert.equal((await call("GET", url)).status, 405);
assert.equal((await db.select().from(duplicatiBackups)).length, 2, "Documents and Photos");
});
test("anyone signed in sees each backup, where it stands and what is wrong; the history of one is there too", async () => {
who = "viewer";
await d.recordReport(token, report({ ParsedResult: "Fatal", ErrorsActualLength: 1, Errors: ["destination is missing"], EndTime: new Date().toISOString() }, "Photos"));
const body = (await (await call("GET", "/duplicati")).json()) as any;
assert.deepEqual(body.sources.map((s: any) => [s.name, s.serverName, s.backups]), [["Jackie", "Jackie", 2]]);
const photos = body.backups.find((b: any) => b.name === "Photos");
assert.deepEqual([photos.state, photos.sourceName, /destination is missing/.test(photos.problem)], ["failed", "Jackie", true]);
assert.equal(body.backups.find((b: any) => b.name === "Documents").state, "ok");
const runs = (await (await call("GET", `/duplicati/backups/${photos.id}/runs`)).json()) as any;
assert.deepEqual([runs.backup.name, runs.runs.length, runs.runs[0].result], ["Photos", 2, "Fatal"]);
assert.equal((await call("GET", "/duplicati/backups/999/runs")).status, 404);
who = "admin";
});
test("a failed backup is an alert in the Backups kind, with where to look; maintenance on its server silences it", async () => {
invalidateAlertsCache();
const report1 = await getAlerts(true);
const a = report1.alerts.find((x) => /^backup:duplicati-failed:\d+:server:/.test(x.id))!;
assert.ok(a, "the alert is there");
assert.deepEqual([a.severity, a.category, a.source, a.link, a.silenced], ["warning", "backup", "Duplicati", "/duplicati", false]);
assert.match(a.message, /^The backup “Photos” on Jackie failed: destination is missing/);
await startOrExtendWindow("server", serverId, 60, "test", "t");
invalidateAlertsCache();
assert.equal((await getAlerts(true)).alerts.find((x) => x.id === a.id)?.silenced, true);
const { snapshot } = await collectSnapshot();
assert.ok(snapshot.duplicati?.some((p) => p.kind === "failed"), "the health pass reads it too");
const list = evaluateHealth({ servers: [], proxmox: [], synology: [], duplicati: snapshot.duplicati }, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now());
assert.ok(list.some((c) => c.source === "duplicati" && c.key.startsWith("duplicati-failed:")));
});
test("an operator sets how often a backup is expected and whether it alerts; a viewer cannot, and a wrong number is refused", async () => {
const photos = (await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.name, "Photos")))[0];
who = "viewer";
assert.equal((await call("PATCH", `/duplicati/backups/${photos.id}`, { expectedHours: 24 })).status, 403);
who = "operator";
assert.equal((await call("PATCH", `/duplicati/backups/${photos.id}`, { expectedHours: 168, alert: false })).status, 200);
const after = (await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.id, photos.id)))[0];
assert.deepEqual([after.expectedHours, after.alert], [168, false]);
for (const bad of [{ expectedHours: 0 }, { expectedHours: 5000 }, { expectedHours: 1.5 }]) assert.equal((await call("PATCH", `/duplicati/backups/${photos.id}`, bad)).status, 400, JSON.stringify(bad));
assert.equal((await call("PATCH", "/duplicati/backups/999", { alert: true })).status, 404);
invalidateAlertsCache();
assert.ok(!(await getAlerts(true)).alerts.some((x) => x.id.includes("duplicati-failed")), "no alert with it switched off");
await call("PATCH", `/duplicati/backups/${photos.id}`, { alert: true });
who = "admin";
});
test("/metrics has each backup: in order or not, how long since it last worked, and how big it is", async () => {
const text = await buildMetrics();
assert.match(text, /homelab_duplicati_backup_ok\{backup="Documents",source="Jackie"\} 1/);
assert.match(text, /homelab_duplicati_backup_ok\{backup="Photos",source="Jackie"\} 0/);
assert.match(text, /homelab_duplicati_last_success_age_seconds\{backup="Documents",source="Jackie"\} \d+/);
assert.match(text, /homelab_duplicati_source_size_bytes\{backup="Documents",source="Jackie"\} 3221225472/);
});
test("a backup can be forgotten, and a machine removed with all of them; both are for people who may, and in the audit log", async () => {
const photos = (await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.name, "Photos")))[0];
who = "viewer";
assert.equal((await call("DELETE", `/duplicati/backups/${photos.id}`)).status, 403);
who = "operator";
assert.equal((await call("DELETE", `/duplicati/backups/${photos.id}`)).status, 204);
assert.equal((await db.select().from(duplicatiRuns).where(eq(duplicatiRuns.backupId, photos.id))).length, 0, "its history goes with it");
assert.equal((await call("DELETE", `/duplicati/sources/${(await db.select().from(duplicatiSources))[0].id}`)).status, 403, "removing a machine is an administrator's");
who = "admin";
const [src] = await db.select().from(duplicatiSources);
assert.equal((await call("PATCH", `/duplicati/sources/${src.id}`, { name: "Jackie (docker)", serverId })).status, 200);
assert.equal((await call("DELETE", `/duplicati/sources/${src.id}`)).status, 204);
assert.equal((await db.select().from(duplicatiBackups)).length, 0, "its backups go with it");
assert.equal((await call("POST", `/duplicati-report/${token}`, report(), { "content-type": "text/plain" })).status, 404, "and its address stops working");
const actions = (await db.select().from(auditLog).where(eq(auditLog.category, "duplicati"))).map((a) => a.action);
assert.deepEqual(actions.sort(), ["create_source", "delete_source", "forget_backup", "update_backup", "update_backup", "update_source"].sort());
});
});