Duplicati: flag warnings run after run, and forecast how much room backups take

A job whose last three runs in a row all finished with warnings is flagged and
raises an alert (on by default, switchable off per job). The size of the
destination, and its quota where the storage says, are read from the reports
and kept with each run; the Stored column shows the growth and, where the room
is known, when it will be full, with an alert inside the disk-forecast window.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-11-12 23:19:54 +01:00
1 parent 333146e8e3
commit 4f077d4f4a
15 files changed
+10115 -14

No files matched your search

+4
View File
@@ -1770,6 +1770,9 @@ export const duplicatiBackups = sqliteTable(
message: text("message"), // the first error, or warning
warningMessages: text("warning_messages"), // JSON string[]: what the last run's warnings said (up to 20); null: none
alertWarnings: integer("alert_warnings", { mode: "boolean" }).notNull().default(false), // raise an alert when the last run finished with warnings
warnStreakAlert: integer("warn_streak_alert", { mode: "boolean" }).notNull().default(true), // raise an alert when the last few runs in a row all finished with warnings
destinationBytes: integer("destination_bytes"), // how much the backup takes up where it is kept (Duplicati's KnownFileSize); null: not said
quotaBytes: integer("quota_bytes"), // the room the destination has, when its kind of storage says (TotalQuotaSpace); null: not said
lastSuccessAt: text("last_success_at"), // ISO; a backup with warnings still counts
consecutiveFailures: integer("consecutive_failures").notNull().default(0),
expectedHours: integer("expected_hours").notNull().default(48),
@@ -1797,6 +1800,7 @@ export const duplicatiRuns = sqliteTable(
warnings: integer("warnings").notNull().default(0),
errors: integer("errors").notNull().default(0),
warningMessages: text("warning_messages"), // JSON string[]: what this run's warnings said (up to 20); null: none
destinationBytes: integer("destination_bytes"), // how much the backup took up where it is kept after this run; null: not said
},
(t) => [index("duplicati_runs_idx").on(t.backupId, t.at)],
);
+10 -5
View File
@@ -6,7 +6,8 @@ import { duplicatiBackups, duplicatiRuns, duplicatiSources, servers } from "../d
import { requireAuth, requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { invalidateAlertsCache } from "../services/alerts.js";
import { backupState, createSource, DEFAULT_EXPECTED_HOURS, duplicatiProblems, listAll, MAX_SOURCES, publicSource, recordReport, unpackMessages } from "../services/duplicati.js";
import { analyseBackups, backupState, createSource, DEFAULT_EXPECTED_HOURS, duplicatiProblems, listAll, MAX_SOURCES, publicSource, recordReport, STREAK_RUNS, unpackMessages } from "../services/duplicati.js";
import { getSettings } from "../services/settingsStore.js";
import { asyncHandler } from "../utils/asyncHandler.js";
import { rateLimit } from "../utils/rateLimit.js";
@@ -40,12 +41,16 @@ duplicatiRouter.get("/", asyncHandler(async (req, res) => {
const { sources, backups } = await listAll();
const now = Date.now();
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((s) => [s.id, s.name]));
const problems = duplicatiProblems(sources, backups, now);
const byBackup = await analyseBackups(backups, new Date(now));
const forecastDays = (await getSettings()).healthChecks.diskForecastDays;
const problems = duplicatiProblems(sources, backups, now, { byBackup, forecastDays });
res.json({
sources: sources.map((s) => ({ ...publicSource(s, req.currentUser?.role === "admin"), serverName: s.serverId !== null ? (serverNames.get(s.serverId) ?? null) : null, backups: backups.filter((b) => b.sourceId === s.id).length })),
backups: backups.map((b) => ({ ...b, warningMessages: unpackMessages(b.warningMessages), state: backupState(b, now), sourceName: sources.find((s) => s.id === b.sourceId)?.name ?? "", problem: problems.find((p) => new RegExp(`^duplicati-(failed|stale|warnings):${b.id}(:|$)`).test(p.key))?.message ?? null })),
backups: backups.map((b) => ({ ...b, warningMessages: unpackMessages(b.warningMessages), state: backupState(b, now), sourceName: sources.find((s) => s.id === b.sourceId)?.name ?? "", problem: problems.find((p) => new RegExp(`^duplicati-(failed|stale|warnings|streak|growth):${b.id}(:|$)`).test(p.key))?.message ?? null, warningStreak: byBackup.get(b.id)?.streak ?? 0, growth: byBackup.get(b.id)?.growth ?? null })),
servers: [...serverNames].map(([id, name]) => ({ id, name })).sort((a, b) => a.name.localeCompare(b.name)),
defaultExpectedHours: DEFAULT_EXPECTED_HOURS,
streakRuns: STREAK_RUNS,
forecastDays,
});
}));
@@ -98,7 +103,7 @@ duplicatiRouter.delete("/sources/:id", requireRole("admin"), asyncHandler(async
res.status(204).end();
}));
const backupSchema = z.object({ expectedHours: z.number().int().min(1).max(24 * 90).optional(), alert: z.boolean().optional(), alertWarnings: z.boolean().optional() });
const backupSchema = z.object({ expectedHours: z.number().int().min(1).max(24 * 90).optional(), alert: z.boolean().optional(), alertWarnings: z.boolean().optional(), warnStreakAlert: z.boolean().optional() });
// How often a backup is expected, and whether it alerts: an operator's call, like the other backup settings.
duplicatiRouter.patch("/backups/:id", requireRole("operator"), asyncHandler(async (req, res) => {
@@ -107,7 +112,7 @@ duplicatiRouter.patch("/backups/:id", requireRole("operator"), asyncHandler(asyn
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", message: "Hours must be between 1 and 2160." });
const [before] = await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.id, id)).limit(1);
if (!before) return res.status(404).json({ error: "not_found" });
await db.update(duplicatiBackups).set({ ...(parsed.data.expectedHours !== undefined ? { expectedHours: parsed.data.expectedHours } : {}), ...(parsed.data.alert !== undefined ? { alert: parsed.data.alert } : {}), ...(parsed.data.alertWarnings !== undefined ? { alertWarnings: parsed.data.alertWarnings } : {}) }).where(eq(duplicatiBackups.id, id));
await db.update(duplicatiBackups).set({ ...(parsed.data.expectedHours !== undefined ? { expectedHours: parsed.data.expectedHours } : {}), ...(parsed.data.alert !== undefined ? { alert: parsed.data.alert } : {}), ...(parsed.data.alertWarnings !== undefined ? { alertWarnings: parsed.data.alertWarnings } : {}), ...(parsed.data.warnStreakAlert !== undefined ? { warnStreakAlert: parsed.data.warnStreakAlert } : {}) }).where(eq(duplicatiBackups.id, id));
await recordAudit({ actor: req.currentUser!, category: "duplicati", action: "update_backup", targetType: "duplicati_backup", targetId: id, detail: { name: before.name, ...parsed.data } });
invalidateAlertsCache();
res.json({ ok: true });
+1 -1
View File
@@ -156,7 +156,7 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
if (m) return { type: m[1], id: Number(m[3]), category: m[2] === "disk" ? "disk" : "services" };
m = /^fragnet-(disk|memory|backup):(\d+):/.exec(key);
if (m) return { type: "fragnet", id: Number(m[2]), category: m[1] === "disk" ? "disk" : m[1] === "backup" ? "backup" : "monitoring" };
m = /^duplicati-(?:failed|stale|warnings):(\d+)/.exec(key);
m = /^duplicati-(?:failed|stale|warnings|streak|growth):(\d+)/.exec(key);
if (m) return { type: "duplicati", id: Number(m[1]), category: "backup" };
m = /^nextcloud-(?:maintenance|upgrade|disk|cron|checks):(\d+)/.exec(key);
if (m) return { type: "nextcloud", id: Number(m[1]), category: key.startsWith("nextcloud-disk") ? "disk" : "services" };
+130 -5
View File
@@ -8,11 +8,16 @@
* expected to** (a day or two by default, set per job; critical at three times that). A run that finished with warnings still counts as a backup — and says what the warnings were (the first
* twenty, kept with the run) — and a job can be set to **alert when its last run had warnings** (off until switched on: some jobs always have a few). Only runs of the *Backup*
* operation count: a compact, a repair or a verification does not say whether the data is safe.
*
* Two slower things are watched as well: a job whose **last few runs in a row all finished with warnings** (one run with warnings is common; three in a row is a job nobody is looking at — on by
* default, and a job that always has a few can be told not to), and **how much room the backups take where they are kept** (Duplicati says how big the destination is, and for some kinds of
* storage how much room it has): the growth is worked out from the history, and a destination with a known size that will be full within the disk-forecast window is an alert.
*/
import { createHash, randomBytes } from "node:crypto";
import { and, asc, desc, eq, lt, notInArray } from "drizzle-orm";
import { and, asc, desc, eq, inArray, lt, notInArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { duplicatiBackups, duplicatiRuns, duplicatiSources } from "../db/schema.js";
import { getSettings } from "./settingsStore.js";
export type SourceRow = typeof duplicatiSources.$inferSelect;
export type BackupRow = typeof duplicatiBackups.$inferSelect;
@@ -43,6 +48,10 @@ export interface ParsedReport {
deletedFiles: number | null;
warnings: number;
errors: number;
/** How much the backup takes up where it is kept (Duplicati's BackendStatistics.KnownFileSize); null when the report does not say. */
destinationBytes: number | null;
/** The room the destination has (TotalQuotaSpace), when its kind of storage says — zero and negative mean "unknown" and are null. */
quotaBytes: number | null;
message: string | null;
/** What the warnings said, one line each (up to MAX_WARNING_MESSAGES, each cut at MAX_WARNING_LENGTH). Only the JSON format carries them. */
warningMessages: string[];
@@ -174,11 +183,17 @@ export function parseReport(body: unknown): ParsedReport | null {
deletedFiles: asInt(data.DeletedFiles),
warnings: asInt(data.WarningsActualLength) ?? warnings.length,
errors: asInt(data.ErrorsActualLength) ?? errors.length,
...destinationOf(asInt(obj(data.BackendStatistics).KnownFileSize), asInt(obj(data.BackendStatistics).TotalQuotaSpace)),
message: lineOf(errors[0]) ?? lineOf(warnings[0]),
warningMessages: messagesOf(warnings),
};
}
/** Pure: the size of the destination and the room it has, from what Duplicati said; a size below zero or a quota of zero or less is "not said". */
export function destinationOf(known: number | null, quota: number | null): { destinationBytes: number | null; quotaBytes: number | null } {
return { destinationBytes: known !== null && known >= 0 ? known : null, quotaBytes: quota !== null && quota > 0 ? quota : null };
}
function parseText(text: string, header: MessageHeader | null = null): ParsedReport | null {
const value = (key: string) => new RegExp(`^\\s*${key}\\s*:\\s*(.+?)\\s*$`, "im").exec(text)?.[1] ?? null;
const result = value("ParsedResult");
@@ -200,6 +215,7 @@ function parseText(text: string, header: MessageHeader | null = null): ParsedRep
deletedFiles: asInt(value("DeletedFiles")?.match(/^[\d,]+/)?.[0]),
warnings: asInt(value("WarningsActualLength")?.match(/^[\d,]+/)?.[0]) ?? warningLines.length,
errors: asInt(value("ErrorsActualLength")?.match(/^[\d,]+/)?.[0]) ?? 0,
...destinationOf(asInt(value("KnownFileSize")?.match(/^-?[\d,]+/)?.[0]), asInt(value("TotalQuotaSpace")?.match(/^-?[\d,]+/)?.[0])),
message: lineOf(errorLines[0]) ?? lineOf(warningLines[0]),
warningMessages: warningLines,
};
@@ -322,6 +338,9 @@ export async function recordReport(token: string, body: unknown, now: Date = new
deletedFiles: report.deletedFiles,
warnings: report.warnings,
errors: report.errors,
// What a report does not say is left as it was (a run of another kind, or an older Duplicati, must not wipe what was known).
destinationBytes: report.destinationBytes ?? before?.destinationBytes ?? null,
quotaBytes: report.quotaBytes ?? before?.quotaBytes ?? null,
message: report.message,
warningMessages: packMessages(report.warningMessages),
lastSuccessAt: good ? at : (before?.lastSuccessAt ?? null),
@@ -335,7 +354,7 @@ export async function recordReport(token: string, body: unknown, now: Date = new
id = (await db.insert(duplicatiBackups).values({ sourceId: source.id, name: report.backupName, ...values }).returning({ id: duplicatiBackups.id }))[0].id;
}
const changed = (report.addedFiles ?? 0) + (report.modifiedFiles ?? 0) + (report.deletedFiles ?? 0);
await db.insert(duplicatiRuns).values({ backupId: id, at, result: report.result, durationSeconds: report.durationSeconds, sizeBytes: report.sizeBytes, filesChanged: report.addedFiles === null && report.modifiedFiles === null && report.deletedFiles === null ? null : changed, warnings: report.warnings, errors: report.errors, warningMessages: packMessages(report.warningMessages) });
await db.insert(duplicatiRuns).values({ backupId: id, at, result: report.result, durationSeconds: report.durationSeconds, sizeBytes: report.sizeBytes, filesChanged: report.addedFiles === null && report.modifiedFiles === null && report.deletedFiles === null ? null : changed, warnings: report.warnings, errors: report.errors, warningMessages: packMessages(report.warningMessages), destinationBytes: report.destinationBytes });
await pruneRuns(id, now);
return { recorded: true, backupId: id, result: report.result };
}
@@ -358,6 +377,97 @@ export function backupState(b: Pick<BackupRow, "lastResult" | "lastSuccessAt" |
return b.lastResult && /^warning$/i.test(b.lastResult) ? "warning" : "ok";
}
// ─── Warnings in a row, and growth ──────────────────────────────────────────
/** How many runs in a row, the newest, must have finished with warnings for it to be said. */
export const STREAK_RUNS = 3;
/** Pure: how many of the newest runs in a row finished with warnings (the results newest first); a success or a failure ends it. */
export function warningStreak(resultsNewestFirst: string[]): number {
let n = 0;
for (const r of resultsNewestFirst) {
if (!/^warning$/i.test(r)) break;
n++;
}
return n;
}
/** What a growth forecast needs before it is worth saying: enough readings, over long enough, that a straight line describes. */
export const GROWTH_MIN_POINTS = 5;
export const GROWTH_MIN_SPAN_DAYS = 7;
export const GROWTH_MIN_FIT = 0.8;
export interface Growth {
/** Bytes a day, from a straight line through the readings (negative: shrinking). */
perDay: number;
/** The same over thirty days. */
perMonth: number;
/** How well a straight line describes them, 0 to 1 (R²). */
fit: number;
spanDays: number;
/** Where it stands now. */
latest: number;
/** For a destination with a known size that is growing: days until it is full at that rate; null otherwise. */
daysToFull: number | null;
}
/** Pure: how a backup's size has grown, from (time, bytes) readings; null without enough of them. */
export function growthOf(points: { t: number; v: number }[], quotaBytes: number | null): Growth | null {
const p = [...points].sort((a, b) => a.t - b.t);
if (p.length < GROWTH_MIN_POINTS) return null;
const span = p[p.length - 1].t - p[0].t;
if (span < GROWTH_MIN_SPAN_DAYS * DAY_MS) return null;
const xs = p.map((q) => (q.t - p[0].t) / DAY_MS);
const ys = p.map((q) => q.v);
const n = p.length;
const mx = xs.reduce((a, b) => a + b, 0) / n;
const my = ys.reduce((a, b) => a + b, 0) / n;
let num = 0;
let den = 0;
let total = 0;
for (let i = 0; i < n; i++) {
num += (xs[i] - mx) * (ys[i] - my);
den += (xs[i] - mx) ** 2;
total += (ys[i] - my) ** 2;
}
if (den === 0) return null;
const slope = num / den;
const fit = total === 0 ? 0 : Math.round(((num * num) / (den * total)) * 100) / 100;
const latest = ys[n - 1];
return {
perDay: Math.round(slope),
perMonth: Math.round(slope * 30),
fit,
spanDays: Math.round((span / DAY_MS) * 10) / 10,
latest,
daysToFull: quotaBytes !== null && slope > 0 && latest < quotaBytes ? Math.ceil((quotaBytes - latest) / slope) : null,
};
}
/** What is worked out from each job's runs, for the alerts and the page. */
export interface BackupAnalysis {
streak: number;
growth: Growth | null;
}
/** The runs of these jobs, newest first, worked into the streak of warnings and the growth of the destination. One query for all of them. */
export async function analyseBackups(backups: Pick<BackupRow, "id" | "quotaBytes">[], now: Date = new Date()): Promise<Map<number, BackupAnalysis>> {
const out = new Map<number, BackupAnalysis>();
if (backups.length === 0) return out;
const since = new Date(now.getTime() - 180 * DAY_MS).toISOString();
const runs = await db
.select({ backupId: duplicatiRuns.backupId, at: duplicatiRuns.at, result: duplicatiRuns.result, destinationBytes: duplicatiRuns.destinationBytes })
.from(duplicatiRuns)
.where(and(inArray(duplicatiRuns.backupId, backups.map((b) => b.id)), lt(duplicatiRuns.at, new Date(now.getTime() + DAY_MS).toISOString())))
.orderBy(desc(duplicatiRuns.at), desc(duplicatiRuns.id));
for (const b of backups) {
const mine = runs.filter((r) => r.backupId === b.id);
const points = mine.filter((r) => r.destinationBytes !== null && r.at >= since).map((r) => ({ t: Date.parse(r.at), v: r.destinationBytes as number }));
out.set(b.id, { streak: warningStreak(mine.slice(0, 50).map((r) => r.result)), growth: growthOf(points, b.quotaBytes) });
}
return out;
}
const ago = (ms: number) => {
const h = Math.floor(ms / HOUR_MS);
return h < 1 ? "less than an hour" : h < 48 ? `${h} h` : `${Math.floor(h / 24)} days`;
@@ -365,13 +475,13 @@ const ago = (ms: number) => {
export interface DuplicatiProblem {
key: string;
kind: "failed" | "stale" | "warnings";
kind: "failed" | "stale" | "warnings" | "streak" | "growth";
severity: "critical" | "warning";
message: string;
}
/** Pure: what is wrong with the backups, for those that are to alert. The key carries the server a source runs on, so maintenance on it silences them. */
export function duplicatiProblems(sources: Pick<SourceRow, "id" | "name" | "serverId">[], backups: BackupRow[], nowMs: number): DuplicatiProblem[] {
export function duplicatiProblems(sources: Pick<SourceRow, "id" | "name" | "serverId">[], backups: BackupRow[], nowMs: number, analysis?: { byBackup: Map<number, BackupAnalysis>; forecastDays: number }): DuplicatiProblem[] {
const byId = new Map(sources.map((s) => [s.id, s]));
const out: DuplicatiProblem[] = [];
for (const b of backups) {
@@ -391,6 +501,18 @@ export function duplicatiProblems(sources: Pick<SourceRow, "id" | "name" | "serv
const more = b.warnings > 1 ? ` (and ${b.warnings - 1} more)` : "";
out.push({ key: `duplicati-warnings:${b.id}${suffix}`, kind: "warnings", severity: "warning", message: `The backup “${b.name}” on ${source.name} finished with ${b.warnings} warning${b.warnings === 1 ? "" : "s"}${first ? `: ${first.slice(0, 200)}${more}` : ""}` });
}
const a = analysis?.byBackup.get(b.id);
if (!a) continue;
// Warnings run after run: said once, when no alert is already raised for each warning run (that one says it every time).
if (state === "warning" && b.warnStreakAlert && !b.alertWarnings && a.streak >= STREAK_RUNS) {
const first = unpackMessages(b.warningMessages)[0] ?? b.message;
out.push({ key: `duplicati-streak:${b.id}${suffix}`, kind: "streak", severity: "warning", message: `The backup “${b.name}” on ${source.name} has finished with warnings ${a.streak} runs in a row${first ? ` — the last said: ${first.slice(0, 200)}` : ""}` });
}
// A destination that will be full soon: only where its size is known, and only when the growth is steady enough to forecast from.
const g = a.growth;
if (g && b.quotaBytes !== null && g.daysToFull !== null && analysis && analysis.forecastDays > 0 && g.daysToFull <= analysis.forecastDays && g.fit >= GROWTH_MIN_FIT) {
out.push({ key: `duplicati-growth:${b.id}${suffix}`, kind: "growth", severity: g.daysToFull <= 3 ? "critical" : "warning", message: `The backup “${b.name}” on ${source.name} is growing ${Math.round(g.perDay / 1e6)} MB a day and its storage will be full in about ${g.daysToFull} day${g.daysToFull === 1 ? "" : "s"}` });
}
}
return out;
}
@@ -403,7 +525,10 @@ export async function listAll(): Promise<{ sources: SourceRow[]; backups: Backup
/** What is wrong with the backups right now, from what is stored. */
export async function currentProblems(nowMs: number = Date.now()): Promise<DuplicatiProblem[]> {
const { sources, backups } = await listAll();
return sources.length === 0 ? [] : duplicatiProblems(sources, backups, nowMs);
if (sources.length === 0) return [];
const byBackup = await analyseBackups(backups, new Date(nowMs));
const forecastDays = (await getSettings()).healthChecks.diskForecastDays;
return duplicatiProblems(sources, backups, nowMs, { byBackup, forecastDays });
}
/** A source as lists show it: everything but the hash. */
+1 -1
View File
@@ -69,7 +69,7 @@ export function subjectOfConditionKey(key: string): string | null {
if (m) return `integration:${m[1]}`;
m = /^authentik-(?:outpost|outpost-version|cert|token|worker|logins|errors):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^duplicati-(?:failed|stale|warnings):\d+:server:(\d+)$/.exec(key);
m = /^duplicati-(?:failed|stale|warnings|streak|growth):\d+:server:(\d+)$/.exec(key);
if (m) return `server:${m[1]}`;
m = /^(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:(\d+)$/.exec(key);
if (m) return `server:${m[1]}`;