Duplicati: flag warnings run after run, and forecast how much room backups take
A job whose last three runs in a row all finished with warnings is flagged and raises an alert (on by default, switchable off per job). The size of the destination, and its quota where the storage says, are read from the reports and kept with each run; the Stored column shows the growth and, where the room is known, when it will be full, with an alert inside the disk-forecast window. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
333146e8e3
commit
4f077d4f4a
15 files changed
+10115
-14
No files matched your search
@@ -0,0 +1,4 @@
|
||||
ALTER TABLE `duplicati_backups` ADD `warn_streak_alert` integer DEFAULT true NOT NULL;--> statement-breakpoint
|
||||
ALTER TABLE `duplicati_backups` ADD `destination_bytes` integer;--> statement-breakpoint
|
||||
ALTER TABLE `duplicati_backups` ADD `quota_bytes` integer;--> statement-breakpoint
|
||||
ALTER TABLE `duplicati_runs` ADD `destination_bytes` integer;
|
||||
File diff suppressed because it is too large.
Load diff
@@ -680,6 +680,13 @@
|
||||
"when": 1794520424972,
|
||||
"tag": "0096_wall_displays",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 97,
|
||||
"version": "6",
|
||||
"when": 1794521526082,
|
||||
"tag": "0097_duplicati_followups",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1770,6 +1770,9 @@ export const duplicatiBackups = sqliteTable(
|
||||
message: text("message"), // the first error, or warning
|
||||
warningMessages: text("warning_messages"), // JSON string[]: what the last run's warnings said (up to 20); null: none
|
||||
alertWarnings: integer("alert_warnings", { mode: "boolean" }).notNull().default(false), // raise an alert when the last run finished with warnings
|
||||
warnStreakAlert: integer("warn_streak_alert", { mode: "boolean" }).notNull().default(true), // raise an alert when the last few runs in a row all finished with warnings
|
||||
destinationBytes: integer("destination_bytes"), // how much the backup takes up where it is kept (Duplicati's KnownFileSize); null: not said
|
||||
quotaBytes: integer("quota_bytes"), // the room the destination has, when its kind of storage says (TotalQuotaSpace); null: not said
|
||||
lastSuccessAt: text("last_success_at"), // ISO; a backup with warnings still counts
|
||||
consecutiveFailures: integer("consecutive_failures").notNull().default(0),
|
||||
expectedHours: integer("expected_hours").notNull().default(48),
|
||||
@@ -1797,6 +1800,7 @@ export const duplicatiRuns = sqliteTable(
|
||||
warnings: integer("warnings").notNull().default(0),
|
||||
errors: integer("errors").notNull().default(0),
|
||||
warningMessages: text("warning_messages"), // JSON string[]: what this run's warnings said (up to 20); null: none
|
||||
destinationBytes: integer("destination_bytes"), // how much the backup took up where it is kept after this run; null: not said
|
||||
},
|
||||
(t) => [index("duplicati_runs_idx").on(t.backupId, t.at)],
|
||||
);
|
||||
|
||||
@@ -6,7 +6,8 @@ import { duplicatiBackups, duplicatiRuns, duplicatiSources, servers } from "../d
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { backupState, createSource, DEFAULT_EXPECTED_HOURS, duplicatiProblems, listAll, MAX_SOURCES, publicSource, recordReport, unpackMessages } from "../services/duplicati.js";
|
||||
import { analyseBackups, backupState, createSource, DEFAULT_EXPECTED_HOURS, duplicatiProblems, listAll, MAX_SOURCES, publicSource, recordReport, STREAK_RUNS, unpackMessages } from "../services/duplicati.js";
|
||||
import { getSettings } from "../services/settingsStore.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
import { rateLimit } from "../utils/rateLimit.js";
|
||||
|
||||
@@ -40,12 +41,16 @@ duplicatiRouter.get("/", asyncHandler(async (req, res) => {
|
||||
const { sources, backups } = await listAll();
|
||||
const now = Date.now();
|
||||
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((s) => [s.id, s.name]));
|
||||
const problems = duplicatiProblems(sources, backups, now);
|
||||
const byBackup = await analyseBackups(backups, new Date(now));
|
||||
const forecastDays = (await getSettings()).healthChecks.diskForecastDays;
|
||||
const problems = duplicatiProblems(sources, backups, now, { byBackup, forecastDays });
|
||||
res.json({
|
||||
sources: sources.map((s) => ({ ...publicSource(s, req.currentUser?.role === "admin"), serverName: s.serverId !== null ? (serverNames.get(s.serverId) ?? null) : null, backups: backups.filter((b) => b.sourceId === s.id).length })),
|
||||
backups: backups.map((b) => ({ ...b, warningMessages: unpackMessages(b.warningMessages), state: backupState(b, now), sourceName: sources.find((s) => s.id === b.sourceId)?.name ?? "", problem: problems.find((p) => new RegExp(`^duplicati-(failed|stale|warnings):${b.id}(:|$)`).test(p.key))?.message ?? null })),
|
||||
backups: backups.map((b) => ({ ...b, warningMessages: unpackMessages(b.warningMessages), state: backupState(b, now), sourceName: sources.find((s) => s.id === b.sourceId)?.name ?? "", problem: problems.find((p) => new RegExp(`^duplicati-(failed|stale|warnings|streak|growth):${b.id}(:|$)`).test(p.key))?.message ?? null, warningStreak: byBackup.get(b.id)?.streak ?? 0, growth: byBackup.get(b.id)?.growth ?? null })),
|
||||
servers: [...serverNames].map(([id, name]) => ({ id, name })).sort((a, b) => a.name.localeCompare(b.name)),
|
||||
defaultExpectedHours: DEFAULT_EXPECTED_HOURS,
|
||||
streakRuns: STREAK_RUNS,
|
||||
forecastDays,
|
||||
});
|
||||
}));
|
||||
|
||||
@@ -98,7 +103,7 @@ duplicatiRouter.delete("/sources/:id", requireRole("admin"), asyncHandler(async
|
||||
res.status(204).end();
|
||||
}));
|
||||
|
||||
const backupSchema = z.object({ expectedHours: z.number().int().min(1).max(24 * 90).optional(), alert: z.boolean().optional(), alertWarnings: z.boolean().optional() });
|
||||
const backupSchema = z.object({ expectedHours: z.number().int().min(1).max(24 * 90).optional(), alert: z.boolean().optional(), alertWarnings: z.boolean().optional(), warnStreakAlert: z.boolean().optional() });
|
||||
|
||||
// How often a backup is expected, and whether it alerts: an operator's call, like the other backup settings.
|
||||
duplicatiRouter.patch("/backups/:id", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
@@ -107,7 +112,7 @@ duplicatiRouter.patch("/backups/:id", requireRole("operator"), asyncHandler(asyn
|
||||
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", message: "Hours must be between 1 and 2160." });
|
||||
const [before] = await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
await db.update(duplicatiBackups).set({ ...(parsed.data.expectedHours !== undefined ? { expectedHours: parsed.data.expectedHours } : {}), ...(parsed.data.alert !== undefined ? { alert: parsed.data.alert } : {}), ...(parsed.data.alertWarnings !== undefined ? { alertWarnings: parsed.data.alertWarnings } : {}) }).where(eq(duplicatiBackups.id, id));
|
||||
await db.update(duplicatiBackups).set({ ...(parsed.data.expectedHours !== undefined ? { expectedHours: parsed.data.expectedHours } : {}), ...(parsed.data.alert !== undefined ? { alert: parsed.data.alert } : {}), ...(parsed.data.alertWarnings !== undefined ? { alertWarnings: parsed.data.alertWarnings } : {}), ...(parsed.data.warnStreakAlert !== undefined ? { warnStreakAlert: parsed.data.warnStreakAlert } : {}) }).where(eq(duplicatiBackups.id, id));
|
||||
await recordAudit({ actor: req.currentUser!, category: "duplicati", action: "update_backup", targetType: "duplicati_backup", targetId: id, detail: { name: before.name, ...parsed.data } });
|
||||
invalidateAlertsCache();
|
||||
res.json({ ok: true });
|
||||
|
||||
@@ -156,7 +156,7 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
|
||||
if (m) return { type: m[1], id: Number(m[3]), category: m[2] === "disk" ? "disk" : "services" };
|
||||
m = /^fragnet-(disk|memory|backup):(\d+):/.exec(key);
|
||||
if (m) return { type: "fragnet", id: Number(m[2]), category: m[1] === "disk" ? "disk" : m[1] === "backup" ? "backup" : "monitoring" };
|
||||
m = /^duplicati-(?:failed|stale|warnings):(\d+)/.exec(key);
|
||||
m = /^duplicati-(?:failed|stale|warnings|streak|growth):(\d+)/.exec(key);
|
||||
if (m) return { type: "duplicati", id: Number(m[1]), category: "backup" };
|
||||
m = /^nextcloud-(?:maintenance|upgrade|disk|cron|checks):(\d+)/.exec(key);
|
||||
if (m) return { type: "nextcloud", id: Number(m[1]), category: key.startsWith("nextcloud-disk") ? "disk" : "services" };
|
||||
|
||||
@@ -8,11 +8,16 @@
|
||||
* expected to** (a day or two by default, set per job; critical at three times that). A run that finished with warnings still counts as a backup — and says what the warnings were (the first
|
||||
* twenty, kept with the run) — and a job can be set to **alert when its last run had warnings** (off until switched on: some jobs always have a few). Only runs of the *Backup*
|
||||
* operation count: a compact, a repair or a verification does not say whether the data is safe.
|
||||
*
|
||||
* Two slower things are watched as well: a job whose **last few runs in a row all finished with warnings** (one run with warnings is common; three in a row is a job nobody is looking at — on by
|
||||
* default, and a job that always has a few can be told not to), and **how much room the backups take where they are kept** (Duplicati says how big the destination is, and for some kinds of
|
||||
* storage how much room it has): the growth is worked out from the history, and a destination with a known size that will be full within the disk-forecast window is an alert.
|
||||
*/
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { and, asc, desc, eq, lt, notInArray } from "drizzle-orm";
|
||||
import { and, asc, desc, eq, inArray, lt, notInArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { duplicatiBackups, duplicatiRuns, duplicatiSources } from "../db/schema.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
|
||||
export type SourceRow = typeof duplicatiSources.$inferSelect;
|
||||
export type BackupRow = typeof duplicatiBackups.$inferSelect;
|
||||
@@ -43,6 +48,10 @@ export interface ParsedReport {
|
||||
deletedFiles: number | null;
|
||||
warnings: number;
|
||||
errors: number;
|
||||
/** How much the backup takes up where it is kept (Duplicati's BackendStatistics.KnownFileSize); null when the report does not say. */
|
||||
destinationBytes: number | null;
|
||||
/** The room the destination has (TotalQuotaSpace), when its kind of storage says — zero and negative mean "unknown" and are null. */
|
||||
quotaBytes: number | null;
|
||||
message: string | null;
|
||||
/** What the warnings said, one line each (up to MAX_WARNING_MESSAGES, each cut at MAX_WARNING_LENGTH). Only the JSON format carries them. */
|
||||
warningMessages: string[];
|
||||
@@ -174,11 +183,17 @@ export function parseReport(body: unknown): ParsedReport | null {
|
||||
deletedFiles: asInt(data.DeletedFiles),
|
||||
warnings: asInt(data.WarningsActualLength) ?? warnings.length,
|
||||
errors: asInt(data.ErrorsActualLength) ?? errors.length,
|
||||
...destinationOf(asInt(obj(data.BackendStatistics).KnownFileSize), asInt(obj(data.BackendStatistics).TotalQuotaSpace)),
|
||||
message: lineOf(errors[0]) ?? lineOf(warnings[0]),
|
||||
warningMessages: messagesOf(warnings),
|
||||
};
|
||||
}
|
||||
|
||||
/** Pure: the size of the destination and the room it has, from what Duplicati said; a size below zero or a quota of zero or less is "not said". */
|
||||
export function destinationOf(known: number | null, quota: number | null): { destinationBytes: number | null; quotaBytes: number | null } {
|
||||
return { destinationBytes: known !== null && known >= 0 ? known : null, quotaBytes: quota !== null && quota > 0 ? quota : null };
|
||||
}
|
||||
|
||||
function parseText(text: string, header: MessageHeader | null = null): ParsedReport | null {
|
||||
const value = (key: string) => new RegExp(`^\\s*${key}\\s*:\\s*(.+?)\\s*$`, "im").exec(text)?.[1] ?? null;
|
||||
const result = value("ParsedResult");
|
||||
@@ -200,6 +215,7 @@ function parseText(text: string, header: MessageHeader | null = null): ParsedRep
|
||||
deletedFiles: asInt(value("DeletedFiles")?.match(/^[\d,]+/)?.[0]),
|
||||
warnings: asInt(value("WarningsActualLength")?.match(/^[\d,]+/)?.[0]) ?? warningLines.length,
|
||||
errors: asInt(value("ErrorsActualLength")?.match(/^[\d,]+/)?.[0]) ?? 0,
|
||||
...destinationOf(asInt(value("KnownFileSize")?.match(/^-?[\d,]+/)?.[0]), asInt(value("TotalQuotaSpace")?.match(/^-?[\d,]+/)?.[0])),
|
||||
message: lineOf(errorLines[0]) ?? lineOf(warningLines[0]),
|
||||
warningMessages: warningLines,
|
||||
};
|
||||
@@ -322,6 +338,9 @@ export async function recordReport(token: string, body: unknown, now: Date = new
|
||||
deletedFiles: report.deletedFiles,
|
||||
warnings: report.warnings,
|
||||
errors: report.errors,
|
||||
// What a report does not say is left as it was (a run of another kind, or an older Duplicati, must not wipe what was known).
|
||||
destinationBytes: report.destinationBytes ?? before?.destinationBytes ?? null,
|
||||
quotaBytes: report.quotaBytes ?? before?.quotaBytes ?? null,
|
||||
message: report.message,
|
||||
warningMessages: packMessages(report.warningMessages),
|
||||
lastSuccessAt: good ? at : (before?.lastSuccessAt ?? null),
|
||||
@@ -335,7 +354,7 @@ export async function recordReport(token: string, body: unknown, now: Date = new
|
||||
id = (await db.insert(duplicatiBackups).values({ sourceId: source.id, name: report.backupName, ...values }).returning({ id: duplicatiBackups.id }))[0].id;
|
||||
}
|
||||
const changed = (report.addedFiles ?? 0) + (report.modifiedFiles ?? 0) + (report.deletedFiles ?? 0);
|
||||
await db.insert(duplicatiRuns).values({ backupId: id, at, result: report.result, durationSeconds: report.durationSeconds, sizeBytes: report.sizeBytes, filesChanged: report.addedFiles === null && report.modifiedFiles === null && report.deletedFiles === null ? null : changed, warnings: report.warnings, errors: report.errors, warningMessages: packMessages(report.warningMessages) });
|
||||
await db.insert(duplicatiRuns).values({ backupId: id, at, result: report.result, durationSeconds: report.durationSeconds, sizeBytes: report.sizeBytes, filesChanged: report.addedFiles === null && report.modifiedFiles === null && report.deletedFiles === null ? null : changed, warnings: report.warnings, errors: report.errors, warningMessages: packMessages(report.warningMessages), destinationBytes: report.destinationBytes });
|
||||
await pruneRuns(id, now);
|
||||
return { recorded: true, backupId: id, result: report.result };
|
||||
}
|
||||
@@ -358,6 +377,97 @@ export function backupState(b: Pick<BackupRow, "lastResult" | "lastSuccessAt" |
|
||||
return b.lastResult && /^warning$/i.test(b.lastResult) ? "warning" : "ok";
|
||||
}
|
||||
|
||||
// ─── Warnings in a row, and growth ──────────────────────────────────────────
|
||||
|
||||
/** How many runs in a row, the newest, must have finished with warnings for it to be said. */
|
||||
export const STREAK_RUNS = 3;
|
||||
|
||||
/** Pure: how many of the newest runs in a row finished with warnings (the results newest first); a success or a failure ends it. */
|
||||
export function warningStreak(resultsNewestFirst: string[]): number {
|
||||
let n = 0;
|
||||
for (const r of resultsNewestFirst) {
|
||||
if (!/^warning$/i.test(r)) break;
|
||||
n++;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
/** What a growth forecast needs before it is worth saying: enough readings, over long enough, that a straight line describes. */
|
||||
export const GROWTH_MIN_POINTS = 5;
|
||||
export const GROWTH_MIN_SPAN_DAYS = 7;
|
||||
export const GROWTH_MIN_FIT = 0.8;
|
||||
|
||||
export interface Growth {
|
||||
/** Bytes a day, from a straight line through the readings (negative: shrinking). */
|
||||
perDay: number;
|
||||
/** The same over thirty days. */
|
||||
perMonth: number;
|
||||
/** How well a straight line describes them, 0 to 1 (R²). */
|
||||
fit: number;
|
||||
spanDays: number;
|
||||
/** Where it stands now. */
|
||||
latest: number;
|
||||
/** For a destination with a known size that is growing: days until it is full at that rate; null otherwise. */
|
||||
daysToFull: number | null;
|
||||
}
|
||||
|
||||
/** Pure: how a backup's size has grown, from (time, bytes) readings; null without enough of them. */
|
||||
export function growthOf(points: { t: number; v: number }[], quotaBytes: number | null): Growth | null {
|
||||
const p = [...points].sort((a, b) => a.t - b.t);
|
||||
if (p.length < GROWTH_MIN_POINTS) return null;
|
||||
const span = p[p.length - 1].t - p[0].t;
|
||||
if (span < GROWTH_MIN_SPAN_DAYS * DAY_MS) return null;
|
||||
const xs = p.map((q) => (q.t - p[0].t) / DAY_MS);
|
||||
const ys = p.map((q) => q.v);
|
||||
const n = p.length;
|
||||
const mx = xs.reduce((a, b) => a + b, 0) / n;
|
||||
const my = ys.reduce((a, b) => a + b, 0) / n;
|
||||
let num = 0;
|
||||
let den = 0;
|
||||
let total = 0;
|
||||
for (let i = 0; i < n; i++) {
|
||||
num += (xs[i] - mx) * (ys[i] - my);
|
||||
den += (xs[i] - mx) ** 2;
|
||||
total += (ys[i] - my) ** 2;
|
||||
}
|
||||
if (den === 0) return null;
|
||||
const slope = num / den;
|
||||
const fit = total === 0 ? 0 : Math.round(((num * num) / (den * total)) * 100) / 100;
|
||||
const latest = ys[n - 1];
|
||||
return {
|
||||
perDay: Math.round(slope),
|
||||
perMonth: Math.round(slope * 30),
|
||||
fit,
|
||||
spanDays: Math.round((span / DAY_MS) * 10) / 10,
|
||||
latest,
|
||||
daysToFull: quotaBytes !== null && slope > 0 && latest < quotaBytes ? Math.ceil((quotaBytes - latest) / slope) : null,
|
||||
};
|
||||
}
|
||||
|
||||
/** What is worked out from each job's runs, for the alerts and the page. */
|
||||
export interface BackupAnalysis {
|
||||
streak: number;
|
||||
growth: Growth | null;
|
||||
}
|
||||
|
||||
/** The runs of these jobs, newest first, worked into the streak of warnings and the growth of the destination. One query for all of them. */
|
||||
export async function analyseBackups(backups: Pick<BackupRow, "id" | "quotaBytes">[], now: Date = new Date()): Promise<Map<number, BackupAnalysis>> {
|
||||
const out = new Map<number, BackupAnalysis>();
|
||||
if (backups.length === 0) return out;
|
||||
const since = new Date(now.getTime() - 180 * DAY_MS).toISOString();
|
||||
const runs = await db
|
||||
.select({ backupId: duplicatiRuns.backupId, at: duplicatiRuns.at, result: duplicatiRuns.result, destinationBytes: duplicatiRuns.destinationBytes })
|
||||
.from(duplicatiRuns)
|
||||
.where(and(inArray(duplicatiRuns.backupId, backups.map((b) => b.id)), lt(duplicatiRuns.at, new Date(now.getTime() + DAY_MS).toISOString())))
|
||||
.orderBy(desc(duplicatiRuns.at), desc(duplicatiRuns.id));
|
||||
for (const b of backups) {
|
||||
const mine = runs.filter((r) => r.backupId === b.id);
|
||||
const points = mine.filter((r) => r.destinationBytes !== null && r.at >= since).map((r) => ({ t: Date.parse(r.at), v: r.destinationBytes as number }));
|
||||
out.set(b.id, { streak: warningStreak(mine.slice(0, 50).map((r) => r.result)), growth: growthOf(points, b.quotaBytes) });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const ago = (ms: number) => {
|
||||
const h = Math.floor(ms / HOUR_MS);
|
||||
return h < 1 ? "less than an hour" : h < 48 ? `${h} h` : `${Math.floor(h / 24)} days`;
|
||||
@@ -365,13 +475,13 @@ const ago = (ms: number) => {
|
||||
|
||||
export interface DuplicatiProblem {
|
||||
key: string;
|
||||
kind: "failed" | "stale" | "warnings";
|
||||
kind: "failed" | "stale" | "warnings" | "streak" | "growth";
|
||||
severity: "critical" | "warning";
|
||||
message: string;
|
||||
}
|
||||
|
||||
/** Pure: what is wrong with the backups, for those that are to alert. The key carries the server a source runs on, so maintenance on it silences them. */
|
||||
export function duplicatiProblems(sources: Pick<SourceRow, "id" | "name" | "serverId">[], backups: BackupRow[], nowMs: number): DuplicatiProblem[] {
|
||||
export function duplicatiProblems(sources: Pick<SourceRow, "id" | "name" | "serverId">[], backups: BackupRow[], nowMs: number, analysis?: { byBackup: Map<number, BackupAnalysis>; forecastDays: number }): DuplicatiProblem[] {
|
||||
const byId = new Map(sources.map((s) => [s.id, s]));
|
||||
const out: DuplicatiProblem[] = [];
|
||||
for (const b of backups) {
|
||||
@@ -391,6 +501,18 @@ export function duplicatiProblems(sources: Pick<SourceRow, "id" | "name" | "serv
|
||||
const more = b.warnings > 1 ? ` (and ${b.warnings - 1} more)` : "";
|
||||
out.push({ key: `duplicati-warnings:${b.id}${suffix}`, kind: "warnings", severity: "warning", message: `The backup “${b.name}” on ${source.name} finished with ${b.warnings} warning${b.warnings === 1 ? "" : "s"}${first ? `: ${first.slice(0, 200)}${more}` : ""}` });
|
||||
}
|
||||
const a = analysis?.byBackup.get(b.id);
|
||||
if (!a) continue;
|
||||
// Warnings run after run: said once, when no alert is already raised for each warning run (that one says it every time).
|
||||
if (state === "warning" && b.warnStreakAlert && !b.alertWarnings && a.streak >= STREAK_RUNS) {
|
||||
const first = unpackMessages(b.warningMessages)[0] ?? b.message;
|
||||
out.push({ key: `duplicati-streak:${b.id}${suffix}`, kind: "streak", severity: "warning", message: `The backup “${b.name}” on ${source.name} has finished with warnings ${a.streak} runs in a row${first ? ` — the last said: ${first.slice(0, 200)}` : ""}` });
|
||||
}
|
||||
// A destination that will be full soon: only where its size is known, and only when the growth is steady enough to forecast from.
|
||||
const g = a.growth;
|
||||
if (g && b.quotaBytes !== null && g.daysToFull !== null && analysis && analysis.forecastDays > 0 && g.daysToFull <= analysis.forecastDays && g.fit >= GROWTH_MIN_FIT) {
|
||||
out.push({ key: `duplicati-growth:${b.id}${suffix}`, kind: "growth", severity: g.daysToFull <= 3 ? "critical" : "warning", message: `The backup “${b.name}” on ${source.name} is growing ${Math.round(g.perDay / 1e6)} MB a day and its storage will be full in about ${g.daysToFull} day${g.daysToFull === 1 ? "" : "s"}` });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -403,7 +525,10 @@ export async function listAll(): Promise<{ sources: SourceRow[]; backups: Backup
|
||||
/** What is wrong with the backups right now, from what is stored. */
|
||||
export async function currentProblems(nowMs: number = Date.now()): Promise<DuplicatiProblem[]> {
|
||||
const { sources, backups } = await listAll();
|
||||
return sources.length === 0 ? [] : duplicatiProblems(sources, backups, nowMs);
|
||||
if (sources.length === 0) return [];
|
||||
const byBackup = await analyseBackups(backups, new Date(nowMs));
|
||||
const forecastDays = (await getSettings()).healthChecks.diskForecastDays;
|
||||
return duplicatiProblems(sources, backups, nowMs, { byBackup, forecastDays });
|
||||
}
|
||||
|
||||
/** A source as lists show it: everything but the hash. */
|
||||
|
||||
@@ -69,7 +69,7 @@ export function subjectOfConditionKey(key: string): string | null {
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^authentik-(?:outpost|outpost-version|cert|token|worker|logins|errors):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^duplicati-(?:failed|stale|warnings):\d+:server:(\d+)$/.exec(key);
|
||||
m = /^duplicati-(?:failed|stale|warnings|streak|growth):\d+:server:(\d+)$/.exec(key);
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:(\d+)$/.exec(key);
|
||||
if (m) return `server:${m[1]}`;
|
||||
|
||||
@@ -0,0 +1,263 @@
|
||||
/**
|
||||
* Duplicati, the slower things: a job whose last runs in a row all had warnings, and how much room the backups take where they are kept (read from what Duplicati reports, worked into a growth and a
|
||||
* forecast, and an alert when a destination with a known size will be full within the forecast window).
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { Server } from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-dupfollow-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, duplicatiSources, duplicatiBackups, duplicatiRuns } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const d = await import("../src/services/duplicati.js");
|
||||
const { duplicatiRouter } = await import("../src/routes/duplicati.js");
|
||||
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const DAY = 86_400_000;
|
||||
const NOW = new Date("2026-11-10T12:00:00.000Z");
|
||||
const daysAgo = (n: number) => new Date(NOW.getTime() - n * DAY).toISOString();
|
||||
|
||||
const report = (over: Record<string, unknown> = {}, backend: Record<string, unknown> | null = { KnownFileSize: 5_000_000_000, TotalQuotaSpace: 0 }, name = "Documents") =>
|
||||
JSON.stringify({
|
||||
Data: { MainOperation: "Backup", ParsedResult: "Success", EndTime: "2026-10-23T03:35:44.1Z", BeginTime: "2026-10-23T03:30:31.4Z", Duration: "00:05:12.7", Warnings: [], Errors: [], ...(backend ? { BackendStatistics: backend } : {}), ...over },
|
||||
Extra: { OperationName: "Backup", "backup-name": name },
|
||||
});
|
||||
|
||||
describe("what the report says about the destination", () => {
|
||||
test("JSON: the size where the backups are kept, and the room there is when its storage says", () => {
|
||||
const r = d.parseReport(report({}, { KnownFileSize: 12_345_678, TotalQuotaSpace: 500_000_000_000, FreeQuotaSpace: 1 }))!;
|
||||
assert.deepEqual([r.destinationBytes, r.quotaBytes], [12_345_678, 500_000_000_000]);
|
||||
});
|
||||
|
||||
test("a quota of zero or less is not said, a size below zero is not said, and a report without the section says neither", () => {
|
||||
assert.deepEqual(d.destinationOf(100, 0), { destinationBytes: 100, quotaBytes: null });
|
||||
assert.deepEqual(d.destinationOf(100, -1), { destinationBytes: 100, quotaBytes: null });
|
||||
assert.deepEqual(d.destinationOf(-5, 10), { destinationBytes: null, quotaBytes: 10 });
|
||||
assert.deepEqual(d.destinationOf(null, null), { destinationBytes: null, quotaBytes: null });
|
||||
const none = d.parseReport(report({}, null))!;
|
||||
assert.deepEqual([none.destinationBytes, none.quotaBytes], [null, null]);
|
||||
const zero = d.parseReport(report({}, { KnownFileSize: 0, TotalQuotaSpace: 0 }))!;
|
||||
assert.deepEqual([zero.destinationBytes, zero.quotaBytes], [0, null], "an empty destination is a size, not a missing one");
|
||||
});
|
||||
|
||||
test("the plain text form: the bytes come first, the readable size after", () => {
|
||||
const text = ["MainOperation: Backup", "ParsedResult: Success", "Duration: 00:01:00", "BackendStatistics:", " KnownFileSize: 5811709 (5.54 MB)", " TotalQuotaSpace: 107374182400 (100.00 GB)", " FreeQuotaSpace: 0", "backup-name: Docs"].join("\n");
|
||||
const r = d.parseReport(text)!;
|
||||
assert.deepEqual([r.destinationBytes, r.quotaBytes], [5_811_709, 107_374_182_400]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("warnings in a row", () => {
|
||||
test("counted from the newest run; a success, a failure or an unknown word ends it", () => {
|
||||
assert.equal(d.warningStreak([]), 0);
|
||||
assert.equal(d.warningStreak(["Warning", "Warning", "Warning", "Success", "Warning"]), 3);
|
||||
assert.equal(d.warningStreak(["warning", "WARNING"]), 2);
|
||||
assert.equal(d.warningStreak(["Success", "Warning", "Warning"]), 0);
|
||||
assert.equal(d.warningStreak(["Warning", "Error", "Warning"]), 1);
|
||||
assert.equal(d.STREAK_RUNS, 3);
|
||||
});
|
||||
});
|
||||
|
||||
describe("growth", () => {
|
||||
const steady = (days: number, perDay: number, start = 1e9, every = 1) => Array.from({ length: Math.floor(days / every) + 1 }, (_, i) => ({ t: NOW.getTime() - (days - i * every) * DAY, v: start + i * every * perDay }));
|
||||
|
||||
test("a steady climb: bytes a day and a month, how well a line fits, and when a known quota is full", () => {
|
||||
const g = d.growthOf(steady(20, 50_000_000), 5e9)!;
|
||||
assert.equal(g.perDay, 50_000_000);
|
||||
assert.equal(g.perMonth, 1_500_000_000);
|
||||
assert.equal(g.fit, 1);
|
||||
assert.equal(g.spanDays, 20);
|
||||
assert.equal(g.latest, 2e9);
|
||||
assert.equal(g.daysToFull, 60, "(5 GB − 2 GB) at 50 MB a day");
|
||||
});
|
||||
|
||||
test("no quota, nothing to fill; shrinking, nothing to fill; already over, nothing to fill", () => {
|
||||
assert.equal(d.growthOf(steady(20, 50_000_000), null)!.daysToFull, null);
|
||||
const shrinking = d.growthOf(steady(20, -20_000_000, 5e9), 9e9)!;
|
||||
assert.ok(shrinking.perDay < 0);
|
||||
assert.equal(shrinking.daysToFull, null);
|
||||
assert.equal(d.growthOf(steady(20, 50_000_000), 1e9)!.daysToFull, null);
|
||||
});
|
||||
|
||||
test("not enough to go on: too few readings, or too short a time", () => {
|
||||
assert.equal(d.growthOf(steady(20, 1, 1e9, 10), null), null, "three readings");
|
||||
assert.equal(d.growthOf(steady(5, 1_000, 1e9, 1), null), null, "five days");
|
||||
assert.equal(d.growthOf([], null), null);
|
||||
const flat = d.growthOf(steady(20, 0), 5e9)!;
|
||||
assert.deepEqual([flat.perDay, flat.fit, flat.daysToFull], [0, 0, null], "a size that does not move has no fit and nothing to fill");
|
||||
});
|
||||
|
||||
test("a jagged size (a compaction now and then) fits badly", () => {
|
||||
const jagged = steady(30, 0).map((p, i) => ({ ...p, v: 1e9 + (i % 5 === 0 ? 0 : 3e8) * (i % 3) }));
|
||||
assert.ok(d.growthOf(jagged, 5e9)!.fit < d.GROWTH_MIN_FIT);
|
||||
});
|
||||
});
|
||||
|
||||
describe("what a report does to the destination, and the alerts from it", () => {
|
||||
let sourceId = 0;
|
||||
let token = "";
|
||||
before(async () => {
|
||||
const made = await d.createSource({ name: "Jackie", serverId: null, createdBy: "Ada" });
|
||||
sourceId = made.row.id;
|
||||
token = made.token;
|
||||
});
|
||||
after(() => {
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const backupId = async (name: string) => (await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.name, name)))[0].id;
|
||||
|
||||
test("it is written on the job and on the run; a report that does not say leaves what was known", async () => {
|
||||
await d.recordReport(token, report({}, { KnownFileSize: 7_000_000_000, TotalQuotaSpace: 9_000_000_000 }, "Docs"), new Date("2026-11-01T04:00:00Z"));
|
||||
const id = await backupId("Docs");
|
||||
let [b] = await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.id, id));
|
||||
assert.deepEqual([b.destinationBytes, b.quotaBytes], [7_000_000_000, 9_000_000_000]);
|
||||
assert.equal((await db.select().from(duplicatiRuns).where(eq(duplicatiRuns.backupId, id)))[0].destinationBytes, 7_000_000_000);
|
||||
|
||||
await d.recordReport(token, report({}, null, "Docs"), new Date("2026-11-02T04:00:00Z"));
|
||||
[b] = await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.id, id));
|
||||
assert.deepEqual([b.destinationBytes, b.quotaBytes], [7_000_000_000, 9_000_000_000], "kept");
|
||||
const runs = await db.select().from(duplicatiRuns).where(eq(duplicatiRuns.backupId, id));
|
||||
assert.deepEqual(runs.map((r) => r.destinationBytes).sort(), [7_000_000_000, null], "the run itself says nothing");
|
||||
});
|
||||
|
||||
/** A job with a history made directly: the results newest last, one a day, and (optionally) a growing size. */
|
||||
async function jobWith(name: string, results: string[], size?: { start: number; perDay: number; quota: number | null }, flags: Partial<typeof duplicatiBackups.$inferInsert> = {}) {
|
||||
const [b] = await db
|
||||
.insert(duplicatiBackups)
|
||||
.values({ sourceId, name, lastResult: results[results.length - 1], lastSuccessAt: daysAgo(0.5), lastReportAt: daysAgo(0.5), warnings: results[results.length - 1] === "Warning" ? 2 : 0, warningMessages: JSON.stringify(["locked file", "another"]), quotaBytes: size?.quota ?? null, ...flags })
|
||||
.returning();
|
||||
await db.insert(duplicatiRuns).values(results.map((result, i) => ({ backupId: b.id, at: daysAgo(results.length - 1 - i + 0.5), result, warnings: result === "Warning" ? 1 : 0, errors: 0, destinationBytes: size ? size.start + i * size.perDay : null })));
|
||||
return b;
|
||||
}
|
||||
const problemsOf = async (b: typeof duplicatiBackups.$inferSelect, forecastDays = 14) => {
|
||||
const [fresh] = await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.id, b.id));
|
||||
const [source] = await db.select().from(duplicatiSources).where(eq(duplicatiSources.id, sourceId));
|
||||
const byBackup = await d.analyseBackups([fresh], NOW);
|
||||
return d.duplicatiProblems([source], [fresh], NOW.getTime(), { byBackup, forecastDays }).filter((p) => p.key.endsWith(`:${b.id}`) || p.key.includes(`:${b.id}:`));
|
||||
};
|
||||
|
||||
test("three warnings in a row is said once, with what the last said; two is not, and a success after them ends it", async () => {
|
||||
const three = await jobWith("Three", ["Success", "Warning", "Warning", "Warning"]);
|
||||
const p = await problemsOf(three);
|
||||
assert.deepEqual(p.map((x) => [x.key, x.kind, x.severity]), [[`duplicati-streak:${three.id}`, "streak", "warning"]]);
|
||||
assert.match(p[0].message, /“Three” on Jackie has finished with warnings 3 runs in a row — the last said: locked file/);
|
||||
|
||||
const two = await jobWith("Two", ["Success", "Warning", "Warning"]);
|
||||
assert.deepEqual(await problemsOf(two), []);
|
||||
const healed = await jobWith("Healed", ["Warning", "Warning", "Warning", "Success"]);
|
||||
assert.deepEqual(await problemsOf(healed), []);
|
||||
});
|
||||
|
||||
test("a job told not to, or one that already alerts on every warning run, does not say it twice; a failed or silenced job is left to its own alerts", async () => {
|
||||
assert.deepEqual(await problemsOf(await jobWith("Quiet", ["Warning", "Warning", "Warning"], undefined, { warnStreakAlert: false })), []);
|
||||
const every = await problemsOf(await jobWith("Every", ["Warning", "Warning", "Warning"], undefined, { alertWarnings: true }));
|
||||
assert.deepEqual(every.map((x) => x.kind), ["warnings"], "the one that says it each time");
|
||||
assert.deepEqual(await problemsOf(await jobWith("Muted", ["Warning", "Warning", "Warning"], undefined, { alert: false })), []);
|
||||
});
|
||||
|
||||
test("a destination with a known size that fills within the window is an alert; further off, unknown, or erratic is not", async () => {
|
||||
// 20 days at 100 MB a day from 1 GB: now 2.9 GB of 3.4 GB, five days to full
|
||||
const soon = await jobWith("Soon", Array(20).fill("Success"), { start: 1e9, perDay: 100e6, quota: 3.4e9 });
|
||||
const p = await problemsOf(soon);
|
||||
assert.deepEqual(p.map((x) => [x.kind, x.severity]), [["growth", "warning"]]);
|
||||
assert.match(p[0].message, /“Soon” on Jackie is growing 100 MB a day and its storage will be full in about 5 days/);
|
||||
|
||||
const urgent = await jobWith("Urgent", Array(20).fill("Success"), { start: 1e9, perDay: 100e6, quota: 2.95e9 });
|
||||
assert.equal((await problemsOf(urgent))[0].severity, "critical", "three days or less");
|
||||
|
||||
assert.deepEqual(await problemsOf(soon, 3), [], "five days is outside a three day window");
|
||||
assert.deepEqual(await problemsOf(soon, 0), [], "switched off with zero");
|
||||
assert.deepEqual(await problemsOf(await jobWith("Far", Array(20).fill("Success"), { start: 1e9, perDay: 100e6, quota: 50e9 })), []);
|
||||
assert.deepEqual(await problemsOf(await jobWith("NoQuota", Array(20).fill("Success"), { start: 1e9, perDay: 100e6, quota: null })), [], "nothing to fill when it does not say how much room there is");
|
||||
});
|
||||
|
||||
test("the alerts belong to the server the machine runs on, so maintenance there silences them, and they are backup alerts about this integration", () => {
|
||||
assert.equal(subjectOfConditionKey("duplicati-streak:4:server:9"), "server:9");
|
||||
assert.equal(subjectOfConditionKey("duplicati-growth:4:server:9"), "server:9");
|
||||
assert.equal(subjectOfConditionKey("duplicati-growth:4"), null);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the page's data", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "operator";
|
||||
let sourceId = 0;
|
||||
let jobId = 0;
|
||||
const call = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "operator", name: "Olle", role: "operator" },
|
||||
{ oidcSub: "viewer", name: "Vera", role: "viewer" },
|
||||
]);
|
||||
const made = await d.createSource({ name: "Route machine", serverId: null, createdBy: "Ada" });
|
||||
sourceId = made.row.id;
|
||||
const [b] = await db.insert(duplicatiBackups).values({ sourceId, name: "Routed", lastResult: "Warning", lastSuccessAt: new Date().toISOString(), warnings: 1, quotaBytes: 4e9 }).returning();
|
||||
jobId = b.id;
|
||||
const now = Date.now();
|
||||
await db.insert(duplicatiRuns).values(Array.from({ length: 20 }, (_, i) => ({ backupId: b.id, at: new Date(now - (19 - i + 0.2) * DAY).toISOString(), result: i >= 17 ? "Warning" : "Success", destinationBytes: 1e9 + i * 100e6 })));
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/duplicati", duplicatiRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const x = a.listen(0, "127.0.0.1", () => resolve(x));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/duplicati`;
|
||||
});
|
||||
after(() => app.close());
|
||||
|
||||
test("each job says how many warnings in a row and how it is growing; the page is told the threshold and the window", async () => {
|
||||
who = "viewer";
|
||||
const body = await (await call("GET", "/")).json();
|
||||
const job = body.backups.find((b: { id: number }) => b.id === jobId);
|
||||
assert.equal(job.warningStreak, 3);
|
||||
assert.equal(job.growth.perDay, 100_000_000);
|
||||
assert.equal(job.growth.fit, 1);
|
||||
assert.equal(job.growth.daysToFull, Math.ceil((4e9 - 2.9e9) / 100e6));
|
||||
assert.equal(job.warnStreakAlert, true);
|
||||
assert.equal(job.destinationBytes, null, "this job's last report was not one that said");
|
||||
assert.equal(body.streakRuns, 3);
|
||||
assert.equal(body.forecastDays, 14);
|
||||
assert.match(job.problem, /warnings 3 runs in a row|growing 100 MB a day/);
|
||||
who = "operator";
|
||||
});
|
||||
|
||||
test("an operator can switch the streak alert off and on; a viewer cannot; the history carries the size", async () => {
|
||||
assert.equal((await call("PATCH", `/backups/${jobId}`, { warnStreakAlert: false })).status, 200);
|
||||
assert.equal((await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.id, jobId)))[0].warnStreakAlert, false);
|
||||
assert.equal((await call("PATCH", `/backups/${jobId}`, { warnStreakAlert: true })).status, 200);
|
||||
assert.equal((await call("PATCH", `/backups/${jobId}`, { warnStreakAlert: "yes" })).status, 400);
|
||||
who = "viewer";
|
||||
assert.equal((await call("PATCH", `/backups/${jobId}`, { warnStreakAlert: false })).status, 403);
|
||||
const runs = await (await call("GET", `/backups/${jobId}/runs?limit=3`)).json();
|
||||
assert.equal(runs.runs.length, 3);
|
||||
assert.equal(runs.runs[0].destinationBytes, 1e9 + 19 * 100e6);
|
||||
who = "operator";
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user