Show what Duplicati's warnings said, and let a job alert on them
The first twenty warning messages of each run are kept with the run and shown on the job and in its history. A job can be set to raise a warning-level alert when its last run finished with warnings (off by default; a muted job raises nothing; maintenance on the machine's server silences it). Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d78aba8bf9
commit
64961b1141
14 files changed
+9241
-24
No files matched your search
@@ -0,0 +1,3 @@
|
||||
ALTER TABLE `duplicati_backups` ADD `warning_messages` text;--> statement-breakpoint
|
||||
ALTER TABLE `duplicati_backups` ADD `alert_warnings` integer DEFAULT false NOT NULL;--> statement-breakpoint
|
||||
ALTER TABLE `duplicati_runs` ADD `warning_messages` text;
|
||||
File diff suppressed because it is too large.
Load diff
@@ -617,6 +617,13 @@
|
||||
"when": 1794176955959,
|
||||
"tag": "0087_majestic_praxagora",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 88,
|
||||
"version": "6",
|
||||
"when": 1794262055100,
|
||||
"tag": "0088_grey_silhouette",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1699,6 +1699,8 @@ export const duplicatiBackups = sqliteTable(
|
||||
warnings: integer("warnings").notNull().default(0),
|
||||
errors: integer("errors").notNull().default(0),
|
||||
message: text("message"), // the first error, or warning
|
||||
warningMessages: text("warning_messages"), // JSON string[]: what the last run's warnings said (up to 20); null: none
|
||||
alertWarnings: integer("alert_warnings", { mode: "boolean" }).notNull().default(false), // raise an alert when the last run finished with warnings
|
||||
lastSuccessAt: text("last_success_at"), // ISO; a backup with warnings still counts
|
||||
consecutiveFailures: integer("consecutive_failures").notNull().default(0),
|
||||
expectedHours: integer("expected_hours").notNull().default(48),
|
||||
@@ -1725,6 +1727,7 @@ export const duplicatiRuns = sqliteTable(
|
||||
filesChanged: integer("files_changed"),
|
||||
warnings: integer("warnings").notNull().default(0),
|
||||
errors: integer("errors").notNull().default(0),
|
||||
warningMessages: text("warning_messages"), // JSON string[]: what this run's warnings said (up to 20); null: none
|
||||
},
|
||||
(t) => [index("duplicati_runs_idx").on(t.backupId, t.at)],
|
||||
);
|
||||
|
||||
@@ -6,7 +6,7 @@ import { duplicatiBackups, duplicatiRuns, duplicatiSources, servers } from "../d
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { backupState, createSource, DEFAULT_EXPECTED_HOURS, duplicatiProblems, listAll, MAX_SOURCES, publicSource, recordReport } from "../services/duplicati.js";
|
||||
import { backupState, createSource, DEFAULT_EXPECTED_HOURS, duplicatiProblems, listAll, MAX_SOURCES, publicSource, recordReport, unpackMessages } from "../services/duplicati.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
import { rateLimit } from "../utils/rateLimit.js";
|
||||
|
||||
@@ -43,7 +43,7 @@ duplicatiRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const problems = duplicatiProblems(sources, backups, now);
|
||||
res.json({
|
||||
sources: sources.map((s) => ({ ...publicSource(s), serverName: s.serverId !== null ? (serverNames.get(s.serverId) ?? null) : null, backups: backups.filter((b) => b.sourceId === s.id).length })),
|
||||
backups: backups.map((b) => ({ ...b, state: backupState(b, now), sourceName: sources.find((s) => s.id === b.sourceId)?.name ?? "", problem: problems.find((p) => new RegExp(`^duplicati-(failed|stale):${b.id}(:|$)`).test(p.key))?.message ?? null })),
|
||||
backups: backups.map((b) => ({ ...b, warningMessages: unpackMessages(b.warningMessages), state: backupState(b, now), sourceName: sources.find((s) => s.id === b.sourceId)?.name ?? "", problem: problems.find((p) => new RegExp(`^duplicati-(failed|stale|warnings):${b.id}(:|$)`).test(p.key))?.message ?? null })),
|
||||
servers: [...serverNames].map(([id, name]) => ({ id, name })).sort((a, b) => a.name.localeCompare(b.name)),
|
||||
defaultExpectedHours: DEFAULT_EXPECTED_HOURS,
|
||||
});
|
||||
@@ -56,7 +56,8 @@ duplicatiRouter.get("/backups/:id/runs", asyncHandler(async (req, res) => {
|
||||
if (!b) return res.status(404).json({ error: "not_found" });
|
||||
const asked = Number(req.query.limit);
|
||||
const limit = Number.isFinite(asked) ? Math.min(400, Math.max(1, Math.round(asked))) : 60;
|
||||
res.json({ backup: b, runs: await db.select().from(duplicatiRuns).where(eq(duplicatiRuns.backupId, id)).orderBy(desc(duplicatiRuns.at), desc(duplicatiRuns.id)).limit(limit) });
|
||||
const runs = await db.select().from(duplicatiRuns).where(eq(duplicatiRuns.backupId, id)).orderBy(desc(duplicatiRuns.at), desc(duplicatiRuns.id)).limit(limit);
|
||||
res.json({ backup: b, runs: runs.map((r) => ({ ...r, warningMessages: unpackMessages(r.warningMessages) })) });
|
||||
}));
|
||||
|
||||
const sourceSchema = z.object({ name: z.string().trim().min(1).max(100), serverId: z.number().int().positive().nullable().optional() });
|
||||
@@ -97,7 +98,7 @@ duplicatiRouter.delete("/sources/:id", requireRole("admin"), asyncHandler(async
|
||||
res.status(204).end();
|
||||
}));
|
||||
|
||||
const backupSchema = z.object({ expectedHours: z.number().int().min(1).max(24 * 90).optional(), alert: z.boolean().optional() });
|
||||
const backupSchema = z.object({ expectedHours: z.number().int().min(1).max(24 * 90).optional(), alert: z.boolean().optional(), alertWarnings: z.boolean().optional() });
|
||||
|
||||
// How often a backup is expected, and whether it alerts: an operator's call, like the other backup settings.
|
||||
duplicatiRouter.patch("/backups/:id", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
@@ -106,7 +107,7 @@ duplicatiRouter.patch("/backups/:id", requireRole("operator"), asyncHandler(asyn
|
||||
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", message: "Hours must be between 1 and 2160." });
|
||||
const [before] = await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
await db.update(duplicatiBackups).set({ ...(parsed.data.expectedHours !== undefined ? { expectedHours: parsed.data.expectedHours } : {}), ...(parsed.data.alert !== undefined ? { alert: parsed.data.alert } : {}) }).where(eq(duplicatiBackups.id, id));
|
||||
await db.update(duplicatiBackups).set({ ...(parsed.data.expectedHours !== undefined ? { expectedHours: parsed.data.expectedHours } : {}), ...(parsed.data.alert !== undefined ? { alert: parsed.data.alert } : {}), ...(parsed.data.alertWarnings !== undefined ? { alertWarnings: parsed.data.alertWarnings } : {}) }).where(eq(duplicatiBackups.id, id));
|
||||
await recordAudit({ actor: req.currentUser!, category: "duplicati", action: "update_backup", targetType: "duplicati_backup", targetId: id, detail: { name: before.name, ...parsed.data } });
|
||||
invalidateAlertsCache();
|
||||
res.json({ ok: true });
|
||||
|
||||
@@ -156,7 +156,7 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
|
||||
if (m) return { type: m[1], id: Number(m[3]), category: m[2] === "disk" ? "disk" : "services" };
|
||||
m = /^fragnet-(disk|memory|backup):(\d+):/.exec(key);
|
||||
if (m) return { type: "fragnet", id: Number(m[2]), category: m[1] === "disk" ? "disk" : m[1] === "backup" ? "backup" : "monitoring" };
|
||||
m = /^duplicati-(?:failed|stale):(\d+)/.exec(key);
|
||||
m = /^duplicati-(?:failed|stale|warnings):(\d+)/.exec(key);
|
||||
if (m) return { type: "duplicati", id: Number(m[1]), category: "backup" };
|
||||
m = /^nextcloud-(?:maintenance|upgrade|disk|cron|checks):(\d+)/.exec(key);
|
||||
if (m) return { type: "nextcloud", id: Number(m[1]), category: key.startsWith("nextcloud-disk") ? "disk" : "services" };
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
* the source was, what changed — is kept with a history.
|
||||
*
|
||||
* What makes an alert: a backup whose last run **failed** (Error or Fatal; critical from the second failure in a row), and one that has **not succeeded for longer than it is
|
||||
* expected to** (a day or two by default, set per job; critical at three times that). A run that finished with warnings still counts as a backup. Only runs of the *Backup*
|
||||
* expected to** (a day or two by default, set per job; critical at three times that). A run that finished with warnings still counts as a backup — and says what the warnings were (the first
|
||||
* twenty, kept with the run) — and a job can be set to **alert when its last run had warnings** (off until switched on: some jobs always have a few). Only runs of the *Backup*
|
||||
* operation count: a compact, a repair or a verification does not say whether the data is safe.
|
||||
*/
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
@@ -43,8 +44,13 @@ export interface ParsedReport {
|
||||
warnings: number;
|
||||
errors: number;
|
||||
message: string | null;
|
||||
/** What the warnings said, one line each (up to MAX_WARNING_MESSAGES, each cut at MAX_WARNING_LENGTH). Only the JSON format carries them. */
|
||||
warningMessages: string[];
|
||||
}
|
||||
|
||||
export const MAX_WARNING_MESSAGES = 20;
|
||||
export const MAX_WARNING_LENGTH = 500;
|
||||
|
||||
/** "00:05:12.7" or "1.02:03:04" in seconds. */
|
||||
export function parseDuration(text: unknown): number | null {
|
||||
if (typeof text !== "string") return null;
|
||||
@@ -66,6 +72,29 @@ const lineOf = (v: unknown): string | null => {
|
||||
const text = typeof v === "string" ? v : v !== null && typeof v === "object" ? JSON.stringify(v) : null;
|
||||
return text ? text.replace(/\s+/g, " ").trim().slice(0, 300) : null;
|
||||
};
|
||||
/** Pure: the messages of a list of warnings, one line each; the empty ones left out. */
|
||||
export function messagesOf(list: unknown): string[] {
|
||||
const out: string[] = [];
|
||||
for (const item of Array.isArray(list) ? list : []) {
|
||||
const text = typeof item === "string" ? item : item !== null && typeof item === "object" ? JSON.stringify(item) : null;
|
||||
const line = text ? text.replace(/\s+/g, " ").trim().slice(0, MAX_WARNING_LENGTH) : "";
|
||||
if (line) out.push(line);
|
||||
if (out.length >= MAX_WARNING_MESSAGES) break;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** What is stored of a list of messages (null for none), and what is read back of it (a list, whatever is in the column). */
|
||||
export const packMessages = (m: string[]): string | null => (m.length > 0 ? JSON.stringify(m) : null);
|
||||
export function unpackMessages(stored: string | null): string[] {
|
||||
if (!stored) return [];
|
||||
try {
|
||||
const v = JSON.parse(stored);
|
||||
return Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure: the report Duplicati sent — JSON (`send-http-result-output-format=Json`: the results at the top, or under `Data`, with the job's name among `Extra`), or the plain text it sends
|
||||
@@ -107,6 +136,7 @@ export function parseReport(body: unknown): ParsedReport | null {
|
||||
warnings: asInt(data.WarningsActualLength) ?? warnings.length,
|
||||
errors: asInt(data.ErrorsActualLength) ?? errors.length,
|
||||
message: lineOf(errors[0]) ?? lineOf(warnings[0]),
|
||||
warningMessages: messagesOf(warnings),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -130,6 +160,7 @@ function parseText(text: string): ParsedReport | null {
|
||||
warnings: asInt(value("WarningsActualLength")?.match(/^[\d,]+/)?.[0]) ?? 0,
|
||||
errors: asInt(value("ErrorsActualLength")?.match(/^[\d,]+/)?.[0]) ?? 0,
|
||||
message: null,
|
||||
warningMessages: [],
|
||||
};
|
||||
}
|
||||
|
||||
@@ -178,6 +209,7 @@ export async function recordReport(token: string, body: unknown, now: Date = new
|
||||
warnings: report.warnings,
|
||||
errors: report.errors,
|
||||
message: report.message,
|
||||
warningMessages: packMessages(report.warningMessages),
|
||||
lastSuccessAt: good ? at : (before?.lastSuccessAt ?? null),
|
||||
consecutiveFailures: good ? 0 : (before?.consecutiveFailures ?? 0) + 1,
|
||||
};
|
||||
@@ -189,7 +221,7 @@ export async function recordReport(token: string, body: unknown, now: Date = new
|
||||
id = (await db.insert(duplicatiBackups).values({ sourceId: source.id, name: report.backupName, ...values }).returning({ id: duplicatiBackups.id }))[0].id;
|
||||
}
|
||||
const changed = (report.addedFiles ?? 0) + (report.modifiedFiles ?? 0) + (report.deletedFiles ?? 0);
|
||||
await db.insert(duplicatiRuns).values({ backupId: id, at, result: report.result, durationSeconds: report.durationSeconds, sizeBytes: report.sizeBytes, filesChanged: report.addedFiles === null && report.modifiedFiles === null && report.deletedFiles === null ? null : changed, warnings: report.warnings, errors: report.errors });
|
||||
await db.insert(duplicatiRuns).values({ backupId: id, at, result: report.result, durationSeconds: report.durationSeconds, sizeBytes: report.sizeBytes, filesChanged: report.addedFiles === null && report.modifiedFiles === null && report.deletedFiles === null ? null : changed, warnings: report.warnings, errors: report.errors, warningMessages: packMessages(report.warningMessages) });
|
||||
await pruneRuns(id, now);
|
||||
return { recorded: true, backupId: id, result: report.result };
|
||||
}
|
||||
@@ -219,7 +251,7 @@ const ago = (ms: number) => {
|
||||
|
||||
export interface DuplicatiProblem {
|
||||
key: string;
|
||||
kind: "failed" | "stale";
|
||||
kind: "failed" | "stale" | "warnings";
|
||||
severity: "critical" | "warning";
|
||||
message: string;
|
||||
}
|
||||
@@ -239,6 +271,11 @@ export function duplicatiProblems(sources: Pick<SourceRow, "id" | "name" | "serv
|
||||
} else if (state === "stale") {
|
||||
const left = nowMs - Date.parse(b.lastSuccessAt as string);
|
||||
out.push({ key: `duplicati-stale:${b.id}${suffix}`, kind: "stale", severity: left > 3 * b.expectedHours * HOUR_MS ? "critical" : "warning", message: `The backup “${b.name}” on ${source.name} hasn't succeeded for ${ago(left)} — it should run at least every ${b.expectedHours >= 48 && b.expectedHours % 24 === 0 ? `${b.expectedHours / 24} days` : `${b.expectedHours} h`}` });
|
||||
} else if (state === "warning" && b.alertWarnings) {
|
||||
const said = unpackMessages(b.warningMessages);
|
||||
const first = said[0] ?? b.message;
|
||||
const more = b.warnings > 1 ? ` (and ${b.warnings - 1} more)` : "";
|
||||
out.push({ key: `duplicati-warnings:${b.id}${suffix}`, kind: "warnings", severity: "warning", message: `The backup “${b.name}” on ${source.name} finished with ${b.warnings} warning${b.warnings === 1 ? "" : "s"}${first ? `: ${first.slice(0, 200)}${more}` : ""}` });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
|
||||
@@ -69,7 +69,7 @@ export function subjectOfConditionKey(key: string): string | null {
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^authentik-(?:outpost|outpost-version|cert|token|worker|logins|errors):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^duplicati-(?:failed|stale):\d+:server:(\d+)$/.exec(key);
|
||||
m = /^duplicati-(?:failed|stale|warnings):\d+:server:(\d+)$/.exec(key);
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:(\d+)$/.exec(key);
|
||||
if (m) return `server:${m[1]}`;
|
||||
|
||||
@@ -156,7 +156,7 @@ describe("what a report does", () => {
|
||||
describe("where a backup stands, and what is an alert", () => {
|
||||
const row = (extra: Partial<import("../src/services/duplicati.js").BackupRow> = {}): import("../src/services/duplicati.js").BackupRow => ({
|
||||
id: 1, sourceId: 1, name: "Documents", lastReportAt: null, lastResult: "Success", lastBegin: null, lastEnd: null, durationSeconds: null, examinedFiles: null, sizeBytes: null, addedFiles: null, modifiedFiles: null, deletedFiles: null,
|
||||
warnings: 0, errors: 0, message: null, lastSuccessAt: new Date(NOW - 5 * HOUR).toISOString(), consecutiveFailures: 0, expectedHours: 48, alert: true, createdAt: "2026-10-01 00:00:00", ...extra,
|
||||
warnings: 0, errors: 0, message: null, warningMessages: null, alertWarnings: false, lastSuccessAt: new Date(NOW - 5 * HOUR).toISOString(), consecutiveFailures: 0, expectedHours: 48, alert: true, createdAt: "2026-10-01 00:00:00", ...extra,
|
||||
});
|
||||
const NOW = Date.parse("2026-10-24T12:00:00Z");
|
||||
const iso = (hoursAgo: number) => new Date(NOW - hoursAgo * HOUR).toISOString();
|
||||
@@ -198,6 +198,91 @@ describe("where a backup stands, and what is an alert", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("what the warnings said", () => {
|
||||
test("a report's warnings are kept as one line each, the first twenty, each cut at 500 characters; the first is the message", () => {
|
||||
const r = d.parseReport(report({ ParsedResult: "Warning", WarningsActualLength: 3, Warnings: ["Failed to process path: C:\\locked.db\n access denied", " second ", { code: "x" }] }))!;
|
||||
assert.deepEqual(r.warningMessages, ["Failed to process path: C:\\locked.db access denied", "second", '{"code":"x"}']);
|
||||
assert.equal(r.message, "Failed to process path: C:\\locked.db access denied");
|
||||
const many = d.parseReport(report({ ParsedResult: "Warning", WarningsActualLength: 90, Warnings: Array.from({ length: 90 }, (_, i) => `w${i}`) }))!;
|
||||
assert.deepEqual([many.warnings, many.warningMessages.length, many.warningMessages[19]], [90, 20, "w19"]);
|
||||
assert.equal(d.parseReport(report({ ParsedResult: "Warning", Warnings: ["x".repeat(900)] }))!.warningMessages[0].length, 500);
|
||||
});
|
||||
|
||||
test("empty lines, things that are not a list and the plain text give none", () => {
|
||||
assert.deepEqual(d.messagesOf(["", " ", null, 5, "real"]), ["real"]);
|
||||
for (const nothing of [undefined, null, "text", {}, 7]) assert.deepEqual(d.messagesOf(nothing), []);
|
||||
assert.deepEqual(d.parseReport(["MainOperation: Backup", "ParsedResult: Warning", "WarningsActualLength: 2", "backup-name: Docs"].join("\n"))!.warningMessages, []);
|
||||
assert.deepEqual(d.parseReport(report())!.warningMessages, []);
|
||||
});
|
||||
|
||||
test("what is stored is a list read back as a list, whatever is in the column", () => {
|
||||
assert.equal(d.packMessages([]), null);
|
||||
assert.deepEqual(d.unpackMessages(d.packMessages(["a", "b"])), ["a", "b"]);
|
||||
for (const junk of [null, "", "not json", "{}", "5", '["ok", 5, null]']) assert.ok(Array.isArray(d.unpackMessages(junk)), String(junk));
|
||||
assert.deepEqual(d.unpackMessages('["ok", 5, null]'), ["ok"]);
|
||||
});
|
||||
|
||||
describe("with a report that arrives", () => {
|
||||
let token = "";
|
||||
before(async () => {
|
||||
token = (await d.createSource({ name: "Warny", serverId: null, createdBy: "Ada" })).token;
|
||||
});
|
||||
const job = async () => (await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.name, "Warned")))[0];
|
||||
|
||||
test("the messages go with the run and with the job; a later clean run empties the job's, and the history keeps what was said then", async () => {
|
||||
await d.recordReport(token, report({ ParsedResult: "Warning", WarningsActualLength: 2, Warnings: ["file one locked", "file two locked"], EndTime: "2026-10-26T03:35:00Z" }, "Warned"), new Date("2026-10-26T03:36:00Z"));
|
||||
const b = await job();
|
||||
assert.deepEqual([b.warnings, d.unpackMessages(b.warningMessages)], [2, ["file one locked", "file two locked"]]);
|
||||
await d.recordReport(token, report({ EndTime: "2026-10-27T03:35:00Z" }, "Warned"), new Date("2026-10-27T03:36:00Z"));
|
||||
const after = await job();
|
||||
assert.deepEqual([after.warnings, after.warningMessages], [0, null]);
|
||||
const runs = await db.select().from(duplicatiRuns).where(eq(duplicatiRuns.backupId, b.id));
|
||||
assert.deepEqual(runs.map((r) => [r.result, d.unpackMessages(r.warningMessages)]).sort(), [["Success", []], ["Warning", ["file one locked", "file two locked"]]]);
|
||||
});
|
||||
|
||||
test("a job is set not to alert on warnings until somebody switches it on", async () => {
|
||||
assert.equal((await job()).alertWarnings, false);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("a warning as an alert", () => {
|
||||
const NOW = Date.parse("2026-10-24T12:00:00Z");
|
||||
const HR = 3_600_000;
|
||||
const row = (extra: Partial<import("../src/services/duplicati.js").BackupRow> = {}): import("../src/services/duplicati.js").BackupRow => ({
|
||||
id: 1, sourceId: 1, name: "Documents", lastReportAt: null, lastResult: "Warning", lastBegin: null, lastEnd: null, durationSeconds: null, examinedFiles: null, sizeBytes: null, addedFiles: null, modifiedFiles: null, deletedFiles: null,
|
||||
warnings: 2, errors: 0, message: "file one locked", warningMessages: JSON.stringify(["file one locked", "file two locked"]), alertWarnings: true, lastSuccessAt: new Date(NOW - 5 * HR).toISOString(), consecutiveFailures: 0, expectedHours: 48, alert: true, createdAt: "2026-10-01 00:00:00", ...extra,
|
||||
});
|
||||
const p = (b: Partial<import("../src/services/duplicati.js").BackupRow>, serverId: number | null = null) => d.duplicatiProblems([{ id: 1, name: "Jackie", serverId }], [row(b)], NOW);
|
||||
|
||||
test("a run with warnings is an alert only for a job set to alert on them: a warning, with the count and the first thing it said", () => {
|
||||
assert.deepEqual(p({ alertWarnings: false }), []);
|
||||
const [a] = p({});
|
||||
assert.deepEqual([a.key, a.kind, a.severity], ["duplicati-warnings:1", "warnings", "warning"]);
|
||||
assert.equal(a.message, "The backup “Documents” on Jackie finished with 2 warnings: file one locked (and 1 more)");
|
||||
});
|
||||
|
||||
test("one warning says so in the singular and has no 'more'; with no text it still says how many", () => {
|
||||
assert.equal(p({ warnings: 1, warningMessages: JSON.stringify(["only one"]) })[0].message, "The backup “Documents” on Jackie finished with 1 warning: only one");
|
||||
assert.equal(p({ warningMessages: null, message: null })[0].message, "The backup “Documents” on Jackie finished with 2 warnings");
|
||||
assert.ok(p({ warningMessages: JSON.stringify(["y".repeat(400)]) })[0].message.length < 330, "a long message is cut");
|
||||
});
|
||||
|
||||
test("a clean run, a failed one, an overdue one and a muted job do not raise it; the failure or the lateness is the alert", () => {
|
||||
assert.deepEqual(p({ lastResult: "Success", warnings: 0, warningMessages: null }), []);
|
||||
assert.deepEqual(p({ lastResult: "Error", consecutiveFailures: 1 }).map((x) => x.kind), ["failed"]);
|
||||
assert.deepEqual(p({ lastSuccessAt: new Date(NOW - 80 * HR).toISOString() }).map((x) => x.kind), ["stale"]);
|
||||
assert.deepEqual(p({ alert: false }), [], "a job muted entirely says nothing, warnings included");
|
||||
});
|
||||
|
||||
test("the server a machine runs on is in the key, so its maintenance silences it, like the others", () => {
|
||||
const [a] = p({}, 9);
|
||||
assert.equal(a.key, "duplicati-warnings:1:server:9");
|
||||
assert.equal(subjectOfConditionKey(a.key), "server:9");
|
||||
assert.equal(subjectOfConditionKey("duplicati-warnings:1"), null);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the address Duplicati posts to, the pages, the alerts and /metrics", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
@@ -315,6 +400,42 @@ describe("the address Duplicati posts to, the pages, the alerts and /metrics", (
|
||||
who = "admin";
|
||||
});
|
||||
|
||||
test("warnings: the page can read what they said, per job and per run; an operator can switch the alert on, and a viewer cannot", async () => {
|
||||
const docs = (await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.name, "Documents")))[0];
|
||||
await d.recordReport(token, report({ ParsedResult: "Warning", WarningsActualLength: 2, Warnings: ["file one locked", "file two locked"], EndTime: new Date().toISOString() }, "Documents"));
|
||||
who = "viewer";
|
||||
const body = (await (await call("GET", "/duplicati")).json()) as any;
|
||||
const b = body.backups.find((x: any) => x.name === "Documents");
|
||||
assert.deepEqual([b.state, b.warnings, b.warningMessages, b.alertWarnings, b.problem], ["warning", 2, ["file one locked", "file two locked"], false, null]);
|
||||
const runs = (await (await call("GET", `/duplicati/backups/${docs.id}/runs`)).json()) as any;
|
||||
assert.deepEqual(runs.runs[0].warningMessages, ["file one locked", "file two locked"]);
|
||||
assert.deepEqual(runs.runs.at(-1).warningMessages, [], "an older run that had none");
|
||||
assert.equal((await call("PATCH", `/duplicati/backups/${docs.id}`, { alertWarnings: true })).status, 403);
|
||||
who = "operator";
|
||||
assert.equal((await call("PATCH", `/duplicati/backups/${docs.id}`, { alertWarnings: "yes" })).status, 400);
|
||||
assert.equal((await call("PATCH", `/duplicati/backups/${docs.id}`, { alertWarnings: true })).status, 200);
|
||||
assert.equal((await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.id, docs.id)))[0].alertWarnings, true);
|
||||
who = "admin";
|
||||
});
|
||||
|
||||
test("a job set to alert on warnings raises one in the Backups kind, with the first message, and the page says what is wrong; switched off, it stops", async () => {
|
||||
invalidateAlertsCache();
|
||||
const a = (await getAlerts(true)).alerts.find((x) => /^backup:duplicati-warnings:\d+:server:/.test(x.id))!;
|
||||
assert.ok(a, "the alert is there");
|
||||
assert.deepEqual([a.severity, a.category, a.source, a.link], ["warning", "backup", "Duplicati", "/duplicati"]);
|
||||
assert.match(a.message, /^The backup “Documents” on Jackie finished with 2 warnings: file one locked \(and 1 more\)/);
|
||||
const body = (await (await call("GET", "/duplicati")).json()) as any;
|
||||
assert.match(body.backups.find((x: any) => x.name === "Documents").problem, /finished with 2 warnings/);
|
||||
const docs = (await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.name, "Documents")))[0];
|
||||
who = "operator";
|
||||
await call("PATCH", `/duplicati/backups/${docs.id}`, { alertWarnings: false });
|
||||
who = "admin";
|
||||
invalidateAlertsCache();
|
||||
assert.ok(!(await getAlerts(true)).alerts.some((x) => x.id.includes("duplicati-warnings")), "no alert with it switched off");
|
||||
assert.equal((await db.select().from(auditLog).where(eq(auditLog.action, "update_backup"))).some((e) => JSON.stringify(e.detail ?? "").includes("alertWarnings")), true, "the change is in the audit log");
|
||||
await d.recordReport(token, report({ EndTime: new Date().toISOString() }, "Documents"));
|
||||
});
|
||||
|
||||
test("/metrics has each backup: in order or not, how long since it last worked, and how big it is", async () => {
|
||||
const text = await buildMetrics();
|
||||
assert.match(text, /homelab_duplicati_backup_ok\{backup="Documents",source="Jackie"\} 1/);
|
||||
@@ -338,6 +459,6 @@ describe("the address Duplicati posts to, the pages, the alerts and /metrics", (
|
||||
assert.equal((await db.select().from(duplicatiBackups)).length, 0, "its backups go with it");
|
||||
assert.equal((await call("POST", `/duplicati-report/${token}`, report(), { "content-type": "text/plain" })).status, 404, "and its address stops working");
|
||||
const actions = (await db.select().from(auditLog).where(eq(auditLog.category, "duplicati"))).map((a) => a.action);
|
||||
assert.deepEqual(actions.sort(), ["create_source", "delete_source", "forget_backup", "update_backup", "update_backup", "update_source"].sort());
|
||||
assert.deepEqual(actions.sort(), ["create_source", "delete_source", "forget_backup", "update_backup", "update_backup", "update_backup", "update_backup", "update_source"].sort());
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user