Show what Duplicati's warnings said, and let a job alert on them
The first twenty warning messages of each run are kept with the run and shown on the job and in its history. A job can be set to raise a warning-level alert when its last run finished with warnings (off by default; a muted job raises nothing; maintenance on the machine's server silences it). Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d78aba8bf9
commit
64961b1141
14 files changed
+9241
-24
No files matched your search
@@ -156,7 +156,7 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
|
||||
if (m) return { type: m[1], id: Number(m[3]), category: m[2] === "disk" ? "disk" : "services" };
|
||||
m = /^fragnet-(disk|memory|backup):(\d+):/.exec(key);
|
||||
if (m) return { type: "fragnet", id: Number(m[2]), category: m[1] === "disk" ? "disk" : m[1] === "backup" ? "backup" : "monitoring" };
|
||||
m = /^duplicati-(?:failed|stale):(\d+)/.exec(key);
|
||||
m = /^duplicati-(?:failed|stale|warnings):(\d+)/.exec(key);
|
||||
if (m) return { type: "duplicati", id: Number(m[1]), category: "backup" };
|
||||
m = /^nextcloud-(?:maintenance|upgrade|disk|cron|checks):(\d+)/.exec(key);
|
||||
if (m) return { type: "nextcloud", id: Number(m[1]), category: key.startsWith("nextcloud-disk") ? "disk" : "services" };
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
* the source was, what changed — is kept with a history.
|
||||
*
|
||||
* What makes an alert: a backup whose last run **failed** (Error or Fatal; critical from the second failure in a row), and one that has **not succeeded for longer than it is
|
||||
* expected to** (a day or two by default, set per job; critical at three times that). A run that finished with warnings still counts as a backup. Only runs of the *Backup*
|
||||
* expected to** (a day or two by default, set per job; critical at three times that). A run that finished with warnings still counts as a backup — and says what the warnings were (the first
|
||||
* twenty, kept with the run) — and a job can be set to **alert when its last run had warnings** (off until switched on: some jobs always have a few). Only runs of the *Backup*
|
||||
* operation count: a compact, a repair or a verification does not say whether the data is safe.
|
||||
*/
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
@@ -43,8 +44,13 @@ export interface ParsedReport {
|
||||
warnings: number;
|
||||
errors: number;
|
||||
message: string | null;
|
||||
/** What the warnings said, one line each (up to MAX_WARNING_MESSAGES, each cut at MAX_WARNING_LENGTH). Only the JSON format carries them. */
|
||||
warningMessages: string[];
|
||||
}
|
||||
|
||||
export const MAX_WARNING_MESSAGES = 20;
|
||||
export const MAX_WARNING_LENGTH = 500;
|
||||
|
||||
/** "00:05:12.7" or "1.02:03:04" in seconds. */
|
||||
export function parseDuration(text: unknown): number | null {
|
||||
if (typeof text !== "string") return null;
|
||||
@@ -66,6 +72,29 @@ const lineOf = (v: unknown): string | null => {
|
||||
const text = typeof v === "string" ? v : v !== null && typeof v === "object" ? JSON.stringify(v) : null;
|
||||
return text ? text.replace(/\s+/g, " ").trim().slice(0, 300) : null;
|
||||
};
|
||||
/** Pure: the messages of a list of warnings, one line each; the empty ones left out. */
|
||||
export function messagesOf(list: unknown): string[] {
|
||||
const out: string[] = [];
|
||||
for (const item of Array.isArray(list) ? list : []) {
|
||||
const text = typeof item === "string" ? item : item !== null && typeof item === "object" ? JSON.stringify(item) : null;
|
||||
const line = text ? text.replace(/\s+/g, " ").trim().slice(0, MAX_WARNING_LENGTH) : "";
|
||||
if (line) out.push(line);
|
||||
if (out.length >= MAX_WARNING_MESSAGES) break;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** What is stored of a list of messages (null for none), and what is read back of it (a list, whatever is in the column). */
|
||||
export const packMessages = (m: string[]): string | null => (m.length > 0 ? JSON.stringify(m) : null);
|
||||
export function unpackMessages(stored: string | null): string[] {
|
||||
if (!stored) return [];
|
||||
try {
|
||||
const v = JSON.parse(stored);
|
||||
return Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure: the report Duplicati sent — JSON (`send-http-result-output-format=Json`: the results at the top, or under `Data`, with the job's name among `Extra`), or the plain text it sends
|
||||
@@ -107,6 +136,7 @@ export function parseReport(body: unknown): ParsedReport | null {
|
||||
warnings: asInt(data.WarningsActualLength) ?? warnings.length,
|
||||
errors: asInt(data.ErrorsActualLength) ?? errors.length,
|
||||
message: lineOf(errors[0]) ?? lineOf(warnings[0]),
|
||||
warningMessages: messagesOf(warnings),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -130,6 +160,7 @@ function parseText(text: string): ParsedReport | null {
|
||||
warnings: asInt(value("WarningsActualLength")?.match(/^[\d,]+/)?.[0]) ?? 0,
|
||||
errors: asInt(value("ErrorsActualLength")?.match(/^[\d,]+/)?.[0]) ?? 0,
|
||||
message: null,
|
||||
warningMessages: [],
|
||||
};
|
||||
}
|
||||
|
||||
@@ -178,6 +209,7 @@ export async function recordReport(token: string, body: unknown, now: Date = new
|
||||
warnings: report.warnings,
|
||||
errors: report.errors,
|
||||
message: report.message,
|
||||
warningMessages: packMessages(report.warningMessages),
|
||||
lastSuccessAt: good ? at : (before?.lastSuccessAt ?? null),
|
||||
consecutiveFailures: good ? 0 : (before?.consecutiveFailures ?? 0) + 1,
|
||||
};
|
||||
@@ -189,7 +221,7 @@ export async function recordReport(token: string, body: unknown, now: Date = new
|
||||
id = (await db.insert(duplicatiBackups).values({ sourceId: source.id, name: report.backupName, ...values }).returning({ id: duplicatiBackups.id }))[0].id;
|
||||
}
|
||||
const changed = (report.addedFiles ?? 0) + (report.modifiedFiles ?? 0) + (report.deletedFiles ?? 0);
|
||||
await db.insert(duplicatiRuns).values({ backupId: id, at, result: report.result, durationSeconds: report.durationSeconds, sizeBytes: report.sizeBytes, filesChanged: report.addedFiles === null && report.modifiedFiles === null && report.deletedFiles === null ? null : changed, warnings: report.warnings, errors: report.errors });
|
||||
await db.insert(duplicatiRuns).values({ backupId: id, at, result: report.result, durationSeconds: report.durationSeconds, sizeBytes: report.sizeBytes, filesChanged: report.addedFiles === null && report.modifiedFiles === null && report.deletedFiles === null ? null : changed, warnings: report.warnings, errors: report.errors, warningMessages: packMessages(report.warningMessages) });
|
||||
await pruneRuns(id, now);
|
||||
return { recorded: true, backupId: id, result: report.result };
|
||||
}
|
||||
@@ -219,7 +251,7 @@ const ago = (ms: number) => {
|
||||
|
||||
export interface DuplicatiProblem {
|
||||
key: string;
|
||||
kind: "failed" | "stale";
|
||||
kind: "failed" | "stale" | "warnings";
|
||||
severity: "critical" | "warning";
|
||||
message: string;
|
||||
}
|
||||
@@ -239,6 +271,11 @@ export function duplicatiProblems(sources: Pick<SourceRow, "id" | "name" | "serv
|
||||
} else if (state === "stale") {
|
||||
const left = nowMs - Date.parse(b.lastSuccessAt as string);
|
||||
out.push({ key: `duplicati-stale:${b.id}${suffix}`, kind: "stale", severity: left > 3 * b.expectedHours * HOUR_MS ? "critical" : "warning", message: `The backup “${b.name}” on ${source.name} hasn't succeeded for ${ago(left)} — it should run at least every ${b.expectedHours >= 48 && b.expectedHours % 24 === 0 ? `${b.expectedHours / 24} days` : `${b.expectedHours} h`}` });
|
||||
} else if (state === "warning" && b.alertWarnings) {
|
||||
const said = unpackMessages(b.warningMessages);
|
||||
const first = said[0] ?? b.message;
|
||||
const more = b.warnings > 1 ? ` (and ${b.warnings - 1} more)` : "";
|
||||
out.push({ key: `duplicati-warnings:${b.id}${suffix}`, kind: "warnings", severity: "warning", message: `The backup “${b.name}” on ${source.name} finished with ${b.warnings} warning${b.warnings === 1 ? "" : "s"}${first ? `: ${first.slice(0, 200)}${more}` : ""}` });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
|
||||
@@ -69,7 +69,7 @@ export function subjectOfConditionKey(key: string): string | null {
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^authentik-(?:outpost|outpost-version|cert|token|worker|logins|errors):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^duplicati-(?:failed|stale):\d+:server:(\d+)$/.exec(key);
|
||||
m = /^duplicati-(?:failed|stale|warnings):\d+:server:(\d+)$/.exec(key);
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:(\d+)$/.exec(key);
|
||||
if (m) return `server:${m[1]}`;
|
||||
|
||||
Reference in new issue
Block a user