Each machine's card shows whether the last report was a backup, another operation that was set aside, or one that could not be read (with the reason), and counts of each since it was added. Administrators can see the start of an unreadable report. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
552 lines
38 KiB
TypeScript
552 lines
38 KiB
TypeScript
/**
|
|
* Duplicati's reports: reading them (JSON in both shapes, and the plain text), what a report does to a backup and its history, where a backup stands, what is an alert, and the
|
|
* address Duplicati posts to and the pages around it.
|
|
*
|
|
* Run with: npm test --workspace server
|
|
*/
|
|
import { test, describe, before, after } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import type { Server } from "node:http";
|
|
import type { AddressInfo } from "node:net";
|
|
|
|
const dir = mkdtempSync(join(tmpdir(), "hm-duplicati-"));
|
|
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
|
process.env.SESSION_DIR = join(dir, "sessions");
|
|
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
|
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
|
|
|
const express = (await import("express")).default;
|
|
const { db } = await import("../src/db/client.js");
|
|
const { users, servers, duplicatiSources, duplicatiBackups, duplicatiRuns, auditLog } = await import("../src/db/schema.js");
|
|
const { runMigrations } = await import("../src/db/migrate.js");
|
|
const d = await import("../src/services/duplicati.js");
|
|
const { duplicatiReportRouter, duplicatiRouter } = await import("../src/routes/duplicati.js");
|
|
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
|
|
const { subjectOfConditionKey, startOrExtendWindow } = await import("../src/services/maintenance.js");
|
|
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
|
const { buildMetrics } = await import("../src/services/metrics.js");
|
|
const { eq } = await import("drizzle-orm");
|
|
|
|
await runMigrations();
|
|
|
|
const HOUR = 3_600_000;
|
|
const DAY = 86_400_000;
|
|
|
|
/** A report as Duplicati sends it with the output format set to Json. */
|
|
const report = (over: Record<string, unknown> = {}, name = "Documents", extra: Record<string, unknown> = {}) =>
|
|
JSON.stringify({
|
|
Data: { DeletedFiles: 1, ModifiedFiles: 3, ExaminedFiles: 1520, AddedFiles: 2, SizeOfExaminedFiles: 3221225472, MainOperation: "Backup", ParsedResult: "Success", EndTime: "2026-10-23T03:35:44.1Z", BeginTime: "2026-10-23T03:30:31.4Z", Duration: "00:05:12.7", WarningsActualLength: 0, ErrorsActualLength: 0, Warnings: [], Errors: [], ...over },
|
|
Extra: { OperationName: "Backup", "machine-id": "abc", "backup-name": name, "backup-id": "DB-1", ...extra },
|
|
});
|
|
|
|
describe("reading a report", () => {
|
|
test("a duration, in either of Duplicati's forms", () => {
|
|
assert.equal(d.parseDuration("00:05:12.7"), 312);
|
|
assert.equal(d.parseDuration("1.02:03:04"), 86_400 + 2 * 3600 + 3 * 60 + 4);
|
|
assert.equal(d.parseDuration("12:00:00"), 43_200);
|
|
for (const bad of [null, "", "5 minutes", 312, undefined]) assert.equal(d.parseDuration(bad), null);
|
|
});
|
|
|
|
test("JSON with the results under Data and the job's name among Extra", () => {
|
|
const r = d.parseReport(report())!;
|
|
assert.deepEqual([r.operation, r.result, r.backupName, r.durationSeconds, r.examinedFiles, r.sizeBytes, r.addedFiles, r.modifiedFiles, r.deletedFiles, r.warnings, r.errors, r.message], ["Backup", "Success", "Documents", 312, 1520, 3221225472, 2, 3, 1, 0, 0, null]);
|
|
assert.deepEqual([r.begin, r.end], ["2026-10-23T03:30:31.400Z", "2026-10-23T03:35:44.100Z"]);
|
|
});
|
|
|
|
test("JSON with the results at the top, and a job known only by its id or by nothing", () => {
|
|
const top = d.parseReport({ MainOperation: "Backup", ParsedResult: "Warning", WarningsActualLength: 2, Warnings: ["file locked", "another"], Extra: { "backup-name": "Photos" } })!;
|
|
assert.deepEqual([top.backupName, top.result, top.warnings, top.message], ["Photos", "Warning", 2, "file locked"]);
|
|
assert.equal(d.parseReport(report({}, "", { "backup-name": "", "backup-id": "DB-7" }))!.backupName, "Backup DB-7");
|
|
assert.equal(d.parseReport(JSON.stringify({ ParsedResult: "Success" }))!.backupName, "Unnamed backup");
|
|
});
|
|
|
|
test("a failure says why, with the first error", () => {
|
|
const r = d.parseReport(report({ ParsedResult: "Fatal", ErrorsActualLength: 1, Errors: ["The backup storage destination is missing.\nIt may be offline"] }))!;
|
|
assert.deepEqual([r.result, r.errors, r.message], ["Fatal", 1, "The backup storage destination is missing. It may be offline"]);
|
|
});
|
|
|
|
test("the plain text Duplicati sends by default", () => {
|
|
const text = ["DeletedFiles: 0", "ModifiedFiles: 3", "ExaminedFiles: 1,520", "SizeOfExaminedFiles: 3221225472 (3.00 GB)", "MainOperation: Backup", "ParsedResult: Success", "Duration: 00:05:12.7", "WarningsActualLength: 0", "backup-name: Documents"].join("\n");
|
|
const r = d.parseReport(text)!;
|
|
assert.deepEqual([r.operation, r.result, r.backupName, r.durationSeconds, r.examinedFiles, r.sizeBytes, r.modifiedFiles], ["Backup", "Success", "Documents", 312, 1520, 3221225472, 3]);
|
|
});
|
|
|
|
test("what is not a report is not read", () => {
|
|
for (const nothing of ["", "hello", "{}", "{not json", JSON.stringify({ Data: { MainOperation: "Backup" } }), null, 5, []]) assert.equal(d.parseReport(nothing as never), null, String(nothing));
|
|
assert.equal(d.parseReport(report({ MainOperation: "Compact" }))?.operation, "Compact");
|
|
});
|
|
});
|
|
|
|
describe("what a report does", () => {
|
|
let token = "";
|
|
let sourceId = 0;
|
|
|
|
before(async () => {
|
|
const made = await d.createSource({ name: "Jackie", serverId: null, createdBy: "Ada" });
|
|
token = made.token;
|
|
sourceId = made.row.id;
|
|
});
|
|
|
|
const backup = async (name = "Documents") => (await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.name, name)))[0];
|
|
|
|
test("the secret is shown once and only its hash is kept", async () => {
|
|
assert.match(token, /^hlm_d_[\w-]{20,}$/);
|
|
const [row] = await db.select().from(duplicatiSources).where(eq(duplicatiSources.id, sourceId));
|
|
assert.equal(row.tokenPrefix, token.slice(0, 12));
|
|
assert.ok(!JSON.stringify(row).includes(token), "the token itself is nowhere in the row");
|
|
assert.equal((await d.sourceForToken(token))?.id, sourceId);
|
|
for (const bad of ["", "short", `${token}x`, "x".repeat(200)]) assert.equal(await d.sourceForToken(bad), null);
|
|
});
|
|
|
|
test("a first report teaches the app the job; what it says is kept", async () => {
|
|
const t0 = new Date("2026-10-23T04:00:00Z");
|
|
assert.deepEqual(await d.recordReport(token, report(), t0), { recorded: true, backupId: (await backup())?.id ?? 1, result: "Success" });
|
|
const b = await backup();
|
|
assert.deepEqual([b.name, b.lastResult, b.durationSeconds, b.sizeBytes, b.addedFiles, b.modifiedFiles, b.deletedFiles, b.consecutiveFailures, b.expectedHours, b.alert], ["Documents", "Success", 312, 3221225472, 2, 3, 1, 0, 48, true]);
|
|
assert.equal(b.lastSuccessAt, "2026-10-23T03:35:44.100Z", "when the backup finished, from its own report");
|
|
const [src] = await db.select().from(duplicatiSources).where(eq(duplicatiSources.id, sourceId));
|
|
assert.equal(src.lastReportAt, t0.toISOString());
|
|
const runs = await db.select().from(duplicatiRuns).where(eq(duplicatiRuns.backupId, b.id));
|
|
assert.deepEqual(runs.map((r) => [r.result, r.durationSeconds, r.filesChanged]), [["Success", 312, 6]]);
|
|
});
|
|
|
|
test("failures are counted in a row, the last good one is kept, and a good run (warnings included) ends the run of failures", async () => {
|
|
const fail = (end: string) => report({ ParsedResult: "Error", ErrorsActualLength: 1, Errors: ["Unable to connect to the backend"], EndTime: end });
|
|
await d.recordReport(token, fail("2026-10-24T03:35:00Z"), new Date("2026-10-24T03:36:00Z"));
|
|
let b = await backup();
|
|
assert.deepEqual([b.lastResult, b.consecutiveFailures, b.lastSuccessAt, b.message], ["Error", 1, "2026-10-23T03:35:44.100Z", "Unable to connect to the backend"]);
|
|
await d.recordReport(token, fail("2026-10-25T03:35:00Z"), new Date("2026-10-25T03:36:00Z"));
|
|
assert.equal((await backup()).consecutiveFailures, 2);
|
|
await d.recordReport(token, report({ ParsedResult: "Warning", WarningsActualLength: 1, Warnings: ["locked file"], EndTime: "2026-10-26T03:35:00Z" }), new Date("2026-10-26T03:36:00Z"));
|
|
b = await backup();
|
|
assert.deepEqual([b.lastResult, b.consecutiveFailures, b.lastSuccessAt, b.warnings], ["Warning", 0, "2026-10-26T03:35:00.000Z", 1]);
|
|
});
|
|
|
|
test("a compact, a repair or a test is set aside; an unreadable report and an unknown address are told apart", async () => {
|
|
const before = (await db.select().from(duplicatiRuns)).length;
|
|
assert.deepEqual(await d.recordReport(token, report({ MainOperation: "Compact" })), { recorded: false, reason: "not_a_backup" });
|
|
assert.deepEqual(await d.recordReport(token, "nonsense"), { recorded: false, reason: "unreadable" });
|
|
assert.deepEqual(await d.recordReport("hlm_d_not-a-real-token-at-all", report()), { recorded: false, reason: "unknown_token" });
|
|
assert.equal((await db.select().from(duplicatiRuns)).length, before, "nothing was written down");
|
|
});
|
|
|
|
test("two jobs on one machine are two backups; the same name on another machine is another", async () => {
|
|
await d.recordReport(token, report({}, "Photos"), new Date("2026-10-26T04:00:00Z"));
|
|
const other = await d.createSource({ name: "Nova", serverId: null, createdBy: "Ada" });
|
|
await d.recordReport(other.token, report({}, "Documents"), new Date("2026-10-26T04:00:00Z"));
|
|
const all = await db.select().from(duplicatiBackups);
|
|
assert.deepEqual(all.map((b) => `${b.sourceId}:${b.name}`).sort(), [`${sourceId}:Documents`, `${sourceId}:Photos`, `${other.row.id}:Documents`].sort());
|
|
});
|
|
|
|
test("a job's history is kept at most 400 reports long, and none older than 400 days", async () => {
|
|
const b = await backup("Photos");
|
|
const base = Date.parse("2026-10-26T00:00:00Z");
|
|
for (let i = 0; i < 405; i++) await db.insert(duplicatiRuns).values({ backupId: b.id, at: new Date(base - (i + 1) * HOUR).toISOString(), result: "Success", warnings: 0, errors: 0 });
|
|
await db.insert(duplicatiRuns).values({ backupId: b.id, at: new Date(base - 500 * DAY).toISOString(), result: "Success", warnings: 0, errors: 0 });
|
|
await d.recordReport(token, report({ EndTime: "2026-10-26T05:00:00Z" }, "Photos"), new Date("2026-10-26T05:01:00Z"));
|
|
const runs = await db.select().from(duplicatiRuns).where(eq(duplicatiRuns.backupId, b.id));
|
|
assert.equal(runs.length, 400);
|
|
assert.ok(runs.every((r) => Date.parse(r.at) > base - 400 * DAY));
|
|
});
|
|
});
|
|
|
|
describe("where a backup stands, and what is an alert", () => {
|
|
const row = (extra: Partial<import("../src/services/duplicati.js").BackupRow> = {}): import("../src/services/duplicati.js").BackupRow => ({
|
|
id: 1, sourceId: 1, name: "Documents", lastReportAt: null, lastResult: "Success", lastBegin: null, lastEnd: null, durationSeconds: null, examinedFiles: null, sizeBytes: null, addedFiles: null, modifiedFiles: null, deletedFiles: null,
|
|
warnings: 0, errors: 0, message: null, warningMessages: null, alertWarnings: false, lastSuccessAt: new Date(NOW - 5 * HOUR).toISOString(), consecutiveFailures: 0, expectedHours: 48, alert: true, createdAt: "2026-10-01 00:00:00", ...extra,
|
|
});
|
|
const NOW = Date.parse("2026-10-24T12:00:00Z");
|
|
const iso = (hoursAgo: number) => new Date(NOW - hoursAgo * HOUR).toISOString();
|
|
|
|
test("ok, warning, failed and stale", () => {
|
|
assert.equal(d.backupState(row(), NOW), "ok");
|
|
assert.equal(d.backupState(row({ lastResult: "Warning" }), NOW), "warning");
|
|
assert.equal(d.backupState(row({ lastResult: "Error", consecutiveFailures: 1 }), NOW), "failed");
|
|
assert.equal(d.backupState(row({ lastResult: "Fatal" }), NOW), "failed");
|
|
assert.equal(d.backupState(row({ lastSuccessAt: iso(47) }), NOW), "ok", "inside what is expected");
|
|
assert.equal(d.backupState(row({ lastSuccessAt: iso(49) }), NOW), "stale");
|
|
assert.equal(d.backupState(row({ lastSuccessAt: iso(30), expectedHours: 24 }), NOW), "stale", "it is per job");
|
|
assert.equal(d.backupState(row({ lastResult: "Warning", lastSuccessAt: iso(60) }), NOW), "stale", "overdue beats warnings");
|
|
});
|
|
|
|
test("a failure is a warning, critical from the second in a row; a stale one is critical at three times what is expected", () => {
|
|
const sources = [{ id: 1, name: "Jackie", serverId: null }];
|
|
const p = (b: Partial<import("../src/services/duplicati.js").BackupRow>) => d.duplicatiProblems(sources, [row(b)], NOW);
|
|
assert.deepEqual(p({}), []);
|
|
const first = p({ lastResult: "Error", consecutiveFailures: 1, message: "Unable to connect to the backend", lastSuccessAt: iso(30) })[0];
|
|
assert.deepEqual([first.key, first.kind, first.severity], ["duplicati-failed:1", "failed", "warning"]);
|
|
assert.equal(first.message, "The backup “Documents” on Jackie failed: Unable to connect to the backend (the last good one was 30 h ago)");
|
|
assert.equal(p({ lastResult: "Error", consecutiveFailures: 2 })[0].severity, "critical");
|
|
assert.match(p({ lastResult: "Fatal", consecutiveFailures: 1, lastSuccessAt: null })[0].message, /there has been no good one yet/);
|
|
const stale = p({ lastSuccessAt: iso(72) })[0];
|
|
assert.deepEqual([stale.key, stale.severity], ["duplicati-stale:1", "warning"]);
|
|
assert.equal(stale.message, "The backup “Documents” on Jackie hasn't succeeded for 3 days — it should run at least every 2 days");
|
|
assert.equal(p({ lastSuccessAt: iso(150) })[0].severity, "critical");
|
|
assert.match(p({ lastSuccessAt: iso(30), expectedHours: 24 })[0].message, /every 24 h/);
|
|
});
|
|
|
|
test("one with its alert switched off says nothing; the server a machine runs on is in the key, so its maintenance can silence it", () => {
|
|
assert.deepEqual(d.duplicatiProblems([{ id: 1, name: "Jackie", serverId: null }], [row({ lastResult: "Error", alert: false })], NOW), []);
|
|
const [p] = d.duplicatiProblems([{ id: 1, name: "Jackie", serverId: 9 }], [row({ lastResult: "Error", consecutiveFailures: 1 })], NOW);
|
|
assert.equal(p.key, "duplicati-failed:1:server:9");
|
|
assert.equal(subjectOfConditionKey(p.key), "server:9");
|
|
assert.equal(subjectOfConditionKey("duplicati-stale:4:server:2"), "server:2");
|
|
assert.equal(subjectOfConditionKey("duplicati-failed:1"), null, "a machine that is not tied to a server is not silenced by one");
|
|
});
|
|
});
|
|
|
|
describe("what the last report was", () => {
|
|
test("why a body is not a report, in a sentence, with the start of it", () => {
|
|
assert.deepEqual(d.explainUnreadable(""), { reason: "the body was empty (or not text) — Duplicati should post its result as text or JSON", sample: null });
|
|
assert.equal(d.explainUnreadable(undefined).sample, null);
|
|
const cut = d.explainUnreadable('{"Data": {"MainOperation": "Backup"');
|
|
assert.match(cut.reason, /not valid JSON/);
|
|
assert.equal(cut.sample, '{"Data": {"MainOperation": "Backup"');
|
|
const noResult = d.explainUnreadable(JSON.stringify({ Data: { MainOperation: "Backup", Duration: "00:01:00" }, Extra: {} }));
|
|
assert.match(noResult.reason, /JSON without a ParsedResult field \(it has: MainOperation, Duration\)/);
|
|
assert.match(d.explainUnreadable("hello there").reason, /not JSON, and no "ParsedResult:" line/);
|
|
});
|
|
|
|
test("the sample is the start of what was sent, never more than 600 characters", () => {
|
|
assert.equal(d.explainUnreadable("x".repeat(5000)).sample?.length, 600);
|
|
});
|
|
|
|
test("the counts grow by the kind of report, and the last one is what is shown", () => {
|
|
const a = d.nextActivity(null, "t1", { outcome: "set_aside", operation: "Compact", result: "Success", job: "Docs", reason: null, sample: null });
|
|
assert.deepEqual([a.outcome, a.counts], ["set_aside", { recorded: 0, setAside: 1, unreadable: 0 }]);
|
|
const b = d.nextActivity(a, "t2", { outcome: "unreadable", operation: null, result: null, job: null, reason: "x", sample: "y" });
|
|
const c = d.nextActivity(b, "t3", { outcome: "recorded", operation: "Backup", result: "Success", job: "Docs", reason: null, sample: null });
|
|
assert.deepEqual([c.at, c.outcome, c.counts], ["t3", "recorded", { recorded: 1, setAside: 1, unreadable: 1 }]);
|
|
});
|
|
|
|
test("what is stored is read back as it was, and junk in the column is none", () => {
|
|
const a = d.nextActivity(null, "t1", { outcome: "recorded", operation: "Backup", result: "Success", job: "Docs", reason: null, sample: null });
|
|
assert.deepEqual(d.unpackActivity(JSON.stringify(a)), a);
|
|
for (const junk of [null, "", "nope", "[]", "{}", '{"outcome":"recorded"}']) assert.equal(d.unpackActivity(junk), null, String(junk));
|
|
});
|
|
|
|
describe("with reports that arrive", () => {
|
|
let token = "";
|
|
let id = 0;
|
|
before(async () => {
|
|
const made = await d.createSource({ name: "Quiet", serverId: null, createdBy: "Ada" });
|
|
token = made.token;
|
|
id = made.row.id;
|
|
});
|
|
const activity = async () => d.unpackActivity((await db.select().from(duplicatiSources).where(eq(duplicatiSources.id, id)))[0].lastReport);
|
|
|
|
test("before any report there is nothing to say", async () => {
|
|
assert.equal(await activity(), null);
|
|
});
|
|
|
|
test("a report about something other than a backup is said to be set aside, with what it was", async () => {
|
|
await d.recordReport(token, report({ MainOperation: "Compact", ParsedResult: "Success" }, "Docs"), new Date("2026-10-26T04:00:00Z"));
|
|
const a = (await activity())!;
|
|
assert.deepEqual([a.outcome, a.operation, a.result, a.job, a.counts], ["set_aside", "Compact", "Success", "Docs", { recorded: 0, setAside: 1, unreadable: 0 }]);
|
|
assert.equal((await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.sourceId, id))).length, 0, "and no job was made");
|
|
});
|
|
|
|
test("a report that cannot be read says why, and keeps the start of it", async () => {
|
|
await d.recordReport(token, "this is not what Duplicati sends", new Date("2026-10-26T05:00:00Z"));
|
|
const a = (await activity())!;
|
|
assert.deepEqual([a.outcome, a.sample, a.counts.unreadable], ["unreadable", "this is not what Duplicati sends", 1]);
|
|
assert.match(a.reason ?? "", /ParsedResult/);
|
|
});
|
|
|
|
test("a backup is said to be recorded, with the job and how it ended; the counts carry on; 'last heard' is the time of the last of any kind", async () => {
|
|
await d.recordReport(token, report({ ParsedResult: "Warning" }, "Docs"), new Date("2026-10-26T06:00:00Z"));
|
|
const a = (await activity())!;
|
|
assert.deepEqual([a.outcome, a.operation, a.result, a.job, a.counts], ["recorded", "Backup", "Warning", "Docs", { recorded: 1, setAside: 1, unreadable: 1 }]);
|
|
assert.equal((await db.select().from(duplicatiSources).where(eq(duplicatiSources.id, id)))[0].lastReportAt, "2026-10-26T06:00:00.000Z");
|
|
});
|
|
|
|
test("an address nobody knows leaves nothing behind", async () => {
|
|
const before = await activity();
|
|
assert.deepEqual(await d.recordReport("hlm_d_not-a-real-token-at-all", report()), { recorded: false, reason: "unknown_token" });
|
|
assert.deepEqual(await activity(), before);
|
|
});
|
|
});
|
|
});
|
|
|
|
describe("what the warnings said", () => {
|
|
test("a report's warnings are kept as one line each, the first twenty, each cut at 500 characters; the first is the message", () => {
|
|
const r = d.parseReport(report({ ParsedResult: "Warning", WarningsActualLength: 3, Warnings: ["Failed to process path: C:\\locked.db\n access denied", " second ", { code: "x" }] }))!;
|
|
assert.deepEqual(r.warningMessages, ["Failed to process path: C:\\locked.db access denied", "second", '{"code":"x"}']);
|
|
assert.equal(r.message, "Failed to process path: C:\\locked.db access denied");
|
|
const many = d.parseReport(report({ ParsedResult: "Warning", WarningsActualLength: 90, Warnings: Array.from({ length: 90 }, (_, i) => `w${i}`) }))!;
|
|
assert.deepEqual([many.warnings, many.warningMessages.length, many.warningMessages[19]], [90, 20, "w19"]);
|
|
assert.equal(d.parseReport(report({ ParsedResult: "Warning", Warnings: ["x".repeat(900)] }))!.warningMessages[0].length, 500);
|
|
});
|
|
|
|
test("empty lines, things that are not a list and the plain text give none", () => {
|
|
assert.deepEqual(d.messagesOf(["", " ", null, 5, "real"]), ["real"]);
|
|
for (const nothing of [undefined, null, "text", {}, 7]) assert.deepEqual(d.messagesOf(nothing), []);
|
|
assert.deepEqual(d.parseReport(["MainOperation: Backup", "ParsedResult: Warning", "WarningsActualLength: 2", "backup-name: Docs"].join("\n"))!.warningMessages, []);
|
|
assert.deepEqual(d.parseReport(report())!.warningMessages, []);
|
|
});
|
|
|
|
test("what is stored is a list read back as a list, whatever is in the column", () => {
|
|
assert.equal(d.packMessages([]), null);
|
|
assert.deepEqual(d.unpackMessages(d.packMessages(["a", "b"])), ["a", "b"]);
|
|
for (const junk of [null, "", "not json", "{}", "5", '["ok", 5, null]']) assert.ok(Array.isArray(d.unpackMessages(junk)), String(junk));
|
|
assert.deepEqual(d.unpackMessages('["ok", 5, null]'), ["ok"]);
|
|
});
|
|
|
|
describe("with a report that arrives", () => {
|
|
let token = "";
|
|
before(async () => {
|
|
token = (await d.createSource({ name: "Warny", serverId: null, createdBy: "Ada" })).token;
|
|
});
|
|
const job = async () => (await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.name, "Warned")))[0];
|
|
|
|
test("the messages go with the run and with the job; a later clean run empties the job's, and the history keeps what was said then", async () => {
|
|
await d.recordReport(token, report({ ParsedResult: "Warning", WarningsActualLength: 2, Warnings: ["file one locked", "file two locked"], EndTime: "2026-10-26T03:35:00Z" }, "Warned"), new Date("2026-10-26T03:36:00Z"));
|
|
const b = await job();
|
|
assert.deepEqual([b.warnings, d.unpackMessages(b.warningMessages)], [2, ["file one locked", "file two locked"]]);
|
|
await d.recordReport(token, report({ EndTime: "2026-10-27T03:35:00Z" }, "Warned"), new Date("2026-10-27T03:36:00Z"));
|
|
const after = await job();
|
|
assert.deepEqual([after.warnings, after.warningMessages], [0, null]);
|
|
const runs = await db.select().from(duplicatiRuns).where(eq(duplicatiRuns.backupId, b.id));
|
|
assert.deepEqual(runs.map((r) => [r.result, d.unpackMessages(r.warningMessages)]).sort(), [["Success", []], ["Warning", ["file one locked", "file two locked"]]]);
|
|
});
|
|
|
|
test("a job is set not to alert on warnings until somebody switches it on", async () => {
|
|
assert.equal((await job()).alertWarnings, false);
|
|
});
|
|
});
|
|
});
|
|
|
|
describe("a warning as an alert", () => {
|
|
const NOW = Date.parse("2026-10-24T12:00:00Z");
|
|
const HR = 3_600_000;
|
|
const row = (extra: Partial<import("../src/services/duplicati.js").BackupRow> = {}): import("../src/services/duplicati.js").BackupRow => ({
|
|
id: 1, sourceId: 1, name: "Documents", lastReportAt: null, lastResult: "Warning", lastBegin: null, lastEnd: null, durationSeconds: null, examinedFiles: null, sizeBytes: null, addedFiles: null, modifiedFiles: null, deletedFiles: null,
|
|
warnings: 2, errors: 0, message: "file one locked", warningMessages: JSON.stringify(["file one locked", "file two locked"]), alertWarnings: true, lastSuccessAt: new Date(NOW - 5 * HR).toISOString(), consecutiveFailures: 0, expectedHours: 48, alert: true, createdAt: "2026-10-01 00:00:00", ...extra,
|
|
});
|
|
const p = (b: Partial<import("../src/services/duplicati.js").BackupRow>, serverId: number | null = null) => d.duplicatiProblems([{ id: 1, name: "Jackie", serverId }], [row(b)], NOW);
|
|
|
|
test("a run with warnings is an alert only for a job set to alert on them: a warning, with the count and the first thing it said", () => {
|
|
assert.deepEqual(p({ alertWarnings: false }), []);
|
|
const [a] = p({});
|
|
assert.deepEqual([a.key, a.kind, a.severity], ["duplicati-warnings:1", "warnings", "warning"]);
|
|
assert.equal(a.message, "The backup “Documents” on Jackie finished with 2 warnings: file one locked (and 1 more)");
|
|
});
|
|
|
|
test("one warning says so in the singular and has no 'more'; with no text it still says how many", () => {
|
|
assert.equal(p({ warnings: 1, warningMessages: JSON.stringify(["only one"]) })[0].message, "The backup “Documents” on Jackie finished with 1 warning: only one");
|
|
assert.equal(p({ warningMessages: null, message: null })[0].message, "The backup “Documents” on Jackie finished with 2 warnings");
|
|
assert.ok(p({ warningMessages: JSON.stringify(["y".repeat(400)]) })[0].message.length < 330, "a long message is cut");
|
|
});
|
|
|
|
test("a clean run, a failed one, an overdue one and a muted job do not raise it; the failure or the lateness is the alert", () => {
|
|
assert.deepEqual(p({ lastResult: "Success", warnings: 0, warningMessages: null }), []);
|
|
assert.deepEqual(p({ lastResult: "Error", consecutiveFailures: 1 }).map((x) => x.kind), ["failed"]);
|
|
assert.deepEqual(p({ lastSuccessAt: new Date(NOW - 80 * HR).toISOString() }).map((x) => x.kind), ["stale"]);
|
|
assert.deepEqual(p({ alert: false }), [], "a job muted entirely says nothing, warnings included");
|
|
});
|
|
|
|
test("the server a machine runs on is in the key, so its maintenance silences it, like the others", () => {
|
|
const [a] = p({}, 9);
|
|
assert.equal(a.key, "duplicati-warnings:1:server:9");
|
|
assert.equal(subjectOfConditionKey(a.key), "server:9");
|
|
assert.equal(subjectOfConditionKey("duplicati-warnings:1"), null);
|
|
});
|
|
});
|
|
|
|
describe("the address Duplicati posts to, the pages, the alerts and /metrics", () => {
|
|
let app: Server;
|
|
let base = "";
|
|
let who = "admin";
|
|
let token = "";
|
|
let serverId = 0;
|
|
|
|
before(async () => {
|
|
// what the earlier groups made is not this group's: a clean start (a machine takes its backups and their history with it)
|
|
await db.delete(duplicatiSources);
|
|
await db.insert(users).values([
|
|
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
|
{ oidcSub: "operator", name: "Olga", role: "operator" },
|
|
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
|
]);
|
|
serverId = (await db.insert(servers).values({ name: "Jackie", apiTokenHash: "j", apiTokenPrefix: "j", lastSeenAt: new Date().toISOString() }).returning({ id: servers.id }))[0].id;
|
|
const a = express();
|
|
// the address Duplicati posts to has no session and is mounted first, as in the app
|
|
a.use("/api/duplicati-report", duplicatiReportRouter);
|
|
a.use(express.json());
|
|
a.use((req, _res, next) => {
|
|
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
|
next();
|
|
});
|
|
a.use("/api/duplicati", duplicatiRouter);
|
|
app = await new Promise<Server>((resolve) => {
|
|
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
|
});
|
|
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
|
|
});
|
|
after(() => {
|
|
app.close();
|
|
try {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
} catch {
|
|
// Windows keeps the database file locked until the process exits.
|
|
}
|
|
});
|
|
|
|
const call = (method: string, path: string, body?: unknown, headers: Record<string, string> = {}) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json", connection: "close", ...headers }, body: body === undefined ? undefined : typeof body === "string" ? body : JSON.stringify(body) });
|
|
|
|
test("an administrator makes a machine and is shown its secret address once; nobody else can, and it is in the audit log", async () => {
|
|
who = "operator";
|
|
assert.equal((await call("POST", "/duplicati/sources", { name: "Nope" })).status, 403);
|
|
who = "admin";
|
|
for (const bad of [{}, { name: " " }, { name: "x", serverId: 99999 }]) assert.equal((await call("POST", "/duplicati/sources", bad)).status, 400, JSON.stringify(bad));
|
|
const res = await call("POST", "/duplicati/sources", { name: "Jackie", serverId });
|
|
assert.equal(res.status, 201);
|
|
const made = (await res.json()) as { source: Record<string, unknown>; token: string; path: string };
|
|
token = made.token;
|
|
assert.equal(made.path, `/api/duplicati-report/${token}`);
|
|
assert.ok(!("tokenHash" in made.source));
|
|
const list = JSON.stringify(await (await call("GET", "/duplicati")).json());
|
|
assert.ok(!list.includes(token) && !list.includes("tokenHash"), "never shown again");
|
|
assert.equal((await db.select().from(auditLog).where(eq(auditLog.category, "duplicati"))).length, 1);
|
|
});
|
|
|
|
test("Duplicati posts to the address with no sign-in: JSON or text; a wrong address is not found, and a GET is refused", async () => {
|
|
const url = `/duplicati-report/${token}`;
|
|
const good = await call("POST", url, report({ EndTime: new Date().toISOString() }, "Documents"), { "content-type": "text/plain" });
|
|
assert.deepEqual([good.status, await good.text()], [200, "OK"]);
|
|
const text = await call("POST", url, ["MainOperation: Backup", "ParsedResult: Success", "Duration: 00:01:00", "backup-name: Photos"].join("\n"), { "content-type": "text/plain" });
|
|
assert.equal(text.status, 200);
|
|
const compact = await call("POST", url, report({ MainOperation: "Compact" }), { "content-type": "text/plain" });
|
|
assert.match(await compact.text(), /not a backup/);
|
|
assert.match(await (await call("POST", url, "rubbish", { "content-type": "text/plain" })).text(), /not a report this app can read/);
|
|
assert.equal((await call("POST", "/duplicati-report/hlm_d_this-token-does-not-exist-1234", report())).status, 404);
|
|
assert.equal((await call("GET", url)).status, 405);
|
|
assert.equal((await db.select().from(duplicatiBackups)).length, 2, "Documents and Photos");
|
|
});
|
|
|
|
test("anyone signed in sees each backup, where it stands and what is wrong; the history of one is there too", async () => {
|
|
who = "viewer";
|
|
await d.recordReport(token, report({ ParsedResult: "Fatal", ErrorsActualLength: 1, Errors: ["destination is missing"], EndTime: new Date().toISOString() }, "Photos"));
|
|
const body = (await (await call("GET", "/duplicati")).json()) as any;
|
|
assert.deepEqual(body.sources.map((s: any) => [s.name, s.serverName, s.backups]), [["Jackie", "Jackie", 2]]);
|
|
const photos = body.backups.find((b: any) => b.name === "Photos");
|
|
assert.deepEqual([photos.state, photos.sourceName, /destination is missing/.test(photos.problem)], ["failed", "Jackie", true]);
|
|
assert.equal(body.backups.find((b: any) => b.name === "Documents").state, "ok");
|
|
const runs = (await (await call("GET", `/duplicati/backups/${photos.id}/runs`)).json()) as any;
|
|
assert.deepEqual([runs.backup.name, runs.runs.length, runs.runs[0].result], ["Photos", 2, "Fatal"]);
|
|
assert.equal((await call("GET", "/duplicati/backups/999/runs")).status, 404);
|
|
who = "admin";
|
|
});
|
|
|
|
test("a failed backup is an alert in the Backups kind, with where to look; maintenance on its server silences it", async () => {
|
|
invalidateAlertsCache();
|
|
const report1 = await getAlerts(true);
|
|
const a = report1.alerts.find((x) => /^backup:duplicati-failed:\d+:server:/.test(x.id))!;
|
|
assert.ok(a, "the alert is there");
|
|
assert.deepEqual([a.severity, a.category, a.source, a.link, a.silenced], ["warning", "backup", "Duplicati", "/duplicati", false]);
|
|
assert.match(a.message, /^The backup “Photos” on Jackie failed: destination is missing/);
|
|
await startOrExtendWindow("server", serverId, 60, "test", "t");
|
|
invalidateAlertsCache();
|
|
assert.equal((await getAlerts(true)).alerts.find((x) => x.id === a.id)?.silenced, true);
|
|
const { snapshot } = await collectSnapshot();
|
|
assert.ok(snapshot.duplicati?.some((p) => p.kind === "failed"), "the health pass reads it too");
|
|
const list = evaluateHealth({ servers: [], proxmox: [], synology: [], duplicati: snapshot.duplicati }, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now());
|
|
assert.ok(list.some((c) => c.source === "duplicati" && c.key.startsWith("duplicati-failed:")));
|
|
});
|
|
|
|
test("an operator sets how often a backup is expected and whether it alerts; a viewer cannot, and a wrong number is refused", async () => {
|
|
const photos = (await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.name, "Photos")))[0];
|
|
who = "viewer";
|
|
assert.equal((await call("PATCH", `/duplicati/backups/${photos.id}`, { expectedHours: 24 })).status, 403);
|
|
who = "operator";
|
|
assert.equal((await call("PATCH", `/duplicati/backups/${photos.id}`, { expectedHours: 168, alert: false })).status, 200);
|
|
const after = (await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.id, photos.id)))[0];
|
|
assert.deepEqual([after.expectedHours, after.alert], [168, false]);
|
|
for (const bad of [{ expectedHours: 0 }, { expectedHours: 5000 }, { expectedHours: 1.5 }]) assert.equal((await call("PATCH", `/duplicati/backups/${photos.id}`, bad)).status, 400, JSON.stringify(bad));
|
|
assert.equal((await call("PATCH", "/duplicati/backups/999", { alert: true })).status, 404);
|
|
invalidateAlertsCache();
|
|
assert.ok(!(await getAlerts(true)).alerts.some((x) => x.id.includes("duplicati-failed")), "no alert with it switched off");
|
|
await call("PATCH", `/duplicati/backups/${photos.id}`, { alert: true });
|
|
who = "admin";
|
|
});
|
|
|
|
test("warnings: the page can read what they said, per job and per run; an operator can switch the alert on, and a viewer cannot", async () => {
|
|
const docs = (await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.name, "Documents")))[0];
|
|
await d.recordReport(token, report({ ParsedResult: "Warning", WarningsActualLength: 2, Warnings: ["file one locked", "file two locked"], EndTime: new Date().toISOString() }, "Documents"));
|
|
who = "viewer";
|
|
const body = (await (await call("GET", "/duplicati")).json()) as any;
|
|
const b = body.backups.find((x: any) => x.name === "Documents");
|
|
assert.deepEqual([b.state, b.warnings, b.warningMessages, b.alertWarnings, b.problem], ["warning", 2, ["file one locked", "file two locked"], false, null]);
|
|
const runs = (await (await call("GET", `/duplicati/backups/${docs.id}/runs`)).json()) as any;
|
|
assert.deepEqual(runs.runs[0].warningMessages, ["file one locked", "file two locked"]);
|
|
assert.deepEqual(runs.runs.at(-1).warningMessages, [], "an older run that had none");
|
|
assert.equal((await call("PATCH", `/duplicati/backups/${docs.id}`, { alertWarnings: true })).status, 403);
|
|
who = "operator";
|
|
assert.equal((await call("PATCH", `/duplicati/backups/${docs.id}`, { alertWarnings: "yes" })).status, 400);
|
|
assert.equal((await call("PATCH", `/duplicati/backups/${docs.id}`, { alertWarnings: true })).status, 200);
|
|
assert.equal((await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.id, docs.id)))[0].alertWarnings, true);
|
|
who = "admin";
|
|
});
|
|
|
|
test("the page says what the last report was; only an administrator is given the start of one that could not be read", async () => {
|
|
await call("POST", `/duplicati-report/${token}`, "something that is not a report at all", { "content-type": "text/plain" });
|
|
const source = async () => ((await (await call("GET", "/duplicati")).json()) as any).sources[0];
|
|
who = "admin";
|
|
const asAdmin = await source();
|
|
assert.deepEqual([asAdmin.lastReport.outcome, asAdmin.lastReport.sample, asAdmin.lastReport.counts.unreadable >= 1], ["unreadable", "something that is not a report at all", true]);
|
|
for (const role of ["operator", "viewer"]) {
|
|
who = role;
|
|
const other = await source();
|
|
assert.deepEqual([other.lastReport.outcome, other.lastReport.sample, /ParsedResult/.test(other.lastReport.reason)], ["unreadable", null, true], role);
|
|
}
|
|
who = "admin";
|
|
});
|
|
|
|
test("a job set to alert on warnings raises one in the Backups kind, with the first message, and the page says what is wrong; switched off, it stops", async () => {
|
|
invalidateAlertsCache();
|
|
const a = (await getAlerts(true)).alerts.find((x) => /^backup:duplicati-warnings:\d+:server:/.test(x.id))!;
|
|
assert.ok(a, "the alert is there");
|
|
assert.deepEqual([a.severity, a.category, a.source, a.link], ["warning", "backup", "Duplicati", "/duplicati"]);
|
|
assert.match(a.message, /^The backup “Documents” on Jackie finished with 2 warnings: file one locked \(and 1 more\)/);
|
|
const body = (await (await call("GET", "/duplicati")).json()) as any;
|
|
assert.match(body.backups.find((x: any) => x.name === "Documents").problem, /finished with 2 warnings/);
|
|
const docs = (await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.name, "Documents")))[0];
|
|
who = "operator";
|
|
await call("PATCH", `/duplicati/backups/${docs.id}`, { alertWarnings: false });
|
|
who = "admin";
|
|
invalidateAlertsCache();
|
|
assert.ok(!(await getAlerts(true)).alerts.some((x) => x.id.includes("duplicati-warnings")), "no alert with it switched off");
|
|
assert.equal((await db.select().from(auditLog).where(eq(auditLog.action, "update_backup"))).some((e) => JSON.stringify(e.detail ?? "").includes("alertWarnings")), true, "the change is in the audit log");
|
|
await d.recordReport(token, report({ EndTime: new Date().toISOString() }, "Documents"));
|
|
});
|
|
|
|
test("/metrics has each backup: in order or not, how long since it last worked, and how big it is", async () => {
|
|
const text = await buildMetrics();
|
|
assert.match(text, /homelab_duplicati_backup_ok\{backup="Documents",source="Jackie"\} 1/);
|
|
assert.match(text, /homelab_duplicati_backup_ok\{backup="Photos",source="Jackie"\} 0/);
|
|
assert.match(text, /homelab_duplicati_last_success_age_seconds\{backup="Documents",source="Jackie"\} \d+/);
|
|
assert.match(text, /homelab_duplicati_source_size_bytes\{backup="Documents",source="Jackie"\} 3221225472/);
|
|
});
|
|
|
|
test("a backup can be forgotten, and a machine removed with all of them; both are for people who may, and in the audit log", async () => {
|
|
const photos = (await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.name, "Photos")))[0];
|
|
who = "viewer";
|
|
assert.equal((await call("DELETE", `/duplicati/backups/${photos.id}`)).status, 403);
|
|
who = "operator";
|
|
assert.equal((await call("DELETE", `/duplicati/backups/${photos.id}`)).status, 204);
|
|
assert.equal((await db.select().from(duplicatiRuns).where(eq(duplicatiRuns.backupId, photos.id))).length, 0, "its history goes with it");
|
|
assert.equal((await call("DELETE", `/duplicati/sources/${(await db.select().from(duplicatiSources))[0].id}`)).status, 403, "removing a machine is an administrator's");
|
|
who = "admin";
|
|
const [src] = await db.select().from(duplicatiSources);
|
|
assert.equal((await call("PATCH", `/duplicati/sources/${src.id}`, { name: "Jackie (docker)", serverId })).status, 200);
|
|
assert.equal((await call("DELETE", `/duplicati/sources/${src.id}`)).status, 204);
|
|
assert.equal((await db.select().from(duplicatiBackups)).length, 0, "its backups go with it");
|
|
assert.equal((await call("POST", `/duplicati-report/${token}`, report(), { "content-type": "text/plain" })).status, 404, "and its address stops working");
|
|
const actions = (await db.select().from(auditLog).where(eq(auditLog.category, "duplicati"))).map((a) => a.action);
|
|
assert.deepEqual(actions.sort(), ["create_source", "delete_source", "forget_backup", "update_backup", "update_backup", "update_backup", "update_backup", "update_source"].sort());
|
|
});
|
|
});
|