Say what the last Duplicati report was, so a machine with no jobs explains itself

Each machine's card shows whether the last report was a backup, another
operation that was set aside, or one that could not be read (with the reason),
and counts of each since it was added. Administrators can see the start of an
unreadable report.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-11-09 23:33:03 +01:00
1 parent 64961b1141
commit e517de35af
11 files changed
+9248 -9

No files matched your search

@@ -0,0 +1 @@
ALTER TABLE `duplicati_sources` ADD `last_report` text;
File diff suppressed because it is too large. Load diff
+7
View File
@@ -624,6 +624,13 @@
"when": 1794262055100,
"tag": "0088_grey_silhouette",
"breakpoints": true
},
{
"idx": 89,
"version": "6",
"when": 1794263161221,
"tag": "0089_military_tombstone",
"breakpoints": true
}
]
}
+1
View File
@@ -1675,6 +1675,7 @@ export const duplicatiSources = sqliteTable("duplicati_sources", {
.notNull()
.default(sql`(current_timestamp)`),
lastReportAt: text("last_report_at"), // ISO
lastReport: text("last_report"), // JSON: what the last report turned out to be, and how many of each kind there have been (see services/duplicati.ts, SourceActivity)
});
// One backup job of one source, as its last report left it. A job is learned from its first report (by name) and is expected to succeed every `expectedHours`.
+2 -2
View File
@@ -36,13 +36,13 @@ duplicatiRouter.use(requireAuth);
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
/** Every machine reporting, every backup job and where it stands: for everyone signed in, like the backup pages. */
duplicatiRouter.get("/", asyncHandler(async (_req, res) => {
duplicatiRouter.get("/", asyncHandler(async (req, res) => {
const { sources, backups } = await listAll();
const now = Date.now();
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((s) => [s.id, s.name]));
const problems = duplicatiProblems(sources, backups, now);
res.json({
sources: sources.map((s) => ({ ...publicSource(s), serverName: s.serverId !== null ? (serverNames.get(s.serverId) ?? null) : null, backups: backups.filter((b) => b.sourceId === s.id).length })),
sources: sources.map((s) => ({ ...publicSource(s, req.currentUser?.role === "admin"), serverName: s.serverId !== null ? (serverNames.get(s.serverId) ?? null) : null, backups: backups.filter((b) => b.sourceId === s.id).length })),
backups: backups.map((b) => ({ ...b, warningMessages: unpackMessages(b.warningMessages), state: backupState(b, now), sourceName: sources.find((s) => s.id === b.sourceId)?.name ?? "", problem: problems.find((p) => new RegExp(`^duplicati-(failed|stale|warnings):${b.id}(:|$)`).test(p.key))?.message ?? null })),
servers: [...serverNames].map(([id, name]) => ({ id, name })).sort((a, b) => a.name.localeCompare(b.name)),
defaultExpectedHours: DEFAULT_EXPECTED_HOURS,
+78 -5
View File
@@ -180,6 +180,66 @@ export async function sourceForToken(token: string): Promise<SourceRow | null> {
return (await db.select().from(duplicatiSources).where(eq(duplicatiSources.tokenHash, hash(token))).limit(1))[0] ?? null;
}
/** What the last report on an address turned out to be, so the page can say why a machine that is being heard has no jobs. */
export interface SourceActivity {
/** When it arrived (ISO). */
at: string;
/** "recorded": a backup, written down. "set_aside": a report about something other than a backup. "unreadable": not a report this app can use. */
outcome: "recorded" | "set_aside" | "unreadable";
/** The operation it was about (Backup, Compact, Repair …), when it said. */
operation: string | null;
/** How it ended (Success, Warning …), when it said. */
result: string | null;
/** The job it was about, when it said. */
job: string | null;
/** For an unreadable one: what was wrong with it, in a sentence. */
reason: string | null;
/** For an unreadable one: the start of what was sent, to see its form (shown to administrators only). */
sample: string | null;
/** How many reports of each kind have come since the machine was added. */
counts: { recorded: number; setAside: number; unreadable: number };
}
export const MAX_SAMPLE = 600;
/** Pure: why a body is not a report this app can use, in a sentence, and the start of it. Read for the same two forms parseReport reads. */
export function explainUnreadable(body: unknown): { reason: string; sample: string | null } {
const text = typeof body === "string" ? body.trim() : "";
if (text === "") return { reason: "the body was empty (or not text) — Duplicati should post its result as text or JSON", sample: null };
const sample = text.slice(0, MAX_SAMPLE);
if (text.startsWith("{")) {
let payload: Record<string, unknown> | null = null;
try {
payload = obj(JSON.parse(text));
} catch {
return { reason: "it looks like JSON but is not valid JSON (cut short, perhaps)", sample };
}
const data = Object.keys(obj(payload.Data)).length > 0 ? obj(payload.Data) : payload;
const keys = Object.keys(data).slice(0, 8).join(", ");
return { reason: `JSON without a ParsedResult field (it has: ${keys || "nothing"})`, sample };
}
return { reason: "not JSON, and no \"ParsedResult:\" line in it — set send-http-result-output-format=Json (or leave the default text, which has that line)", sample };
}
/** Pure: the activity after one more report. */
export function nextActivity(previous: SourceActivity | null, at: string, report: Pick<SourceActivity, "outcome" | "operation" | "result" | "job" | "reason" | "sample">): SourceActivity {
const counts = { recorded: previous?.counts.recorded ?? 0, setAside: previous?.counts.setAside ?? 0, unreadable: previous?.counts.unreadable ?? 0 };
if (report.outcome === "recorded") counts.recorded++;
else if (report.outcome === "set_aside") counts.setAside++;
else counts.unreadable++;
return { at, ...report, counts };
}
export function unpackActivity(stored: string | null): SourceActivity | null {
if (!stored) return null;
try {
const v = JSON.parse(stored) as SourceActivity;
return v && typeof v === "object" && typeof v.outcome === "string" && v.counts ? v : null;
} catch {
return null;
}
}
export type RecordOutcome = { recorded: true; backupId: number; result: string } | { recorded: false; reason: "unknown_token" | "unreadable" | "not_a_backup" };
/** Reads a report that arrived on a secret address and writes it down. Other operations than a backup are said to have been seen and set aside. */
@@ -187,10 +247,22 @@ export async function recordReport(token: string, body: unknown, now: Date = new
const source = await sourceForToken(token);
if (!source) return { recorded: false, reason: "unknown_token" };
const nowIso = now.toISOString();
await db.update(duplicatiSources).set({ lastReportAt: nowIso }).where(eq(duplicatiSources.id, source.id));
const previous = unpackActivity(source.lastReport);
/** Writes down that a report came, and what it was. */
const note = async (what: Parameters<typeof nextActivity>[2]) => {
await db.update(duplicatiSources).set({ lastReportAt: nowIso, lastReport: JSON.stringify(nextActivity(previous, nowIso, what)) }).where(eq(duplicatiSources.id, source.id));
};
const report = parseReport(body);
if (!report) return { recorded: false, reason: "unreadable" };
if (!/^backup$/i.test(report.operation)) return { recorded: false, reason: "not_a_backup" };
if (!report) {
const why = explainUnreadable(body);
await note({ outcome: "unreadable", operation: null, result: null, job: null, reason: why.reason, sample: why.sample });
return { recorded: false, reason: "unreadable" };
}
if (!/^backup$/i.test(report.operation)) {
await note({ outcome: "set_aside", operation: report.operation.slice(0, 60), result: report.result.slice(0, 30), job: report.backupName, reason: null, sample: null });
return { recorded: false, reason: "not_a_backup" };
}
await note({ outcome: "recorded", operation: "Backup", result: report.result.slice(0, 30), job: report.backupName, reason: null, sample: null });
const [before] = await db.select().from(duplicatiBackups).where(and(eq(duplicatiBackups.sourceId, source.id), eq(duplicatiBackups.name, report.backupName))).limit(1);
const good = isGood(report.result);
@@ -293,6 +365,7 @@ export async function currentProblems(nowMs: number = Date.now()): Promise<Dupli
}
/** A source as lists show it: everything but the hash. */
export function publicSource(row: SourceRow) {
return { id: row.id, name: row.name, tokenPrefix: row.tokenPrefix, serverId: row.serverId, createdBy: row.createdBy, createdAt: row.createdAt, lastReportAt: row.lastReportAt };
export function publicSource(row: SourceRow, showSample = false) {
const activity = unpackActivity(row.lastReport);
return { id: row.id, name: row.name, tokenPrefix: row.tokenPrefix, serverId: row.serverId, createdBy: row.createdBy, createdAt: row.createdAt, lastReportAt: row.lastReportAt, lastReport: activity && { ...activity, sample: showSample ? activity.sample : null } };
}
+87
View File
@@ -198,6 +198,79 @@ describe("where a backup stands, and what is an alert", () => {
});
});
describe("what the last report was", () => {
test("why a body is not a report, in a sentence, with the start of it", () => {
assert.deepEqual(d.explainUnreadable(""), { reason: "the body was empty (or not text) — Duplicati should post its result as text or JSON", sample: null });
assert.equal(d.explainUnreadable(undefined).sample, null);
const cut = d.explainUnreadable('{"Data": {"MainOperation": "Backup"');
assert.match(cut.reason, /not valid JSON/);
assert.equal(cut.sample, '{"Data": {"MainOperation": "Backup"');
const noResult = d.explainUnreadable(JSON.stringify({ Data: { MainOperation: "Backup", Duration: "00:01:00" }, Extra: {} }));
assert.match(noResult.reason, /JSON without a ParsedResult field \(it has: MainOperation, Duration\)/);
assert.match(d.explainUnreadable("hello there").reason, /not JSON, and no "ParsedResult:" line/);
});
test("the sample is the start of what was sent, never more than 600 characters", () => {
assert.equal(d.explainUnreadable("x".repeat(5000)).sample?.length, 600);
});
test("the counts grow by the kind of report, and the last one is what is shown", () => {
const a = d.nextActivity(null, "t1", { outcome: "set_aside", operation: "Compact", result: "Success", job: "Docs", reason: null, sample: null });
assert.deepEqual([a.outcome, a.counts], ["set_aside", { recorded: 0, setAside: 1, unreadable: 0 }]);
const b = d.nextActivity(a, "t2", { outcome: "unreadable", operation: null, result: null, job: null, reason: "x", sample: "y" });
const c = d.nextActivity(b, "t3", { outcome: "recorded", operation: "Backup", result: "Success", job: "Docs", reason: null, sample: null });
assert.deepEqual([c.at, c.outcome, c.counts], ["t3", "recorded", { recorded: 1, setAside: 1, unreadable: 1 }]);
});
test("what is stored is read back as it was, and junk in the column is none", () => {
const a = d.nextActivity(null, "t1", { outcome: "recorded", operation: "Backup", result: "Success", job: "Docs", reason: null, sample: null });
assert.deepEqual(d.unpackActivity(JSON.stringify(a)), a);
for (const junk of [null, "", "nope", "[]", "{}", '{"outcome":"recorded"}']) assert.equal(d.unpackActivity(junk), null, String(junk));
});
describe("with reports that arrive", () => {
let token = "";
let id = 0;
before(async () => {
const made = await d.createSource({ name: "Quiet", serverId: null, createdBy: "Ada" });
token = made.token;
id = made.row.id;
});
const activity = async () => d.unpackActivity((await db.select().from(duplicatiSources).where(eq(duplicatiSources.id, id)))[0].lastReport);
test("before any report there is nothing to say", async () => {
assert.equal(await activity(), null);
});
test("a report about something other than a backup is said to be set aside, with what it was", async () => {
await d.recordReport(token, report({ MainOperation: "Compact", ParsedResult: "Success" }, "Docs"), new Date("2026-10-26T04:00:00Z"));
const a = (await activity())!;
assert.deepEqual([a.outcome, a.operation, a.result, a.job, a.counts], ["set_aside", "Compact", "Success", "Docs", { recorded: 0, setAside: 1, unreadable: 0 }]);
assert.equal((await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.sourceId, id))).length, 0, "and no job was made");
});
test("a report that cannot be read says why, and keeps the start of it", async () => {
await d.recordReport(token, "this is not what Duplicati sends", new Date("2026-10-26T05:00:00Z"));
const a = (await activity())!;
assert.deepEqual([a.outcome, a.sample, a.counts.unreadable], ["unreadable", "this is not what Duplicati sends", 1]);
assert.match(a.reason ?? "", /ParsedResult/);
});
test("a backup is said to be recorded, with the job and how it ended; the counts carry on; 'last heard' is the time of the last of any kind", async () => {
await d.recordReport(token, report({ ParsedResult: "Warning" }, "Docs"), new Date("2026-10-26T06:00:00Z"));
const a = (await activity())!;
assert.deepEqual([a.outcome, a.operation, a.result, a.job, a.counts], ["recorded", "Backup", "Warning", "Docs", { recorded: 1, setAside: 1, unreadable: 1 }]);
assert.equal((await db.select().from(duplicatiSources).where(eq(duplicatiSources.id, id)))[0].lastReportAt, "2026-10-26T06:00:00.000Z");
});
test("an address nobody knows leaves nothing behind", async () => {
const before = await activity();
assert.deepEqual(await d.recordReport("hlm_d_not-a-real-token-at-all", report()), { recorded: false, reason: "unknown_token" });
assert.deepEqual(await activity(), before);
});
});
});
describe("what the warnings said", () => {
test("a report's warnings are kept as one line each, the first twenty, each cut at 500 characters; the first is the message", () => {
const r = d.parseReport(report({ ParsedResult: "Warning", WarningsActualLength: 3, Warnings: ["Failed to process path: C:\\locked.db\n access denied", " second ", { code: "x" }] }))!;
@@ -418,6 +491,20 @@ describe("the address Duplicati posts to, the pages, the alerts and /metrics", (
who = "admin";
});
test("the page says what the last report was; only an administrator is given the start of one that could not be read", async () => {
await call("POST", `/duplicati-report/${token}`, "something that is not a report at all", { "content-type": "text/plain" });
const source = async () => ((await (await call("GET", "/duplicati")).json()) as any).sources[0];
who = "admin";
const asAdmin = await source();
assert.deepEqual([asAdmin.lastReport.outcome, asAdmin.lastReport.sample, asAdmin.lastReport.counts.unreadable >= 1], ["unreadable", "something that is not a report at all", true]);
for (const role of ["operator", "viewer"]) {
who = role;
const other = await source();
assert.deepEqual([other.lastReport.outcome, other.lastReport.sample, /ParsedResult/.test(other.lastReport.reason)], ["unreadable", null, true], role);
}
who = "admin";
});
test("a job set to alert on warnings raises one in the Backups kind, with the first message, and the page says what is wrong; switched off, it stops", async () => {
invalidateAlertsCache();
const a = (await getAlerts(true)).alerts.find((x) => /^backup:duplicati-warnings:\d+:server:/.test(x.id))!;