Read Duplicati's form-encoded reports
Duplicati posts its message as a form body by default (message=<URL-encoded text>). Unwrap it (any field name, other fields beside it), take the job and operation from the 'Duplicati Backup report for <job>' line, read JSON under that line, and read the Warnings and Errors lists of the text form. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e517de35af
commit
4d653f5a80
3 files changed
+134
-16
No files matched your search
@@ -152,6 +152,7 @@ That works on every Duplicati version, needs no login handshake and leaves nothi
|
||||
```
|
||||
|
||||
`send-http-level=All` matters: by default Duplicati only reports problems, and a backup that quietly stops reporting could not be told from one that works. The plain-text format is also read, but JSON carries the name of the job and the figures reliably.
|
||||
**How Duplicati posts it does not matter:** by default it sends a **form-encoded** body (`message=<the text, URL-encoded>`, the message beginning *"Duplicati Backup report for <job>"*), which is unwrapped — the field may have another name (`send-http-message-parameter-name`) and have others beside it; the text or JSON posted as the body as it is (`send-http-json-urls`, a message set by hand) is read too. The job's name and the operation are taken from that first line when the result does not say them. In the text form the warnings and errors are read from their `Warnings: [ … ]` and `Errors: [ … ]` lists.
|
||||
For one job only, add the same options to that job's advanced options instead.
|
||||
- **What is read** from each report: the operation (only *Backup* counts — a compact, repair or test is set aside), the result (`Success`, `Warning`, `Error`, `Fatal`), the job's name, when it began and ended, the duration, the size of what was examined, the files added, changed and deleted, the number of
|
||||
warnings and errors, the first message and **what the warnings said** (JSON format only; up to twenty, each cut at 500 characters, kept with the run). A job is learned from its first report, by name; the same name on another machine is another job.
|
||||
|
||||
@@ -96,21 +96,60 @@ export function unpackMessages(stored: string | null): string[] {
|
||||
}
|
||||
}
|
||||
|
||||
/** The first line of Duplicati's default message — "Duplicati Backup report for Documents" — which says the operation and the job when the result itself does not. */
|
||||
export interface MessageHeader {
|
||||
operation: string;
|
||||
name: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure: the message itself, whichever way Duplicati wrapped it. By default Duplicati posts a **form-encoded** body (`message=<the text, URL-encoded>`, the parameter being called `message` unless
|
||||
* `send-http-message-parameter-name` says otherwise, possibly with others beside it), and the text is "Duplicati Backup report for <job>" and the result under it; `send-http-json-urls` and
|
||||
* a message set by hand may post the text or the JSON as the body as it is. Returns the text, and the header line when there is one.
|
||||
*/
|
||||
export function unwrapMessage(body: string): { text: string; header: MessageHeader | null } {
|
||||
let text = body.trim();
|
||||
// A form body is one line of key=value pairs; a result in text form starts "Key: value" and one in JSON with a brace, so neither is mistaken for it.
|
||||
if (/^[A-Za-z0-9_.-]+=/.test(text) && !/[\r\n]/.test(text)) {
|
||||
try {
|
||||
const params = new URLSearchParams(text);
|
||||
const values = [...params.values()];
|
||||
const chosen = params.get("message") ?? values.sort((a, b) => b.length - a.length)[0];
|
||||
if (chosen && chosen.trim() !== "") text = chosen.trim();
|
||||
} catch {
|
||||
// not a form after all: it is read as it came
|
||||
}
|
||||
}
|
||||
const m = /^\s*Duplicati\s+(.+?)\s+report\s+for\s+(.+?)\s*$/im.exec(text.slice(0, 400));
|
||||
return { text, header: m ? { operation: m[1].slice(0, 60), name: m[2].slice(0, 100) } : null };
|
||||
}
|
||||
|
||||
/** Pure: the lines of a list in Duplicati's text form ("Warnings: [", one message to a line, "]"), one line each. */
|
||||
function textList(text: string, key: string): string[] {
|
||||
const m = new RegExp(`^\\s*${key}\\s*:\\s*\\[\\s*\\r?\\n([\\s\\S]*?)^\\s*\\]`, "m").exec(text);
|
||||
return m ? messagesOf(m[1].split(/\r?\n/).map((l) => l.trim().replace(/,$/, "")).filter(Boolean)) : [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure: the report Duplicati sent — JSON (`send-http-result-output-format=Json`: the results at the top, or under `Data`, with the job's name among `Extra`), or the plain text it sends
|
||||
* by default ("ParsedResult: Success" lines). Null when it is neither. Read defensively; the exact fields have not been checked against every Duplicati version.
|
||||
* by default ("ParsedResult: Success" lines); in either case possibly wrapped as a form body, or under the "Duplicati Backup report for …" line of its default message. Null when it is none of
|
||||
* these. Read defensively; the exact fields have not been checked against every Duplicati version.
|
||||
*/
|
||||
export function parseReport(body: unknown): ParsedReport | null {
|
||||
let payload: Record<string, unknown> | null = null;
|
||||
let header: MessageHeader | null = null;
|
||||
if (typeof body === "string") {
|
||||
const trimmed = body.trim();
|
||||
if (trimmed.startsWith("{")) {
|
||||
const un = unwrapMessage(body);
|
||||
header = un.header;
|
||||
const at = un.text.startsWith("{") ? 0 : un.text.search(/^\{/m);
|
||||
if (at >= 0) {
|
||||
try {
|
||||
payload = obj(JSON.parse(trimmed));
|
||||
payload = obj(JSON.parse(un.text.slice(at)));
|
||||
} catch {
|
||||
payload = null;
|
||||
if (at === 0) return null;
|
||||
return parseText(un.text, header);
|
||||
}
|
||||
} else return parseText(trimmed);
|
||||
} else return parseText(un.text, header);
|
||||
} else payload = obj(body);
|
||||
if (!payload || Object.keys(payload).length === 0) return null;
|
||||
const data = Object.keys(obj(payload.Data)).length > 0 ? obj(payload.Data) : payload;
|
||||
@@ -122,9 +161,9 @@ export function parseReport(body: unknown): ParsedReport | null {
|
||||
const errors = list(data.Errors);
|
||||
const named = [extra["backup-name"], extra.backupName, extra.BackupName, data.BackupName].find((n) => typeof n === "string" && n.trim() !== "");
|
||||
return {
|
||||
operation: String(data.MainOperation ?? extra.OperationName ?? extra.operationname ?? extra.operation ?? "Backup"),
|
||||
operation: String(data.MainOperation ?? extra.OperationName ?? extra.operationname ?? extra.operation ?? header?.operation ?? "Backup"),
|
||||
result,
|
||||
backupName: typeof named === "string" ? named.trim().slice(0, 100) : typeof extra["backup-id"] === "string" ? `Backup ${extra["backup-id"]}`.slice(0, 100) : "Unnamed backup",
|
||||
backupName: typeof named === "string" ? named.trim().slice(0, 100) : header ? header.name : typeof extra["backup-id"] === "string" ? `Backup ${extra["backup-id"]}`.slice(0, 100) : "Unnamed backup",
|
||||
begin: asIso(data.BeginTime),
|
||||
end: asIso(data.EndTime),
|
||||
durationSeconds: parseDuration(data.Duration),
|
||||
@@ -140,13 +179,15 @@ export function parseReport(body: unknown): ParsedReport | null {
|
||||
};
|
||||
}
|
||||
|
||||
function parseText(text: string): ParsedReport | null {
|
||||
function parseText(text: string, header: MessageHeader | null = null): ParsedReport | null {
|
||||
const value = (key: string) => new RegExp(`^\\s*${key}\\s*:\\s*(.+?)\\s*$`, "im").exec(text)?.[1] ?? null;
|
||||
const result = value("ParsedResult");
|
||||
if (!result) return null;
|
||||
const name = value("backup-name");
|
||||
const name = value("backup-name") ?? header?.name ?? null;
|
||||
const warningLines = textList(text, "Warnings");
|
||||
const errorLines = textList(text, "Errors");
|
||||
return {
|
||||
operation: value("MainOperation") ?? value("OperationName") ?? "Backup",
|
||||
operation: value("MainOperation") ?? value("OperationName") ?? header?.operation ?? "Backup",
|
||||
result,
|
||||
backupName: name ? name.slice(0, 100) : "Unnamed backup",
|
||||
begin: null,
|
||||
@@ -157,10 +198,10 @@ function parseText(text: string): ParsedReport | null {
|
||||
addedFiles: asInt(value("AddedFiles")?.match(/^[\d,]+/)?.[0]),
|
||||
modifiedFiles: asInt(value("ModifiedFiles")?.match(/^[\d,]+/)?.[0]),
|
||||
deletedFiles: asInt(value("DeletedFiles")?.match(/^[\d,]+/)?.[0]),
|
||||
warnings: asInt(value("WarningsActualLength")?.match(/^[\d,]+/)?.[0]) ?? 0,
|
||||
warnings: asInt(value("WarningsActualLength")?.match(/^[\d,]+/)?.[0]) ?? warningLines.length,
|
||||
errors: asInt(value("ErrorsActualLength")?.match(/^[\d,]+/)?.[0]) ?? 0,
|
||||
message: null,
|
||||
warningMessages: [],
|
||||
message: lineOf(errorLines[0]) ?? lineOf(warningLines[0]),
|
||||
warningMessages: warningLines,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -204,9 +245,10 @@ export const MAX_SAMPLE = 600;
|
||||
|
||||
/** Pure: why a body is not a report this app can use, in a sentence, and the start of it. Read for the same two forms parseReport reads. */
|
||||
export function explainUnreadable(body: unknown): { reason: string; sample: string | null } {
|
||||
const text = typeof body === "string" ? body.trim() : "";
|
||||
const raw = typeof body === "string" ? body.trim() : "";
|
||||
const text = raw === "" ? "" : unwrapMessage(raw).text;
|
||||
if (text === "") return { reason: "the body was empty (or not text) — Duplicati should post its result as text or JSON", sample: null };
|
||||
const sample = text.slice(0, MAX_SAMPLE);
|
||||
const sample = raw.slice(0, MAX_SAMPLE);
|
||||
if (text.startsWith("{")) {
|
||||
let payload: Record<string, unknown> | null = null;
|
||||
try {
|
||||
|
||||
@@ -198,6 +198,73 @@ describe("where a backup stands, and what is an alert", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("the ways Duplicati wraps its message", () => {
|
||||
const TEXT = ["DeletedFiles: 0", "ModifiedFiles: 3", "ExaminedFiles: 1,520", "SizeOfExaminedFiles: 3221225472 (3.00 GB)", "AddedFiles: 2", "MainOperation: Backup", "ParsedResult: Success", "Duration: 00:05:12.7", "WarningsActualLength: 0", "ErrorsActualLength: 0"].join("\n");
|
||||
const message = (job = "Documents", result = TEXT, operation = "Backup") => `Duplicati ${operation} report for ${job}\n\n${result}`;
|
||||
/** As Duplicati posts it by default: one form field, the text URL-encoded. */
|
||||
const form = (text: string, field = "message") => `${field}=${encodeURIComponent(text)}`;
|
||||
|
||||
test("the default: a form body whose message is Duplicati's text, with the job named on the first line", () => {
|
||||
const r = d.parseReport(form(message()))!;
|
||||
assert.deepEqual([r.operation, r.result, r.backupName, r.durationSeconds, r.examinedFiles, r.sizeBytes, r.addedFiles, r.modifiedFiles, r.warnings], ["Backup", "Success", "Documents", 312, 1520, 3221225472, 2, 3, 0]);
|
||||
});
|
||||
|
||||
test("a form body as a browser writes it, with + for a space, and with other fields beside the message", () => {
|
||||
const plus = "message=Duplicati+Backup+report+for+My+Photos%0A%0AMainOperation%3A+Backup%0AParsedResult%3A+Warning%0AWarningsActualLength%3A+2&source=laptop&level=all";
|
||||
const r = d.parseReport(plus)!;
|
||||
assert.deepEqual([r.operation, r.result, r.backupName, r.warnings], ["Backup", "Warning", "My Photos", 2]);
|
||||
});
|
||||
|
||||
test("a message parameter with another name is found too: the longest value is the message", () => {
|
||||
const r = d.parseReport(`${form(message("Docs"), "msg")}&x=1`)!;
|
||||
assert.deepEqual([r.result, r.backupName], ["Success", "Docs"]);
|
||||
});
|
||||
|
||||
test("the result in JSON, under the header line, inside a form body or posted as it is", () => {
|
||||
const json = report({}, "", {});
|
||||
const wrapped = d.parseReport(form(message("Photos", JSON.stringify(JSON.parse(json)))))!;
|
||||
assert.deepEqual([wrapped.result, wrapped.backupName, wrapped.sizeBytes], ["Success", "Photos", 3221225472]);
|
||||
const direct = d.parseReport(message("Photos", json))!;
|
||||
assert.deepEqual([direct.result, direct.backupName], ["Success", "Photos"]);
|
||||
assert.equal(d.parseReport(form(json))?.backupName, "Backup DB-1", "JSON alone in a form: the job is known by its id");
|
||||
});
|
||||
|
||||
test("the operation can come from the header line when the result does not say it", () => {
|
||||
const r = d.parseReport(form(message("Docs", "ParsedResult: Success\nDuration: 00:00:05", "Compact")))!;
|
||||
assert.deepEqual([r.operation, r.backupName], ["Compact", "Docs"]);
|
||||
});
|
||||
|
||||
test("a name in the result itself wins over the header's", () => {
|
||||
const r = d.parseReport(form(message("From header", `${TEXT}\nbackup-name: From result`)))!;
|
||||
assert.equal(r.backupName, "From result");
|
||||
});
|
||||
|
||||
test("the warnings and errors of the text form are read as lines", () => {
|
||||
const text = [TEXT.replace("WarningsActualLength: 0", "WarningsActualLength: 2").replace("ParsedResult: Success", "ParsedResult: Warning"), "Warnings: [", " Failed to process path: /a/b.lock,", " Could not read /c/d", "]", "Errors: []"].join("\n");
|
||||
const r = d.parseReport(form(message("Docs", text)))!;
|
||||
assert.deepEqual([r.result, r.warnings, r.warningMessages, r.message], ["Warning", 2, ["Failed to process path: /a/b.lock", "Could not read /c/d"], "Failed to process path: /a/b.lock"]);
|
||||
const failed = d.parseReport(form(message("Docs", ["ParsedResult: Error", "ErrorsActualLength: 1", "Errors: [", " Unable to connect to the backend", "]"].join("\n"))))!;
|
||||
assert.deepEqual([failed.errors, failed.message], [1, "Unable to connect to the backend"]);
|
||||
});
|
||||
|
||||
test("what is a form but has no report in it is not read, and says what is missing", () => {
|
||||
for (const body of ["message=hello", "a=1&b=2", form("Duplicati Backup report for Docs\n\nnothing useful")]) assert.equal(d.parseReport(body), null, body);
|
||||
assert.match(d.explainUnreadable(form("Duplicati Backup report for Docs\n\nnothing useful")).reason, /no "ParsedResult:" line/);
|
||||
assert.equal(d.explainUnreadable("message=hello").sample, "message=hello", "the sample is what was sent, not what was made of it");
|
||||
});
|
||||
|
||||
test("a text result or JSON that is not a form is never taken for one", () => {
|
||||
assert.equal(d.parseReport(TEXT)?.result, "Success");
|
||||
assert.equal(d.parseReport(report())?.result, "Success");
|
||||
assert.deepEqual(d.unwrapMessage("DeletedFiles: 0\nParsedResult: Success"), { text: "DeletedFiles: 0\nParsedResult: Success", header: null });
|
||||
});
|
||||
|
||||
test("the header line is read for the operation and the job, however it is wrapped", () => {
|
||||
assert.deepEqual(d.unwrapMessage(form(message("Jackie's Docs", "x", "Repair"))).header, { operation: "Repair", name: "Jackie's Docs" });
|
||||
assert.equal(d.unwrapMessage("no header here").header, null);
|
||||
});
|
||||
});
|
||||
|
||||
describe("what the last report was", () => {
|
||||
test("why a body is not a report, in a sentence, with the start of it", () => {
|
||||
assert.deepEqual(d.explainUnreadable(""), { reason: "the body was empty (or not text) — Duplicati should post its result as text or JSON", sample: null });
|
||||
@@ -473,6 +540,14 @@ describe("the address Duplicati posts to, the pages, the alerts and /metrics", (
|
||||
who = "admin";
|
||||
});
|
||||
|
||||
test("Duplicati's own way of posting — a form body — is read, and makes the job", async () => {
|
||||
const text = `Duplicati Backup report for Documents\n\nMainOperation: Backup\nParsedResult: Success\nExaminedFiles: 1520\nSizeOfExaminedFiles: 3221225472\nDuration: 00:05:12.7`;
|
||||
const res = await call("POST", `/duplicati-report/${token}`, `message=${encodeURIComponent(text)}`, { "content-type": "application/x-www-form-urlencoded" });
|
||||
assert.deepEqual([res.status, await res.text()], [200, "OK"]);
|
||||
const documents = (await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.name, "Documents")))[0];
|
||||
assert.deepEqual([documents.lastResult, documents.sizeBytes], ["Success", 3221225472]);
|
||||
});
|
||||
|
||||
test("warnings: the page can read what they said, per job and per run; an operator can switch the alert on, and a viewer cannot", async () => {
|
||||
const docs = (await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.name, "Documents")))[0];
|
||||
await d.recordReport(token, report({ ParsedResult: "Warning", WarningsActualLength: 2, Warnings: ["file one locked", "file two locked"], EndTime: new Date().toISOString() }, "Documents"));
|
||||
|
||||
Reference in new issue
Block a user