Read Duplicati's form-encoded reports
Duplicati posts its message as a form body by default (message=<URL-encoded text>). Unwrap it (any field name, other fields beside it), take the job and operation from the 'Duplicati Backup report for <job>' line, read JSON under that line, and read the Warnings and Errors lists of the text form. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e517de35af
commit
4d653f5a80
3 files changed
+134
-16
No files matched your search
@@ -96,21 +96,60 @@ export function unpackMessages(stored: string | null): string[] {
|
||||
}
|
||||
}
|
||||
|
||||
/** The first line of Duplicati's default message — "Duplicati Backup report for Documents" — which says the operation and the job when the result itself does not. */
|
||||
export interface MessageHeader {
|
||||
operation: string;
|
||||
name: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure: the message itself, whichever way Duplicati wrapped it. By default Duplicati posts a **form-encoded** body (`message=<the text, URL-encoded>`, the parameter being called `message` unless
|
||||
* `send-http-message-parameter-name` says otherwise, possibly with others beside it), and the text is "Duplicati Backup report for <job>" and the result under it; `send-http-json-urls` and
|
||||
* a message set by hand may post the text or the JSON as the body as it is. Returns the text, and the header line when there is one.
|
||||
*/
|
||||
export function unwrapMessage(body: string): { text: string; header: MessageHeader | null } {
|
||||
let text = body.trim();
|
||||
// A form body is one line of key=value pairs; a result in text form starts "Key: value" and one in JSON with a brace, so neither is mistaken for it.
|
||||
if (/^[A-Za-z0-9_.-]+=/.test(text) && !/[\r\n]/.test(text)) {
|
||||
try {
|
||||
const params = new URLSearchParams(text);
|
||||
const values = [...params.values()];
|
||||
const chosen = params.get("message") ?? values.sort((a, b) => b.length - a.length)[0];
|
||||
if (chosen && chosen.trim() !== "") text = chosen.trim();
|
||||
} catch {
|
||||
// not a form after all: it is read as it came
|
||||
}
|
||||
}
|
||||
const m = /^\s*Duplicati\s+(.+?)\s+report\s+for\s+(.+?)\s*$/im.exec(text.slice(0, 400));
|
||||
return { text, header: m ? { operation: m[1].slice(0, 60), name: m[2].slice(0, 100) } : null };
|
||||
}
|
||||
|
||||
/** Pure: the lines of a list in Duplicati's text form ("Warnings: [", one message to a line, "]"), one line each. */
|
||||
function textList(text: string, key: string): string[] {
|
||||
const m = new RegExp(`^\\s*${key}\\s*:\\s*\\[\\s*\\r?\\n([\\s\\S]*?)^\\s*\\]`, "m").exec(text);
|
||||
return m ? messagesOf(m[1].split(/\r?\n/).map((l) => l.trim().replace(/,$/, "")).filter(Boolean)) : [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure: the report Duplicati sent — JSON (`send-http-result-output-format=Json`: the results at the top, or under `Data`, with the job's name among `Extra`), or the plain text it sends
|
||||
* by default ("ParsedResult: Success" lines). Null when it is neither. Read defensively; the exact fields have not been checked against every Duplicati version.
|
||||
* by default ("ParsedResult: Success" lines); in either case possibly wrapped as a form body, or under the "Duplicati Backup report for …" line of its default message. Null when it is none of
|
||||
* these. Read defensively; the exact fields have not been checked against every Duplicati version.
|
||||
*/
|
||||
export function parseReport(body: unknown): ParsedReport | null {
|
||||
let payload: Record<string, unknown> | null = null;
|
||||
let header: MessageHeader | null = null;
|
||||
if (typeof body === "string") {
|
||||
const trimmed = body.trim();
|
||||
if (trimmed.startsWith("{")) {
|
||||
const un = unwrapMessage(body);
|
||||
header = un.header;
|
||||
const at = un.text.startsWith("{") ? 0 : un.text.search(/^\{/m);
|
||||
if (at >= 0) {
|
||||
try {
|
||||
payload = obj(JSON.parse(trimmed));
|
||||
payload = obj(JSON.parse(un.text.slice(at)));
|
||||
} catch {
|
||||
payload = null;
|
||||
if (at === 0) return null;
|
||||
return parseText(un.text, header);
|
||||
}
|
||||
} else return parseText(trimmed);
|
||||
} else return parseText(un.text, header);
|
||||
} else payload = obj(body);
|
||||
if (!payload || Object.keys(payload).length === 0) return null;
|
||||
const data = Object.keys(obj(payload.Data)).length > 0 ? obj(payload.Data) : payload;
|
||||
@@ -122,9 +161,9 @@ export function parseReport(body: unknown): ParsedReport | null {
|
||||
const errors = list(data.Errors);
|
||||
const named = [extra["backup-name"], extra.backupName, extra.BackupName, data.BackupName].find((n) => typeof n === "string" && n.trim() !== "");
|
||||
return {
|
||||
operation: String(data.MainOperation ?? extra.OperationName ?? extra.operationname ?? extra.operation ?? "Backup"),
|
||||
operation: String(data.MainOperation ?? extra.OperationName ?? extra.operationname ?? extra.operation ?? header?.operation ?? "Backup"),
|
||||
result,
|
||||
backupName: typeof named === "string" ? named.trim().slice(0, 100) : typeof extra["backup-id"] === "string" ? `Backup ${extra["backup-id"]}`.slice(0, 100) : "Unnamed backup",
|
||||
backupName: typeof named === "string" ? named.trim().slice(0, 100) : header ? header.name : typeof extra["backup-id"] === "string" ? `Backup ${extra["backup-id"]}`.slice(0, 100) : "Unnamed backup",
|
||||
begin: asIso(data.BeginTime),
|
||||
end: asIso(data.EndTime),
|
||||
durationSeconds: parseDuration(data.Duration),
|
||||
@@ -140,13 +179,15 @@ export function parseReport(body: unknown): ParsedReport | null {
|
||||
};
|
||||
}
|
||||
|
||||
function parseText(text: string): ParsedReport | null {
|
||||
function parseText(text: string, header: MessageHeader | null = null): ParsedReport | null {
|
||||
const value = (key: string) => new RegExp(`^\\s*${key}\\s*:\\s*(.+?)\\s*$`, "im").exec(text)?.[1] ?? null;
|
||||
const result = value("ParsedResult");
|
||||
if (!result) return null;
|
||||
const name = value("backup-name");
|
||||
const name = value("backup-name") ?? header?.name ?? null;
|
||||
const warningLines = textList(text, "Warnings");
|
||||
const errorLines = textList(text, "Errors");
|
||||
return {
|
||||
operation: value("MainOperation") ?? value("OperationName") ?? "Backup",
|
||||
operation: value("MainOperation") ?? value("OperationName") ?? header?.operation ?? "Backup",
|
||||
result,
|
||||
backupName: name ? name.slice(0, 100) : "Unnamed backup",
|
||||
begin: null,
|
||||
@@ -157,10 +198,10 @@ function parseText(text: string): ParsedReport | null {
|
||||
addedFiles: asInt(value("AddedFiles")?.match(/^[\d,]+/)?.[0]),
|
||||
modifiedFiles: asInt(value("ModifiedFiles")?.match(/^[\d,]+/)?.[0]),
|
||||
deletedFiles: asInt(value("DeletedFiles")?.match(/^[\d,]+/)?.[0]),
|
||||
warnings: asInt(value("WarningsActualLength")?.match(/^[\d,]+/)?.[0]) ?? 0,
|
||||
warnings: asInt(value("WarningsActualLength")?.match(/^[\d,]+/)?.[0]) ?? warningLines.length,
|
||||
errors: asInt(value("ErrorsActualLength")?.match(/^[\d,]+/)?.[0]) ?? 0,
|
||||
message: null,
|
||||
warningMessages: [],
|
||||
message: lineOf(errorLines[0]) ?? lineOf(warningLines[0]),
|
||||
warningMessages: warningLines,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -204,9 +245,10 @@ export const MAX_SAMPLE = 600;
|
||||
|
||||
/** Pure: why a body is not a report this app can use, in a sentence, and the start of it. Read for the same two forms parseReport reads. */
|
||||
export function explainUnreadable(body: unknown): { reason: string; sample: string | null } {
|
||||
const text = typeof body === "string" ? body.trim() : "";
|
||||
const raw = typeof body === "string" ? body.trim() : "";
|
||||
const text = raw === "" ? "" : unwrapMessage(raw).text;
|
||||
if (text === "") return { reason: "the body was empty (or not text) — Duplicati should post its result as text or JSON", sample: null };
|
||||
const sample = text.slice(0, MAX_SAMPLE);
|
||||
const sample = raw.slice(0, MAX_SAMPLE);
|
||||
if (text.startsWith("{")) {
|
||||
let payload: Record<string, unknown> | null = null;
|
||||
try {
|
||||
|
||||
@@ -198,6 +198,73 @@ describe("where a backup stands, and what is an alert", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("the ways Duplicati wraps its message", () => {
|
||||
const TEXT = ["DeletedFiles: 0", "ModifiedFiles: 3", "ExaminedFiles: 1,520", "SizeOfExaminedFiles: 3221225472 (3.00 GB)", "AddedFiles: 2", "MainOperation: Backup", "ParsedResult: Success", "Duration: 00:05:12.7", "WarningsActualLength: 0", "ErrorsActualLength: 0"].join("\n");
|
||||
const message = (job = "Documents", result = TEXT, operation = "Backup") => `Duplicati ${operation} report for ${job}\n\n${result}`;
|
||||
/** As Duplicati posts it by default: one form field, the text URL-encoded. */
|
||||
const form = (text: string, field = "message") => `${field}=${encodeURIComponent(text)}`;
|
||||
|
||||
test("the default: a form body whose message is Duplicati's text, with the job named on the first line", () => {
|
||||
const r = d.parseReport(form(message()))!;
|
||||
assert.deepEqual([r.operation, r.result, r.backupName, r.durationSeconds, r.examinedFiles, r.sizeBytes, r.addedFiles, r.modifiedFiles, r.warnings], ["Backup", "Success", "Documents", 312, 1520, 3221225472, 2, 3, 0]);
|
||||
});
|
||||
|
||||
test("a form body as a browser writes it, with + for a space, and with other fields beside the message", () => {
|
||||
const plus = "message=Duplicati+Backup+report+for+My+Photos%0A%0AMainOperation%3A+Backup%0AParsedResult%3A+Warning%0AWarningsActualLength%3A+2&source=laptop&level=all";
|
||||
const r = d.parseReport(plus)!;
|
||||
assert.deepEqual([r.operation, r.result, r.backupName, r.warnings], ["Backup", "Warning", "My Photos", 2]);
|
||||
});
|
||||
|
||||
test("a message parameter with another name is found too: the longest value is the message", () => {
|
||||
const r = d.parseReport(`${form(message("Docs"), "msg")}&x=1`)!;
|
||||
assert.deepEqual([r.result, r.backupName], ["Success", "Docs"]);
|
||||
});
|
||||
|
||||
test("the result in JSON, under the header line, inside a form body or posted as it is", () => {
|
||||
const json = report({}, "", {});
|
||||
const wrapped = d.parseReport(form(message("Photos", JSON.stringify(JSON.parse(json)))))!;
|
||||
assert.deepEqual([wrapped.result, wrapped.backupName, wrapped.sizeBytes], ["Success", "Photos", 3221225472]);
|
||||
const direct = d.parseReport(message("Photos", json))!;
|
||||
assert.deepEqual([direct.result, direct.backupName], ["Success", "Photos"]);
|
||||
assert.equal(d.parseReport(form(json))?.backupName, "Backup DB-1", "JSON alone in a form: the job is known by its id");
|
||||
});
|
||||
|
||||
test("the operation can come from the header line when the result does not say it", () => {
|
||||
const r = d.parseReport(form(message("Docs", "ParsedResult: Success\nDuration: 00:00:05", "Compact")))!;
|
||||
assert.deepEqual([r.operation, r.backupName], ["Compact", "Docs"]);
|
||||
});
|
||||
|
||||
test("a name in the result itself wins over the header's", () => {
|
||||
const r = d.parseReport(form(message("From header", `${TEXT}\nbackup-name: From result`)))!;
|
||||
assert.equal(r.backupName, "From result");
|
||||
});
|
||||
|
||||
test("the warnings and errors of the text form are read as lines", () => {
|
||||
const text = [TEXT.replace("WarningsActualLength: 0", "WarningsActualLength: 2").replace("ParsedResult: Success", "ParsedResult: Warning"), "Warnings: [", " Failed to process path: /a/b.lock,", " Could not read /c/d", "]", "Errors: []"].join("\n");
|
||||
const r = d.parseReport(form(message("Docs", text)))!;
|
||||
assert.deepEqual([r.result, r.warnings, r.warningMessages, r.message], ["Warning", 2, ["Failed to process path: /a/b.lock", "Could not read /c/d"], "Failed to process path: /a/b.lock"]);
|
||||
const failed = d.parseReport(form(message("Docs", ["ParsedResult: Error", "ErrorsActualLength: 1", "Errors: [", " Unable to connect to the backend", "]"].join("\n"))))!;
|
||||
assert.deepEqual([failed.errors, failed.message], [1, "Unable to connect to the backend"]);
|
||||
});
|
||||
|
||||
test("what is a form but has no report in it is not read, and says what is missing", () => {
|
||||
for (const body of ["message=hello", "a=1&b=2", form("Duplicati Backup report for Docs\n\nnothing useful")]) assert.equal(d.parseReport(body), null, body);
|
||||
assert.match(d.explainUnreadable(form("Duplicati Backup report for Docs\n\nnothing useful")).reason, /no "ParsedResult:" line/);
|
||||
assert.equal(d.explainUnreadable("message=hello").sample, "message=hello", "the sample is what was sent, not what was made of it");
|
||||
});
|
||||
|
||||
test("a text result or JSON that is not a form is never taken for one", () => {
|
||||
assert.equal(d.parseReport(TEXT)?.result, "Success");
|
||||
assert.equal(d.parseReport(report())?.result, "Success");
|
||||
assert.deepEqual(d.unwrapMessage("DeletedFiles: 0\nParsedResult: Success"), { text: "DeletedFiles: 0\nParsedResult: Success", header: null });
|
||||
});
|
||||
|
||||
test("the header line is read for the operation and the job, however it is wrapped", () => {
|
||||
assert.deepEqual(d.unwrapMessage(form(message("Jackie's Docs", "x", "Repair"))).header, { operation: "Repair", name: "Jackie's Docs" });
|
||||
assert.equal(d.unwrapMessage("no header here").header, null);
|
||||
});
|
||||
});
|
||||
|
||||
describe("what the last report was", () => {
|
||||
test("why a body is not a report, in a sentence, with the start of it", () => {
|
||||
assert.deepEqual(d.explainUnreadable(""), { reason: "the body was empty (or not text) — Duplicati should post its result as text or JSON", sample: null });
|
||||
@@ -473,6 +540,14 @@ describe("the address Duplicati posts to, the pages, the alerts and /metrics", (
|
||||
who = "admin";
|
||||
});
|
||||
|
||||
test("Duplicati's own way of posting — a form body — is read, and makes the job", async () => {
|
||||
const text = `Duplicati Backup report for Documents\n\nMainOperation: Backup\nParsedResult: Success\nExaminedFiles: 1520\nSizeOfExaminedFiles: 3221225472\nDuration: 00:05:12.7`;
|
||||
const res = await call("POST", `/duplicati-report/${token}`, `message=${encodeURIComponent(text)}`, { "content-type": "application/x-www-form-urlencoded" });
|
||||
assert.deepEqual([res.status, await res.text()], [200, "OK"]);
|
||||
const documents = (await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.name, "Documents")))[0];
|
||||
assert.deepEqual([documents.lastResult, documents.sizeBytes], ["Success", 3221225472]);
|
||||
});
|
||||
|
||||
test("warnings: the page can read what they said, per job and per run; an operator can switch the alert on, and a viewer cannot", async () => {
|
||||
const docs = (await db.select().from(duplicatiBackups).where(eq(duplicatiBackups.name, "Documents")))[0];
|
||||
await d.recordReport(token, report({ ParsedResult: "Warning", WarningsActualLength: 2, Warnings: ["file one locked", "file two locked"], EndTime: new Date().toISOString() }, "Documents"));
|
||||
|
||||
Reference in new issue
Block a user