Nextcloud cron and Nextcloud's own checks; bulk close and open again

Nextcloud: its own "Cron last run" setup check (GET /settings/setupcheck,
Nextcloud 28+, found by class name) raises an alert when background jobs have
stopped; other checks that report an error are one alert, warnings are shown
on the page. The core version Nextcloud says is available is shown too. Before
28, cron and the checks are unknown, never "fine".

Alerts page: bulk Close... (one reason, written on each incident, undoable) and
Open again. Bulk Wake now leaves closed alerts alone.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-25 22:12:10 +01:00
1 parent 5ae61a1ece
commit 65214f6f0f
25 files changed
+8108 -35

No files matched your search

+96
View File
@@ -204,3 +204,99 @@ describe("until it gets worse", () => {
assert.deepEqual(reminders.map((r) => [r.key, r.reason]), [["k:x5", "ended"]]);
});
});
describe("closing several at once", () => {
const notesOf = async (alertId: string) => {
const incidentId = (await openIncidentIds([alertId])).get(alertId)!;
return (await getIncident(incidentId))!.notes.map((n) => n.body);
};
const clear = async () => {
await db.delete(alertSnoozes);
};
test("several are closed with one reason; each incident says so; the one that has gone is skipped; they are held back", async () => {
await clear();
const a = await idOfHeartbeat("jobA");
const b = await idOfHeartbeat("jobB");
const r = await call("POST", "/bulk", { action: "close", alertIds: [a.id, b.id, "monitoring:nothing:1"], note: " Both retired with the old host. " });
assert.equal(r.status, 200);
assert.equal(r.body.changed, 2);
assert.deepEqual(r.body.skipped.map((x: { reason: string }) => x.reason), ["it isn't there any more"]);
const rows = await db.select().from(alertSnoozes);
assert.deepEqual(rows.map((x) => [x.kind, x.note, x.snoozedBy]), [["closed", "Both retired with the old host.", "Olga"], ["closed", "Both retired with the old host.", "Olga"]]);
assert.deepEqual(rows.map((x) => x.severity), [a.severity, b.severity]);
assert.ok((await notesOf(a.id)).includes("Closed: Both retired with the old host. (closed together with 1 other)"));
invalidateAlertsCache();
const report = await getAlerts(true);
assert.ok([a.id, b.id].every((id) => report.alerts.find((x) => x.id === id)?.closed?.by === "Olga"));
await call("POST", "/bulk", r.body.undo);
});
test("undo opens them again and says so on each incident; an alert that was closed before the bulk keeps its own closing and note", async () => {
await clear();
const a = await idOfHeartbeat("jobA");
const b = await idOfHeartbeat("jobB");
await call("PUT", "/close", { alertId: a.id, note: "mine from before" });
const r = await call("POST", "/bulk", { action: "close", alertIds: [a.id, b.id], note: "bulk" });
assert.equal(r.body.changed, 2);
assert.equal(r.body.undo.closed, true);
assert.equal((await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, a.id)))[0].note, "bulk", "replaced");
const undone = (id: string) => notesOf(id).then((n) => n.filter((x) => x === "Opened again (closing it was undone).").length);
const [beforeA, beforeB] = [await undone(a.id), await undone(b.id)];
await call("POST", "/bulk", r.body.undo);
const rows = await db.select().from(alertSnoozes);
assert.deepEqual(rows.map((x) => [x.alertId, x.note]), [[a.id, "mine from before"]], "a is closed as it was; b is open again");
assert.equal(await undone(b.id), beforeB + 1, "b is open again and says so");
assert.equal(await undone(a.id), beforeA, "a was not opened by the undo, so says nothing");
await clear();
});
test("open again: only closed alerts are opened, one that is merely snoozed is left as it is, and the incident says so; undo closes them again", async () => {
await clear();
const a = await idOfHeartbeat("jobA");
const b = await idOfHeartbeat("jobB");
await call("PUT", "/close", { alertId: a.id, note: "accepted" });
await call("PUT", "/snooze", { alertId: b.id, until: new Date(Date.now() + 3_600_000).toISOString() });
const r = await call("POST", "/bulk", { action: "reopen", alertIds: [a.id, b.id] });
assert.equal(r.body.changed, 1);
assert.deepEqual(r.body.skipped.map((x: { reason: string }) => x.reason), ["it wasn't closed"]);
assert.deepEqual((await db.select().from(alertSnoozes)).map((x) => [x.alertId, x.kind]), [[b.id, "snooze"]]);
assert.equal((await notesOf(a.id)).at(-1), "Opened again.");
await call("POST", "/bulk", r.body.undo);
const back = (await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, a.id)))[0];
assert.deepEqual([back.kind, back.note], ["closed", "accepted"]);
await clear();
});
test("waking snoozed alerts leaves a closed one alone and says to use Open again", async () => {
await clear();
const a = await idOfHeartbeat("jobA");
await call("PUT", "/close", { alertId: a.id });
const r = await call("POST", "/bulk", { action: "unsnooze", alertIds: [a.id] });
assert.equal(r.body.changed, 0);
assert.match(r.body.skipped[0].reason, /closed, not snoozed — use Open again/);
assert.equal((await db.select().from(alertSnoozes)).length, 1, "still closed");
await clear();
});
test("a note that is too long is refused and nothing is closed; a viewer cannot; both are in the audit log and the change log", async () => {
await clear();
const a = await idOfHeartbeat("jobA");
const long = await call("POST", "/bulk", { action: "close", alertIds: [a.id], note: "x".repeat(501) });
assert.equal(long.status, 400);
assert.equal((await db.select().from(alertSnoozes)).length, 0);
who = "viewer";
assert.equal((await call("POST", "/bulk", { action: "close", alertIds: [a.id] })).status, 403);
assert.equal((await call("POST", "/bulk", { action: "reopen", alertIds: [a.id] })).status, 403);
who = "operator";
await call("POST", "/bulk", { action: "close", alertIds: [a.id], note: "n" });
await call("POST", "/bulk", { action: "reopen", alertIds: [a.id] });
const actions = (await db.select().from(auditLog)).map((e) => `${e.category}/${e.action}`);
assert.ok(actions.includes("alert/bulk_close") && actions.includes("alert/bulk_reopen"));
const titles = (await getChangeLog({ days: 1, limit: 400 })).entries.map((e) => e.title);
assert.ok(titles.some((t) => /alerts? closed at once$/.test(t)), "close");
assert.ok(titles.some((t) => /closed alerts? opened again at once$/.test(t)), "reopen");
assert.ok(titles.some((t) => /^Closing several alerts at once was undone$/.test(t)), "undo");
await clear();
});
});
+99 -6
View File
@@ -36,9 +36,9 @@ await runMigrations();
const MIN = 60_000;
type Read = import("../src/integrations/nextcloud/adapter.js").NextcloudRead;
const read = (extra: Partial<Read> = {}): Read => ({ version: "31.0.2", maintenance: false, needsUpgrade: false, detail: true, freeBytes: 500e9, users: 4, files: 120000, active24h: 2, appUpdates: 1, ...extra });
const read = (extra: Partial<Read> = {}): Read => ({ version: "31.0.2", maintenance: false, needsUpgrade: false, detail: true, freeBytes: 500e9, users: 4, files: 120000, active24h: 2, appUpdates: 1, coreUpdate: null, cron: { state: "success", text: "Last background job execution ran 4 minutes ago." }, setupIssues: [], ...extra });
const INFO = { ocs: { meta: { status: "ok" }, data: { nextcloud: { system: { version: "31.0.2.1", freespace: 412e9, apps: { num_installed: 40, num_updates_available: 3, app_updates: { calendar: "5.1.0" } } }, storage: { num_users: 4, num_files: 120000 } }, activeUsers: { last5minutes: 1, last24hours: 2 } } } };
const INFO = { ocs: { meta: { status: "ok" }, data: { nextcloud: { system: { version: "31.0.2.1", freespace: 412e9, apps: { num_installed: 40, num_updates_available: 3, app_updates: { calendar: "5.1.0" } }, update: { lastupdatedat: 1, available: true, available_version: "31.0.5" } }, storage: { num_users: 4, num_files: 120000 } }, activeUsers: { last5minutes: 1, last24hours: 2 } } } };
describe("what Nextcloud's API says", () => {
test("status.php: the version, maintenance mode and a database that wants upgrading", () => {
@@ -50,16 +50,61 @@ describe("what Nextcloud's API says", () => {
});
test("the monitoring app: room left, users, files, who was active and the app updates", () => {
assert.deepEqual(adapter.parseServerInfo(INFO), { freeBytes: 412e9, users: 4, files: 120000, active24h: 2, appUpdates: 3 });
assert.deepEqual(adapter.parseServerInfo(INFO), { freeBytes: 412e9, users: 4, files: 120000, active24h: 2, appUpdates: 3, coreUpdate: "31.0.5" });
assert.equal(adapter.parseServerInfo({ ocs: { meta: {} } }), null, "no data is not an empty answer");
assert.equal(adapter.parseServerInfo({ ocs: { data: { nextcloud: { system: { apps: { app_updates: { a: "1", b: "2" } } } } } } })?.appUpdates, 2, "counted from the list when no count is given");
assert.equal(adapter.parseServerInfo({ ocs: { data: { nextcloud: { system: {} } } } })?.appUpdates, null);
});
test("a core update: its version when Nextcloud says it, 'newer' when it says only that there is one, nothing when there is none", () => {
const withUpdate = (update: unknown) => adapter.parseServerInfo({ ocs: { data: { nextcloud: { system: { update } } } } })?.coreUpdate;
assert.equal(withUpdate({ available: true, available_version: "31.0.5" }), "31.0.5");
assert.equal(withUpdate({ available: true }), "newer");
assert.equal(withUpdate({ available: false, available_version: "31.0.5" }), null);
assert.equal(withUpdate(undefined), null);
});
describe("Nextcloud's own setup checks", () => {
const CRON = "OCA\\Settings\\SetupChecks\\CronInfo";
const checks = (cron: { severity: string; description: string }) => ({
system: {
[CRON]: { name: "Cron last run", ...cron, descriptionParameters: { link: {} }, linkToDoc: null },
"OCA\\Settings\\SetupChecks\\PhpMemoryLimit": { name: "PHP memory limit", severity: "success", description: "ok" },
"OCA\\Settings\\SetupChecks\\DatabaseHasMissingIndices": { name: "Database missing indices", severity: "error", description: "Missing optional index \"x\" in table \"y\". {link}" },
},
security: { "OCA\\Settings\\SetupChecks\\SecurityHeaders": { name: "HTTP headers", severity: "warning", description: "Some headers are not set." } },
});
test("the Cron check is found by its class, not by its name (which is in the language Nextcloud is set to); the others that say error or warning are kept; successes are not", () => {
const r = adapter.parseSetupChecks(checks({ severity: "error", description: "Last background job execution ran 3 hours ago. Something seems wrong. {link}." }))!;
assert.deepEqual(r.cron, { state: "error", text: "Last background job execution ran 3 hours ago. Something seems wrong." });
assert.deepEqual(r.issues.map((i) => [i.name, i.severity, i.text]), [["Database missing indices", "error", 'Missing optional index "x" in table "y".'], ["HTTP headers", "warning", "Some headers are not set."]]);
});
test("a Cron check that is fine is fine", () => {
const r = adapter.parseSetupChecks(checks({ severity: "success", description: "Last background job execution ran 4 minutes ago." }))!;
assert.deepEqual(r.cron, { state: "success", text: "Last background job execution ran 4 minutes ago." });
});
test("inside an OCS envelope it is the same; no checks at all is no problem; something that is not checks is not read", () => {
assert.equal(adapter.parseSetupChecks({ ocs: { data: checks({ severity: "success", description: "ok" }) } })?.cron?.state, "success");
assert.deepEqual(adapter.parseSetupChecks({}), { cron: null, issues: [] });
assert.equal(adapter.parseSetupChecks({ message: "nope" }), null);
assert.equal(adapter.parseSetupChecks("html"), null);
});
test("an unknown severity is skipped, and no more than the limit are kept", () => {
const many = Object.fromEntries(Array.from({ length: 60 }, (_, i) => [`A\\Check${i}`, { name: `Check ${i}`, severity: "warning", description: null }]));
assert.equal(adapter.parseSetupChecks({ system: { ...many, "A\\Odd": { name: "Odd", severity: "fantastic" } } })?.issues.length, adapter.MAX_ISSUES);
});
});
describe("the adapter against a Nextcloud that answers", () => {
let server: Server;
let url = "";
let infoStatus = 200;
let setupStatus = 200;
let cronSeverity = "error";
let installed = true;
const seen: { path: string; auth: string | undefined; ocs: string | undefined }[] = [];
before(async () => {
@@ -72,6 +117,7 @@ describe("what Nextcloud's API says", () => {
};
if (path === "/status.php") return send(200, { installed, maintenance: true, needsDbUpgrade: false, versionstring: "31.0.2" });
if (path === "/ocs/v2.php/apps/serverinfo/api/v1/info") return infoStatus === 200 ? send(200, INFO) : send(infoStatus, {});
if (path === "/settings/setupcheck") return setupStatus === 200 ? send(200, { system: { "OCA\\Settings\\SetupChecks\\CronInfo": { name: "Cron last run", severity: cronSeverity, description: "Last background job execution ran 3 hours ago. Something seems wrong. {link}." } } }) : send(setupStatus, {});
send(404, {});
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
@@ -95,6 +141,23 @@ describe("what Nextcloud's API says", () => {
assert.equal(info.ocs, "true");
});
test("with a login it also asks Nextcloud's setup checks and reads the Cron check and the core update", async () => {
seen.length = 0;
const r = await adapter.createNextcloudAdapter({ url, username: "admin", appPassword: "pw" }).read();
assert.deepEqual(r.cron, { state: "error", text: "Last background job execution ran 3 hours ago. Something seems wrong." });
assert.equal(r.coreUpdate, "31.0.5");
assert.deepEqual(r.setupIssues, []);
assert.equal(seen.find((x) => x.path === "/settings/setupcheck")?.auth, `Basic ${Buffer.from("admin:pw").toString("base64")}`);
assert.equal((await adapter.createNextcloudAdapter({ url }).read()).cron, null, "without a login it is not asked");
});
test("a Nextcloud from before 28 has no such page: cron and the checks are not known, and the rest is read as before", async () => {
setupStatus = 404;
const r = await adapter.createNextcloudAdapter({ url, username: "admin", appPassword: "pw" }).read();
assert.deepEqual([r.cron, r.setupIssues, r.detail, r.freeBytes], [null, null, true, 412e9]);
setupStatus = 200;
});
test("a login that is refused (or an app that is off) costs only the detail, not the status", async () => {
infoStatus = 401;
const r = await adapter.createNextcloudAdapter({ url, username: "admin", appPassword: "wrong" }).read();
@@ -127,6 +190,12 @@ describe("what is stored", () => {
await nc.storeRead(id, read({ detail: false, freeBytes: null, users: null, files: null, active24h: null, appUpdates: null }), new Date());
const s = await state();
assert.deepEqual([s.detail, s.freeBytes, s.appUpdates], [false, null, null]);
await nc.storeRead(id, read({ cron: { state: "error", text: "ran 3 hours ago" }, coreUpdate: "31.0.5", setupIssues: [{ name: "Index", severity: "error", text: "missing" }] }), new Date());
const t = await state();
assert.deepEqual([t.cronState, t.cronText, t.coreUpdate, nc.setupIssuesOf(t)?.map((i) => i.name)], ["error", "ran 3 hours ago", "31.0.5", ["Index"]]);
await nc.storeRead(id, read({ cron: null, setupIssues: null }), new Date());
const u = await state();
assert.deepEqual([u.cronState, nc.setupIssuesOf(u)], [null, null], "not asked is null, not fine and not empty");
});
test("a read that failed is said, and what was stored is left alone", async () => {
@@ -158,6 +227,25 @@ describe("what is wrong", () => {
assert.match(p[1].message, /occ maintenance:mode --off/);
});
test("background jobs: Nextcloud's own Cron check saying error is a warning, with what it said; success, anything else and 'not known' are nothing", () => {
const cron = (state: string | null, text: string | null = "Last background job execution ran 3 hours ago. Something seems wrong.") => nc.nextcloudProblems(1, "Cloud", { ...row(), cronState: state, cronText: text });
assert.deepEqual(cron("error").map((p) => [p.key, p.severity]), [["nextcloud-cron:1", "warning"]]);
assert.match(cron("error")[0].message, /Cloud: its background jobs are not running — Last background job execution ran 3 hours ago\. Something seems wrong\./);
assert.match(cron("error", null)[0].message, /the last run was more than an hour ago/);
for (const state of ["success", "info", "warning", null]) assert.deepEqual(cron(state), [], String(state));
});
test("Nextcloud's other checks that say error are one alert naming them (with the text when there is one); warnings and checks that could not be asked are not alerts", () => {
const issues = (list: { name: string; severity: string; text: string | null }[] | null) => nc.nextcloudProblems(1, "Cloud", { ...row(), setupIssues: list && JSON.stringify(list) });
assert.deepEqual(issues([{ name: "Index", severity: "warning", text: null }]), []);
assert.deepEqual(issues(null), []);
const one = issues([{ name: "Database missing indices", severity: "error", text: "Missing optional index." }, { name: "Headers", severity: "warning", text: null }]);
assert.deepEqual(one.map((p) => p.key), ["nextcloud-checks:1"]);
assert.match(one[0].message, /report an error — Database missing indices \(Missing optional index\.\)$/);
const many = issues(["A", "B", "C", "D", "E"].map((name) => ({ name, severity: "error", text: "t" })));
assert.match(many[0].message, /report 5 errors — A; B; C and 2 more$/);
});
test("the room for the data: a warning under 20 GB, critical under 5 GB", () => {
assert.deepEqual(keys(row({ freeBytes: 20e9 })), []);
assert.deepEqual(keys(row({ freeBytes: 19e9 })), ["nextcloud-disk:1=warning"]);
@@ -176,7 +264,7 @@ describe("the health pass, the page and /metrics", () => {
before(async () => {
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
ncId = (await db.insert(integrations).values({ type: "nextcloud", name: "Family cloud", baseUrl: "http://10.9.8.2", config: JSON.stringify({ url: "http://10.9.8.2" }) }).returning({ id: integrations.id }))[0].id;
await nc.storeRead(ncId, read({ maintenance: true, needsUpgrade: true, freeBytes: 3e9 }), new Date());
await nc.storeRead(ncId, read({ maintenance: true, needsUpgrade: true, freeBytes: 3e9, cron: { state: "error", text: "Last background job execution ran 3 hours ago. Something seems wrong." }, coreUpdate: "31.0.5", setupIssues: [{ name: "Database missing indices", severity: "error", text: null }, { name: "HTTP headers", severity: "warning", text: null }] }), new Date());
await fresh();
const a = express();
a.use((req, _res, next) => {
@@ -204,7 +292,7 @@ describe("the health pass, the page and /metrics", () => {
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
const c = list.find((x) => x.key === `nextcloud-upgrade:${ncId}`)!;
assert.deepEqual([c.severity, c.source], ["critical", `nextcloud:${ncId}`]);
for (const k of ["nextcloud-upgrade", "nextcloud-maintenance", "nextcloud-disk"]) assert.equal(subjectOfConditionKey(`${k}:${ncId}`), `integration:${ncId}`, "maintenance on it silences its alerts");
for (const k of ["nextcloud-upgrade", "nextcloud-maintenance", "nextcloud-disk", "nextcloud-cron", "nextcloud-checks"]) assert.equal(subjectOfConditionKey(`${k}:${ncId}`), `integration:${ncId}`, "maintenance on it silences its alerts");
await stale();
({ snapshot, held } = await collectSnapshot());
assert.deepEqual([snapshot.nextcloud ?? [], held.has(`nextcloud:${ncId}`)], [[], true]);
@@ -218,13 +306,17 @@ describe("the health pass, the page and /metrics", () => {
assert.deepEqual([up.severity, up.category, up.source, up.link, up.target?.type, up.target?.id], ["critical", "services", "Nextcloud", "/nextcloud", "integration", ncId]);
assert.equal(report.alerts.find((x) => x.id === `disk:nextcloud-disk:${ncId}`)?.severity, "critical");
assert.ok(report.alerts.some((x) => x.id === `services:nextcloud-maintenance:${ncId}`));
const cronAlert = report.alerts.find((x) => x.id === `services:nextcloud-cron:${ncId}`)!;
assert.deepEqual([cronAlert.severity, cronAlert.category, cronAlert.link], ["warning", "services", "/nextcloud"]);
assert.ok(report.alerts.some((x) => x.id === `services:nextcloud-checks:${ncId}`));
});
test("anyone signed in sees each Nextcloud; an old reading says nothing is wrong", async () => {
const body = (await (await fetch(base)).json()) as { nextclouds: { id: number; fresh: boolean; state: { maintenance: boolean; freeBytes: number; appUpdates: number } | null; problems: unknown[] }[] };
const n = body.nextclouds.find((x) => x.id === ncId)!;
assert.deepEqual([n.fresh, n.state?.maintenance, n.state?.freeBytes, n.state?.appUpdates], [true, true, 3e9, 1]);
assert.equal(n.problems.length, 3);
assert.equal(n.problems.length, 5);
assert.deepEqual([n.state?.coreUpdate, n.state?.cron?.state, n.state?.setupIssues?.map((i) => i.severity)], ["31.0.5", "error", ["error", "warning"]]);
await stale();
const old = (await (await fetch(base)).json()) as typeof body;
assert.deepEqual(old.nextclouds.find((x) => x.id === ncId)!.problems, []);
@@ -237,6 +329,7 @@ describe("the health pass, the page and /metrics", () => {
assert.match(text, /homelab_nextcloud_free_bytes\{name="Family cloud"\} 3000000000|homelab_nextcloud_free_bytes\{name="Family cloud"\} 3e\+9/);
assert.match(text, /homelab_nextcloud_active_users_24h\{name="Family cloud"\} 2/);
assert.match(text, /homelab_nextcloud_app_updates\{name="Family cloud"\} 1/);
assert.match(text, /homelab_nextcloud_cron_ok\{name="Family cloud"\} 0/);
await stale();
text = await buildMetrics(new Date());
assert.doesNotMatch(text, /homelab_nextcloud/);