Nextcloud cron and Nextcloud's own checks; bulk close and open again
Nextcloud: its own "Cron last run" setup check (GET /settings/setupcheck, Nextcloud 28+, found by class name) raises an alert when background jobs have stopped; other checks that report an error are one alert, warnings are shown on the page. The core version Nextcloud says is available is shown too. Before 28, cron and the checks are unknown, never "fine". Alerts page: bulk Close... (one reason, written on each incident, undoable) and Open again. Bulk Wake now leaves closed alerts alone. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5ae61a1ece
commit
65214f6f0f
25 files changed
+8108
-35
No files matched your search
@@ -36,9 +36,9 @@ await runMigrations();
|
||||
|
||||
const MIN = 60_000;
|
||||
type Read = import("../src/integrations/nextcloud/adapter.js").NextcloudRead;
|
||||
const read = (extra: Partial<Read> = {}): Read => ({ version: "31.0.2", maintenance: false, needsUpgrade: false, detail: true, freeBytes: 500e9, users: 4, files: 120000, active24h: 2, appUpdates: 1, ...extra });
|
||||
const read = (extra: Partial<Read> = {}): Read => ({ version: "31.0.2", maintenance: false, needsUpgrade: false, detail: true, freeBytes: 500e9, users: 4, files: 120000, active24h: 2, appUpdates: 1, coreUpdate: null, cron: { state: "success", text: "Last background job execution ran 4 minutes ago." }, setupIssues: [], ...extra });
|
||||
|
||||
const INFO = { ocs: { meta: { status: "ok" }, data: { nextcloud: { system: { version: "31.0.2.1", freespace: 412e9, apps: { num_installed: 40, num_updates_available: 3, app_updates: { calendar: "5.1.0" } } }, storage: { num_users: 4, num_files: 120000 } }, activeUsers: { last5minutes: 1, last24hours: 2 } } } };
|
||||
const INFO = { ocs: { meta: { status: "ok" }, data: { nextcloud: { system: { version: "31.0.2.1", freespace: 412e9, apps: { num_installed: 40, num_updates_available: 3, app_updates: { calendar: "5.1.0" } }, update: { lastupdatedat: 1, available: true, available_version: "31.0.5" } }, storage: { num_users: 4, num_files: 120000 } }, activeUsers: { last5minutes: 1, last24hours: 2 } } } };
|
||||
|
||||
describe("what Nextcloud's API says", () => {
|
||||
test("status.php: the version, maintenance mode and a database that wants upgrading", () => {
|
||||
@@ -50,16 +50,61 @@ describe("what Nextcloud's API says", () => {
|
||||
});
|
||||
|
||||
test("the monitoring app: room left, users, files, who was active and the app updates", () => {
|
||||
assert.deepEqual(adapter.parseServerInfo(INFO), { freeBytes: 412e9, users: 4, files: 120000, active24h: 2, appUpdates: 3 });
|
||||
assert.deepEqual(adapter.parseServerInfo(INFO), { freeBytes: 412e9, users: 4, files: 120000, active24h: 2, appUpdates: 3, coreUpdate: "31.0.5" });
|
||||
assert.equal(adapter.parseServerInfo({ ocs: { meta: {} } }), null, "no data is not an empty answer");
|
||||
assert.equal(adapter.parseServerInfo({ ocs: { data: { nextcloud: { system: { apps: { app_updates: { a: "1", b: "2" } } } } } } })?.appUpdates, 2, "counted from the list when no count is given");
|
||||
assert.equal(adapter.parseServerInfo({ ocs: { data: { nextcloud: { system: {} } } } })?.appUpdates, null);
|
||||
});
|
||||
|
||||
test("a core update: its version when Nextcloud says it, 'newer' when it says only that there is one, nothing when there is none", () => {
|
||||
const withUpdate = (update: unknown) => adapter.parseServerInfo({ ocs: { data: { nextcloud: { system: { update } } } } })?.coreUpdate;
|
||||
assert.equal(withUpdate({ available: true, available_version: "31.0.5" }), "31.0.5");
|
||||
assert.equal(withUpdate({ available: true }), "newer");
|
||||
assert.equal(withUpdate({ available: false, available_version: "31.0.5" }), null);
|
||||
assert.equal(withUpdate(undefined), null);
|
||||
});
|
||||
|
||||
describe("Nextcloud's own setup checks", () => {
|
||||
const CRON = "OCA\\Settings\\SetupChecks\\CronInfo";
|
||||
const checks = (cron: { severity: string; description: string }) => ({
|
||||
system: {
|
||||
[CRON]: { name: "Cron last run", ...cron, descriptionParameters: { link: {} }, linkToDoc: null },
|
||||
"OCA\\Settings\\SetupChecks\\PhpMemoryLimit": { name: "PHP memory limit", severity: "success", description: "ok" },
|
||||
"OCA\\Settings\\SetupChecks\\DatabaseHasMissingIndices": { name: "Database missing indices", severity: "error", description: "Missing optional index \"x\" in table \"y\". {link}" },
|
||||
},
|
||||
security: { "OCA\\Settings\\SetupChecks\\SecurityHeaders": { name: "HTTP headers", severity: "warning", description: "Some headers are not set." } },
|
||||
});
|
||||
|
||||
test("the Cron check is found by its class, not by its name (which is in the language Nextcloud is set to); the others that say error or warning are kept; successes are not", () => {
|
||||
const r = adapter.parseSetupChecks(checks({ severity: "error", description: "Last background job execution ran 3 hours ago. Something seems wrong. {link}." }))!;
|
||||
assert.deepEqual(r.cron, { state: "error", text: "Last background job execution ran 3 hours ago. Something seems wrong." });
|
||||
assert.deepEqual(r.issues.map((i) => [i.name, i.severity, i.text]), [["Database missing indices", "error", 'Missing optional index "x" in table "y".'], ["HTTP headers", "warning", "Some headers are not set."]]);
|
||||
});
|
||||
|
||||
test("a Cron check that is fine is fine", () => {
|
||||
const r = adapter.parseSetupChecks(checks({ severity: "success", description: "Last background job execution ran 4 minutes ago." }))!;
|
||||
assert.deepEqual(r.cron, { state: "success", text: "Last background job execution ran 4 minutes ago." });
|
||||
});
|
||||
|
||||
test("inside an OCS envelope it is the same; no checks at all is no problem; something that is not checks is not read", () => {
|
||||
assert.equal(adapter.parseSetupChecks({ ocs: { data: checks({ severity: "success", description: "ok" }) } })?.cron?.state, "success");
|
||||
assert.deepEqual(adapter.parseSetupChecks({}), { cron: null, issues: [] });
|
||||
assert.equal(adapter.parseSetupChecks({ message: "nope" }), null);
|
||||
assert.equal(adapter.parseSetupChecks("html"), null);
|
||||
});
|
||||
|
||||
test("an unknown severity is skipped, and no more than the limit are kept", () => {
|
||||
const many = Object.fromEntries(Array.from({ length: 60 }, (_, i) => [`A\\Check${i}`, { name: `Check ${i}`, severity: "warning", description: null }]));
|
||||
assert.equal(adapter.parseSetupChecks({ system: { ...many, "A\\Odd": { name: "Odd", severity: "fantastic" } } })?.issues.length, adapter.MAX_ISSUES);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the adapter against a Nextcloud that answers", () => {
|
||||
let server: Server;
|
||||
let url = "";
|
||||
let infoStatus = 200;
|
||||
let setupStatus = 200;
|
||||
let cronSeverity = "error";
|
||||
let installed = true;
|
||||
const seen: { path: string; auth: string | undefined; ocs: string | undefined }[] = [];
|
||||
before(async () => {
|
||||
@@ -72,6 +117,7 @@ describe("what Nextcloud's API says", () => {
|
||||
};
|
||||
if (path === "/status.php") return send(200, { installed, maintenance: true, needsDbUpgrade: false, versionstring: "31.0.2" });
|
||||
if (path === "/ocs/v2.php/apps/serverinfo/api/v1/info") return infoStatus === 200 ? send(200, INFO) : send(infoStatus, {});
|
||||
if (path === "/settings/setupcheck") return setupStatus === 200 ? send(200, { system: { "OCA\\Settings\\SetupChecks\\CronInfo": { name: "Cron last run", severity: cronSeverity, description: "Last background job execution ran 3 hours ago. Something seems wrong. {link}." } } }) : send(setupStatus, {});
|
||||
send(404, {});
|
||||
});
|
||||
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
@@ -95,6 +141,23 @@ describe("what Nextcloud's API says", () => {
|
||||
assert.equal(info.ocs, "true");
|
||||
});
|
||||
|
||||
test("with a login it also asks Nextcloud's setup checks and reads the Cron check and the core update", async () => {
|
||||
seen.length = 0;
|
||||
const r = await adapter.createNextcloudAdapter({ url, username: "admin", appPassword: "pw" }).read();
|
||||
assert.deepEqual(r.cron, { state: "error", text: "Last background job execution ran 3 hours ago. Something seems wrong." });
|
||||
assert.equal(r.coreUpdate, "31.0.5");
|
||||
assert.deepEqual(r.setupIssues, []);
|
||||
assert.equal(seen.find((x) => x.path === "/settings/setupcheck")?.auth, `Basic ${Buffer.from("admin:pw").toString("base64")}`);
|
||||
assert.equal((await adapter.createNextcloudAdapter({ url }).read()).cron, null, "without a login it is not asked");
|
||||
});
|
||||
|
||||
test("a Nextcloud from before 28 has no such page: cron and the checks are not known, and the rest is read as before", async () => {
|
||||
setupStatus = 404;
|
||||
const r = await adapter.createNextcloudAdapter({ url, username: "admin", appPassword: "pw" }).read();
|
||||
assert.deepEqual([r.cron, r.setupIssues, r.detail, r.freeBytes], [null, null, true, 412e9]);
|
||||
setupStatus = 200;
|
||||
});
|
||||
|
||||
test("a login that is refused (or an app that is off) costs only the detail, not the status", async () => {
|
||||
infoStatus = 401;
|
||||
const r = await adapter.createNextcloudAdapter({ url, username: "admin", appPassword: "wrong" }).read();
|
||||
@@ -127,6 +190,12 @@ describe("what is stored", () => {
|
||||
await nc.storeRead(id, read({ detail: false, freeBytes: null, users: null, files: null, active24h: null, appUpdates: null }), new Date());
|
||||
const s = await state();
|
||||
assert.deepEqual([s.detail, s.freeBytes, s.appUpdates], [false, null, null]);
|
||||
await nc.storeRead(id, read({ cron: { state: "error", text: "ran 3 hours ago" }, coreUpdate: "31.0.5", setupIssues: [{ name: "Index", severity: "error", text: "missing" }] }), new Date());
|
||||
const t = await state();
|
||||
assert.deepEqual([t.cronState, t.cronText, t.coreUpdate, nc.setupIssuesOf(t)?.map((i) => i.name)], ["error", "ran 3 hours ago", "31.0.5", ["Index"]]);
|
||||
await nc.storeRead(id, read({ cron: null, setupIssues: null }), new Date());
|
||||
const u = await state();
|
||||
assert.deepEqual([u.cronState, nc.setupIssuesOf(u)], [null, null], "not asked is null, not fine and not empty");
|
||||
});
|
||||
|
||||
test("a read that failed is said, and what was stored is left alone", async () => {
|
||||
@@ -158,6 +227,25 @@ describe("what is wrong", () => {
|
||||
assert.match(p[1].message, /occ maintenance:mode --off/);
|
||||
});
|
||||
|
||||
test("background jobs: Nextcloud's own Cron check saying error is a warning, with what it said; success, anything else and 'not known' are nothing", () => {
|
||||
const cron = (state: string | null, text: string | null = "Last background job execution ran 3 hours ago. Something seems wrong.") => nc.nextcloudProblems(1, "Cloud", { ...row(), cronState: state, cronText: text });
|
||||
assert.deepEqual(cron("error").map((p) => [p.key, p.severity]), [["nextcloud-cron:1", "warning"]]);
|
||||
assert.match(cron("error")[0].message, /Cloud: its background jobs are not running — Last background job execution ran 3 hours ago\. Something seems wrong\./);
|
||||
assert.match(cron("error", null)[0].message, /the last run was more than an hour ago/);
|
||||
for (const state of ["success", "info", "warning", null]) assert.deepEqual(cron(state), [], String(state));
|
||||
});
|
||||
|
||||
test("Nextcloud's other checks that say error are one alert naming them (with the text when there is one); warnings and checks that could not be asked are not alerts", () => {
|
||||
const issues = (list: { name: string; severity: string; text: string | null }[] | null) => nc.nextcloudProblems(1, "Cloud", { ...row(), setupIssues: list && JSON.stringify(list) });
|
||||
assert.deepEqual(issues([{ name: "Index", severity: "warning", text: null }]), []);
|
||||
assert.deepEqual(issues(null), []);
|
||||
const one = issues([{ name: "Database missing indices", severity: "error", text: "Missing optional index." }, { name: "Headers", severity: "warning", text: null }]);
|
||||
assert.deepEqual(one.map((p) => p.key), ["nextcloud-checks:1"]);
|
||||
assert.match(one[0].message, /report an error — Database missing indices \(Missing optional index\.\)$/);
|
||||
const many = issues(["A", "B", "C", "D", "E"].map((name) => ({ name, severity: "error", text: "t" })));
|
||||
assert.match(many[0].message, /report 5 errors — A; B; C and 2 more$/);
|
||||
});
|
||||
|
||||
test("the room for the data: a warning under 20 GB, critical under 5 GB", () => {
|
||||
assert.deepEqual(keys(row({ freeBytes: 20e9 })), []);
|
||||
assert.deepEqual(keys(row({ freeBytes: 19e9 })), ["nextcloud-disk:1=warning"]);
|
||||
@@ -176,7 +264,7 @@ describe("the health pass, the page and /metrics", () => {
|
||||
before(async () => {
|
||||
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
|
||||
ncId = (await db.insert(integrations).values({ type: "nextcloud", name: "Family cloud", baseUrl: "http://10.9.8.2", config: JSON.stringify({ url: "http://10.9.8.2" }) }).returning({ id: integrations.id }))[0].id;
|
||||
await nc.storeRead(ncId, read({ maintenance: true, needsUpgrade: true, freeBytes: 3e9 }), new Date());
|
||||
await nc.storeRead(ncId, read({ maintenance: true, needsUpgrade: true, freeBytes: 3e9, cron: { state: "error", text: "Last background job execution ran 3 hours ago. Something seems wrong." }, coreUpdate: "31.0.5", setupIssues: [{ name: "Database missing indices", severity: "error", text: null }, { name: "HTTP headers", severity: "warning", text: null }] }), new Date());
|
||||
await fresh();
|
||||
const a = express();
|
||||
a.use((req, _res, next) => {
|
||||
@@ -204,7 +292,7 @@ describe("the health pass, the page and /metrics", () => {
|
||||
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
|
||||
const c = list.find((x) => x.key === `nextcloud-upgrade:${ncId}`)!;
|
||||
assert.deepEqual([c.severity, c.source], ["critical", `nextcloud:${ncId}`]);
|
||||
for (const k of ["nextcloud-upgrade", "nextcloud-maintenance", "nextcloud-disk"]) assert.equal(subjectOfConditionKey(`${k}:${ncId}`), `integration:${ncId}`, "maintenance on it silences its alerts");
|
||||
for (const k of ["nextcloud-upgrade", "nextcloud-maintenance", "nextcloud-disk", "nextcloud-cron", "nextcloud-checks"]) assert.equal(subjectOfConditionKey(`${k}:${ncId}`), `integration:${ncId}`, "maintenance on it silences its alerts");
|
||||
await stale();
|
||||
({ snapshot, held } = await collectSnapshot());
|
||||
assert.deepEqual([snapshot.nextcloud ?? [], held.has(`nextcloud:${ncId}`)], [[], true]);
|
||||
@@ -218,13 +306,17 @@ describe("the health pass, the page and /metrics", () => {
|
||||
assert.deepEqual([up.severity, up.category, up.source, up.link, up.target?.type, up.target?.id], ["critical", "services", "Nextcloud", "/nextcloud", "integration", ncId]);
|
||||
assert.equal(report.alerts.find((x) => x.id === `disk:nextcloud-disk:${ncId}`)?.severity, "critical");
|
||||
assert.ok(report.alerts.some((x) => x.id === `services:nextcloud-maintenance:${ncId}`));
|
||||
const cronAlert = report.alerts.find((x) => x.id === `services:nextcloud-cron:${ncId}`)!;
|
||||
assert.deepEqual([cronAlert.severity, cronAlert.category, cronAlert.link], ["warning", "services", "/nextcloud"]);
|
||||
assert.ok(report.alerts.some((x) => x.id === `services:nextcloud-checks:${ncId}`));
|
||||
});
|
||||
|
||||
test("anyone signed in sees each Nextcloud; an old reading says nothing is wrong", async () => {
|
||||
const body = (await (await fetch(base)).json()) as { nextclouds: { id: number; fresh: boolean; state: { maintenance: boolean; freeBytes: number; appUpdates: number } | null; problems: unknown[] }[] };
|
||||
const n = body.nextclouds.find((x) => x.id === ncId)!;
|
||||
assert.deepEqual([n.fresh, n.state?.maintenance, n.state?.freeBytes, n.state?.appUpdates], [true, true, 3e9, 1]);
|
||||
assert.equal(n.problems.length, 3);
|
||||
assert.equal(n.problems.length, 5);
|
||||
assert.deepEqual([n.state?.coreUpdate, n.state?.cron?.state, n.state?.setupIssues?.map((i) => i.severity)], ["31.0.5", "error", ["error", "warning"]]);
|
||||
await stale();
|
||||
const old = (await (await fetch(base)).json()) as typeof body;
|
||||
assert.deepEqual(old.nextclouds.find((x) => x.id === ncId)!.problems, []);
|
||||
@@ -237,6 +329,7 @@ describe("the health pass, the page and /metrics", () => {
|
||||
assert.match(text, /homelab_nextcloud_free_bytes\{name="Family cloud"\} 3000000000|homelab_nextcloud_free_bytes\{name="Family cloud"\} 3e\+9/);
|
||||
assert.match(text, /homelab_nextcloud_active_users_24h\{name="Family cloud"\} 2/);
|
||||
assert.match(text, /homelab_nextcloud_app_updates\{name="Family cloud"\} 1/);
|
||||
assert.match(text, /homelab_nextcloud_cron_ok\{name="Family cloud"\} 0/);
|
||||
await stale();
|
||||
text = await buildMetrics(new Date());
|
||||
assert.doesNotMatch(text, /homelab_nextcloud/);
|
||||
|
||||
Reference in new issue
Block a user