Files
Homelab-manager/server/test/nextcloud.test.ts
T
bobbanandClaude Sonnet 5.5 65214f6f0f Nextcloud cron and Nextcloud's own checks; bulk close and open again
Nextcloud: its own "Cron last run" setup check (GET /settings/setupcheck,
Nextcloud 28+, found by class name) raises an alert when background jobs have
stopped; other checks that report an error are one alert, warnings are shown
on the page. The core version Nextcloud says is available is shown too. Before
28, cron and the checks are unknown, never "fine".

Alerts page: bulk Close... (one reason, written on each incident, undoable) and
Open again. Bulk Wake now leaves closed alerts alone.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-25 22:12:10 +01:00

338 lines
22 KiB
TypeScript

/**
* Nextcloud: reading what its API says (status.php, and the monitoring app with an administrator's app password), what is stored, what is wrong, the health pass, the alerts, the
* page's route and /metrics.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { createServer as createHttpServer, type Server } from "node:http";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-nextcloud-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, integrations, nextcloudState } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const adapter = await import("../src/integrations/nextcloud/adapter.js");
const nc = await import("../src/services/nextcloudSync.js");
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { nextcloudRouter } = await import("../src/routes/nextcloud.js");
const { buildMetrics } = await import("../src/services/metrics.js");
const { setInternalFlag } = await import("../src/services/settingsStore.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const MIN = 60_000;
type Read = import("../src/integrations/nextcloud/adapter.js").NextcloudRead;
const read = (extra: Partial<Read> = {}): Read => ({ version: "31.0.2", maintenance: false, needsUpgrade: false, detail: true, freeBytes: 500e9, users: 4, files: 120000, active24h: 2, appUpdates: 1, coreUpdate: null, cron: { state: "success", text: "Last background job execution ran 4 minutes ago." }, setupIssues: [], ...extra });
const INFO = { ocs: { meta: { status: "ok" }, data: { nextcloud: { system: { version: "31.0.2.1", freespace: 412e9, apps: { num_installed: 40, num_updates_available: 3, app_updates: { calendar: "5.1.0" } }, update: { lastupdatedat: 1, available: true, available_version: "31.0.5" } }, storage: { num_users: 4, num_files: 120000 } }, activeUsers: { last5minutes: 1, last24hours: 2 } } } };
describe("what Nextcloud's API says", () => {
test("status.php: the version, maintenance mode and a database that wants upgrading", () => {
assert.deepEqual(adapter.parseStatus({ installed: true, maintenance: false, needsDbUpgrade: false, version: "31.0.2.1", versionstring: "31.0.2" }), { version: "31.0.2", maintenance: false, needsUpgrade: false, installed: true });
const m = adapter.parseStatus({ installed: true, maintenance: 1, needsDbUpgrade: "1", version: "31.0.2.1" });
assert.deepEqual([m.maintenance, m.needsUpgrade, m.version], [true, true, "31.0.2.1"]);
assert.equal(adapter.parseStatus({ installed: false }).installed, false);
assert.deepEqual(adapter.parseStatus(null), { version: null, maintenance: false, needsUpgrade: false, installed: true });
});
test("the monitoring app: room left, users, files, who was active and the app updates", () => {
assert.deepEqual(adapter.parseServerInfo(INFO), { freeBytes: 412e9, users: 4, files: 120000, active24h: 2, appUpdates: 3, coreUpdate: "31.0.5" });
assert.equal(adapter.parseServerInfo({ ocs: { meta: {} } }), null, "no data is not an empty answer");
assert.equal(adapter.parseServerInfo({ ocs: { data: { nextcloud: { system: { apps: { app_updates: { a: "1", b: "2" } } } } } } })?.appUpdates, 2, "counted from the list when no count is given");
assert.equal(adapter.parseServerInfo({ ocs: { data: { nextcloud: { system: {} } } } })?.appUpdates, null);
});
test("a core update: its version when Nextcloud says it, 'newer' when it says only that there is one, nothing when there is none", () => {
const withUpdate = (update: unknown) => adapter.parseServerInfo({ ocs: { data: { nextcloud: { system: { update } } } } })?.coreUpdate;
assert.equal(withUpdate({ available: true, available_version: "31.0.5" }), "31.0.5");
assert.equal(withUpdate({ available: true }), "newer");
assert.equal(withUpdate({ available: false, available_version: "31.0.5" }), null);
assert.equal(withUpdate(undefined), null);
});
describe("Nextcloud's own setup checks", () => {
const CRON = "OCA\\Settings\\SetupChecks\\CronInfo";
const checks = (cron: { severity: string; description: string }) => ({
system: {
[CRON]: { name: "Cron last run", ...cron, descriptionParameters: { link: {} }, linkToDoc: null },
"OCA\\Settings\\SetupChecks\\PhpMemoryLimit": { name: "PHP memory limit", severity: "success", description: "ok" },
"OCA\\Settings\\SetupChecks\\DatabaseHasMissingIndices": { name: "Database missing indices", severity: "error", description: "Missing optional index \"x\" in table \"y\". {link}" },
},
security: { "OCA\\Settings\\SetupChecks\\SecurityHeaders": { name: "HTTP headers", severity: "warning", description: "Some headers are not set." } },
});
test("the Cron check is found by its class, not by its name (which is in the language Nextcloud is set to); the others that say error or warning are kept; successes are not", () => {
const r = adapter.parseSetupChecks(checks({ severity: "error", description: "Last background job execution ran 3 hours ago. Something seems wrong. {link}." }))!;
assert.deepEqual(r.cron, { state: "error", text: "Last background job execution ran 3 hours ago. Something seems wrong." });
assert.deepEqual(r.issues.map((i) => [i.name, i.severity, i.text]), [["Database missing indices", "error", 'Missing optional index "x" in table "y".'], ["HTTP headers", "warning", "Some headers are not set."]]);
});
test("a Cron check that is fine is fine", () => {
const r = adapter.parseSetupChecks(checks({ severity: "success", description: "Last background job execution ran 4 minutes ago." }))!;
assert.deepEqual(r.cron, { state: "success", text: "Last background job execution ran 4 minutes ago." });
});
test("inside an OCS envelope it is the same; no checks at all is no problem; something that is not checks is not read", () => {
assert.equal(adapter.parseSetupChecks({ ocs: { data: checks({ severity: "success", description: "ok" }) } })?.cron?.state, "success");
assert.deepEqual(adapter.parseSetupChecks({}), { cron: null, issues: [] });
assert.equal(adapter.parseSetupChecks({ message: "nope" }), null);
assert.equal(adapter.parseSetupChecks("html"), null);
});
test("an unknown severity is skipped, and no more than the limit are kept", () => {
const many = Object.fromEntries(Array.from({ length: 60 }, (_, i) => [`A\\Check${i}`, { name: `Check ${i}`, severity: "warning", description: null }]));
assert.equal(adapter.parseSetupChecks({ system: { ...many, "A\\Odd": { name: "Odd", severity: "fantastic" } } })?.issues.length, adapter.MAX_ISSUES);
});
});
describe("the adapter against a Nextcloud that answers", () => {
let server: Server;
let url = "";
let infoStatus = 200;
let setupStatus = 200;
let cronSeverity = "error";
let installed = true;
const seen: { path: string; auth: string | undefined; ocs: string | undefined }[] = [];
before(async () => {
server = createHttpServer((req, res) => {
const path = (req.url ?? "").split("?")[0];
seen.push({ path, auth: req.headers.authorization, ocs: req.headers["ocs-apirequest"] as string | undefined });
const send = (code: number, body: unknown) => {
res.writeHead(code, { "content-type": "application/json" });
res.end(JSON.stringify(body));
};
if (path === "/status.php") return send(200, { installed, maintenance: true, needsDbUpgrade: false, versionstring: "31.0.2" });
if (path === "/ocs/v2.php/apps/serverinfo/api/v1/info") return infoStatus === 200 ? send(200, INFO) : send(infoStatus, {});
if (path === "/settings/setupcheck") return setupStatus === 200 ? send(200, { system: { "OCA\\Settings\\SetupChecks\\CronInfo": { name: "Cron last run", severity: cronSeverity, description: "Last background job execution ran 3 hours ago. Something seems wrong. {link}." } } }) : send(setupStatus, {});
send(404, {});
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
url = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
});
after(() => server.close());
test("without a login only the status is read, and nothing is sent to the monitoring app", async () => {
seen.length = 0;
const r = await adapter.createNextcloudAdapter({ url: `${url}/` }).read();
assert.deepEqual([r.version, r.maintenance, r.detail, r.freeBytes], ["31.0.2", true, false, null]);
assert.deepEqual(seen.map((s) => s.path), ["/status.php"]);
});
test("with an administrator's app password the monitoring app is asked, the way Nextcloud wants it", async () => {
seen.length = 0;
const r = await adapter.createNextcloudAdapter({ url, username: "admin", appPassword: "pw" }).read();
assert.deepEqual([r.detail, r.freeBytes, r.users, r.active24h, r.appUpdates], [true, 412e9, 4, 2, 3]);
const info = seen.find((s) => s.path.includes("serverinfo"))!;
assert.equal(info.auth, `Basic ${Buffer.from("admin:pw").toString("base64")}`);
assert.equal(info.ocs, "true");
});
test("with a login it also asks Nextcloud's setup checks and reads the Cron check and the core update", async () => {
seen.length = 0;
const r = await adapter.createNextcloudAdapter({ url, username: "admin", appPassword: "pw" }).read();
assert.deepEqual(r.cron, { state: "error", text: "Last background job execution ran 3 hours ago. Something seems wrong." });
assert.equal(r.coreUpdate, "31.0.5");
assert.deepEqual(r.setupIssues, []);
assert.equal(seen.find((x) => x.path === "/settings/setupcheck")?.auth, `Basic ${Buffer.from("admin:pw").toString("base64")}`);
assert.equal((await adapter.createNextcloudAdapter({ url }).read()).cron, null, "without a login it is not asked");
});
test("a Nextcloud from before 28 has no such page: cron and the checks are not known, and the rest is read as before", async () => {
setupStatus = 404;
const r = await adapter.createNextcloudAdapter({ url, username: "admin", appPassword: "pw" }).read();
assert.deepEqual([r.cron, r.setupIssues, r.detail, r.freeBytes], [null, null, true, 412e9]);
setupStatus = 200;
});
test("a login that is refused (or an app that is off) costs only the detail, not the status", async () => {
infoStatus = 401;
const r = await adapter.createNextcloudAdapter({ url, username: "admin", appPassword: "wrong" }).read();
assert.deepEqual([r.version, r.detail, r.freeBytes], ["31.0.2", false, null]);
const ping = await adapter.createNextcloudAdapter({ url, username: "admin", appPassword: "wrong" }).ping();
assert.equal(ping.ok, false);
assert.match(ping.error ?? "", /did not accept the login/);
infoStatus = 200;
});
test("one that says it is not installed, and a dead address, are said", async () => {
installed = false;
await assert.rejects(() => adapter.createNextcloudAdapter({ url }).read(), /not installed/);
installed = true;
assert.equal((await adapter.createNextcloudAdapter({ url: "http://127.0.0.1:1" }).ping()).ok, false);
});
});
});
describe("what is stored", () => {
let id = 0;
before(async () => {
id = (await db.insert(integrations).values({ type: "nextcloud", name: "Cloud", baseUrl: "http://10.9.8.1", config: JSON.stringify({ url: "http://10.9.8.1" }) }).returning({ id: integrations.id }))[0].id;
});
const state = async () => (await db.select().from(nextcloudState).where(eq(nextcloudState.integrationId, id)))[0];
test("the state is replaced at each read; what was not asked stays null", async () => {
await nc.storeRead(id, read(), new Date());
assert.deepEqual([(await state()).freeBytes, (await state()).appUpdates], [500e9, 1]);
await nc.storeRead(id, read({ detail: false, freeBytes: null, users: null, files: null, active24h: null, appUpdates: null }), new Date());
const s = await state();
assert.deepEqual([s.detail, s.freeBytes, s.appUpdates], [false, null, null]);
await nc.storeRead(id, read({ cron: { state: "error", text: "ran 3 hours ago" }, coreUpdate: "31.0.5", setupIssues: [{ name: "Index", severity: "error", text: "missing" }] }), new Date());
const t = await state();
assert.deepEqual([t.cronState, t.cronText, t.coreUpdate, nc.setupIssuesOf(t)?.map((i) => i.name)], ["error", "ran 3 hours ago", "31.0.5", ["Index"]]);
await nc.storeRead(id, read({ cron: null, setupIssues: null }), new Date());
const u = await state();
assert.deepEqual([u.cronState, nc.setupIssuesOf(u)], [null, null], "not asked is null, not fine and not empty");
});
test("a read that failed is said, and what was stored is left alone", async () => {
const ok = await nc.syncNextcloud(id, new Date(), async () => read({ version: "31.0.3" }));
assert.equal(ok.ok, true);
const failed = await nc.syncNextcloud(id, new Date(), async () => {
throw new Error("connection refused");
});
assert.deepEqual([failed.ok, failed.error], [false, "connection refused"]);
assert.equal(failed.at, ok.at);
assert.equal((await state()).version, "31.0.3");
assert.equal(nc.isNextcloudFresh((await nc.nextcloudSyncStatuses())[id], Date.now()), false);
});
});
describe("what is wrong", () => {
const row = (extra: Partial<Parameters<typeof nc.nextcloudProblems>[2]> = {}) => ({ maintenance: false, needsUpgrade: false, freeBytes: 500e9 as number | null, ...extra });
const keys = (r: ReturnType<typeof row>) => nc.nextcloudProblems(1, "Cloud", r).map((p) => `${p.key}=${p.severity}`);
test("nothing wrong says nothing; what was not asked is not a problem", () => {
assert.deepEqual(keys(row()), []);
assert.deepEqual(keys(row({ freeBytes: null })), []);
});
test("a database that wants upgrading is critical, maintenance mode a warning, each saying what to run", () => {
assert.deepEqual(keys(row({ needsUpgrade: true, maintenance: true })), ["nextcloud-upgrade:1=critical", "nextcloud-maintenance:1=warning"]);
const p = nc.nextcloudProblems(1, "Cloud", row({ needsUpgrade: true, maintenance: true }));
assert.match(p[0].message, /occ upgrade/);
assert.match(p[1].message, /occ maintenance:mode --off/);
});
test("background jobs: Nextcloud's own Cron check saying error is a warning, with what it said; success, anything else and 'not known' are nothing", () => {
const cron = (state: string | null, text: string | null = "Last background job execution ran 3 hours ago. Something seems wrong.") => nc.nextcloudProblems(1, "Cloud", { ...row(), cronState: state, cronText: text });
assert.deepEqual(cron("error").map((p) => [p.key, p.severity]), [["nextcloud-cron:1", "warning"]]);
assert.match(cron("error")[0].message, /Cloud: its background jobs are not running — Last background job execution ran 3 hours ago\. Something seems wrong\./);
assert.match(cron("error", null)[0].message, /the last run was more than an hour ago/);
for (const state of ["success", "info", "warning", null]) assert.deepEqual(cron(state), [], String(state));
});
test("Nextcloud's other checks that say error are one alert naming them (with the text when there is one); warnings and checks that could not be asked are not alerts", () => {
const issues = (list: { name: string; severity: string; text: string | null }[] | null) => nc.nextcloudProblems(1, "Cloud", { ...row(), setupIssues: list && JSON.stringify(list) });
assert.deepEqual(issues([{ name: "Index", severity: "warning", text: null }]), []);
assert.deepEqual(issues(null), []);
const one = issues([{ name: "Database missing indices", severity: "error", text: "Missing optional index." }, { name: "Headers", severity: "warning", text: null }]);
assert.deepEqual(one.map((p) => p.key), ["nextcloud-checks:1"]);
assert.match(one[0].message, /report an error — Database missing indices \(Missing optional index\.\)$/);
const many = issues(["A", "B", "C", "D", "E"].map((name) => ({ name, severity: "error", text: "t" })));
assert.match(many[0].message, /report 5 errors — A; B; C and 2 more$/);
});
test("the room for the data: a warning under 20 GB, critical under 5 GB", () => {
assert.deepEqual(keys(row({ freeBytes: 20e9 })), []);
assert.deepEqual(keys(row({ freeBytes: 19e9 })), ["nextcloud-disk:1=warning"]);
assert.deepEqual(keys(row({ freeBytes: 4e9 })), ["nextcloud-disk:1=critical"]);
assert.match(nc.nextcloudProblems(1, "Cloud", row({ freeBytes: 4.2e9 }))[0].message, /only 3\.9 GB is left/);
});
});
describe("the health pass, the page and /metrics", () => {
let ncId = 0;
let app: Server;
let base = "";
const fresh = () => setInternalFlag("nextcloudSync", JSON.stringify({ [ncId]: { at: new Date().toISOString(), ok: true, error: null } }));
const stale = () => setInternalFlag("nextcloudSync", JSON.stringify({ [ncId]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null } }));
before(async () => {
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
ncId = (await db.insert(integrations).values({ type: "nextcloud", name: "Family cloud", baseUrl: "http://10.9.8.2", config: JSON.stringify({ url: "http://10.9.8.2" }) }).returning({ id: integrations.id }))[0].id;
await nc.storeRead(ncId, read({ maintenance: true, needsUpgrade: true, freeBytes: 3e9, cron: { state: "error", text: "Last background job execution ran 3 hours ago. Something seems wrong." }, coreUpdate: "31.0.5", setupIssues: [{ name: "Database missing indices", severity: "error", text: null }, { name: "HTTP headers", severity: "warning", text: null }] }), new Date());
await fresh();
const a = express();
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
next();
});
a.use("/api/nextcloud", nextcloudRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/nextcloud`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("the health pass judges it from what is stored, and holds it when the last read is old", async () => {
let { snapshot, held } = await collectSnapshot();
assert.deepEqual(snapshot.nextcloud?.map((n) => n.integrationId), [ncId]);
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
const c = list.find((x) => x.key === `nextcloud-upgrade:${ncId}`)!;
assert.deepEqual([c.severity, c.source], ["critical", `nextcloud:${ncId}`]);
for (const k of ["nextcloud-upgrade", "nextcloud-maintenance", "nextcloud-disk", "nextcloud-cron", "nextcloud-checks"]) assert.equal(subjectOfConditionKey(`${k}:${ncId}`), `integration:${ncId}`, "maintenance on it silences its alerts");
await stale();
({ snapshot, held } = await collectSnapshot());
assert.deepEqual([snapshot.nextcloud ?? [], held.has(`nextcloud:${ncId}`)], [[], true]);
await fresh();
});
test("the Alerts page shows them: the sign-in problems under Services, the disk under Disk, with where to look", async () => {
invalidateAlertsCache();
const report = await getAlerts(true);
const up = report.alerts.find((x) => x.id === `services:nextcloud-upgrade:${ncId}`)!;
assert.deepEqual([up.severity, up.category, up.source, up.link, up.target?.type, up.target?.id], ["critical", "services", "Nextcloud", "/nextcloud", "integration", ncId]);
assert.equal(report.alerts.find((x) => x.id === `disk:nextcloud-disk:${ncId}`)?.severity, "critical");
assert.ok(report.alerts.some((x) => x.id === `services:nextcloud-maintenance:${ncId}`));
const cronAlert = report.alerts.find((x) => x.id === `services:nextcloud-cron:${ncId}`)!;
assert.deepEqual([cronAlert.severity, cronAlert.category, cronAlert.link], ["warning", "services", "/nextcloud"]);
assert.ok(report.alerts.some((x) => x.id === `services:nextcloud-checks:${ncId}`));
});
test("anyone signed in sees each Nextcloud; an old reading says nothing is wrong", async () => {
const body = (await (await fetch(base)).json()) as { nextclouds: { id: number; fresh: boolean; state: { maintenance: boolean; freeBytes: number; appUpdates: number } | null; problems: unknown[] }[] };
const n = body.nextclouds.find((x) => x.id === ncId)!;
assert.deepEqual([n.fresh, n.state?.maintenance, n.state?.freeBytes, n.state?.appUpdates], [true, true, 3e9, 1]);
assert.equal(n.problems.length, 5);
assert.deepEqual([n.state?.coreUpdate, n.state?.cron?.state, n.state?.setupIssues?.map((i) => i.severity)], ["31.0.5", "error", ["error", "warning"]]);
await stale();
const old = (await (await fetch(base)).json()) as typeof body;
assert.deepEqual(old.nextclouds.find((x) => x.id === ncId)!.problems, []);
await fresh();
});
test("/metrics has maintenance mode, the room left, who was active and the app updates — only while the reading is recent", async () => {
let text = await buildMetrics(new Date());
assert.match(text, /homelab_nextcloud_maintenance_mode\{name="Family cloud"\} 1/);
assert.match(text, /homelab_nextcloud_free_bytes\{name="Family cloud"\} 3000000000|homelab_nextcloud_free_bytes\{name="Family cloud"\} 3e\+9/);
assert.match(text, /homelab_nextcloud_active_users_24h\{name="Family cloud"\} 2/);
assert.match(text, /homelab_nextcloud_app_updates\{name="Family cloud"\} 1/);
assert.match(text, /homelab_nextcloud_cron_ok\{name="Family cloud"\} 0/);
await stale();
text = await buildMetrics(new Date());
assert.doesNotMatch(text, /homelab_nextcloud/);
});
});