Nextcloud cron and Nextcloud's own checks; bulk close and open again
Nextcloud: its own "Cron last run" setup check (GET /settings/setupcheck, Nextcloud 28+, found by class name) raises an alert when background jobs have stopped; other checks that report an error are one alert, warnings are shown on the page. The core version Nextcloud says is available is shown too. Before 28, cron and the checks are unknown, never "fine". Alerts page: bulk Close... (one reason, written on each incident, undoable) and Open again. Bulk Wake now leaves closed alerts alone. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5ae61a1ece
commit
65214f6f0f
25 files changed
+8108
-35
No files matched your search
@@ -0,0 +1,4 @@
|
||||
ALTER TABLE `nextcloud_state` ADD `core_update` text;--> statement-breakpoint
|
||||
ALTER TABLE `nextcloud_state` ADD `cron_state` text;--> statement-breakpoint
|
||||
ALTER TABLE `nextcloud_state` ADD `cron_text` text;--> statement-breakpoint
|
||||
ALTER TABLE `nextcloud_state` ADD `setup_issues` text;
|
||||
File diff suppressed because it is too large.
Load diff
@@ -463,6 +463,13 @@
|
||||
"when": 1792958566629,
|
||||
"tag": "0065_fine_the_enforcers",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 66,
|
||||
"version": "6",
|
||||
"when": 1792961992575,
|
||||
"tag": "0066_careless_roulette",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1187,6 +1187,10 @@ export const nextcloudState = sqliteTable("nextcloud_state", {
|
||||
files: integer("files"),
|
||||
active24h: integer("active_24h"),
|
||||
appUpdates: integer("app_updates"), // how many apps have an update waiting; null: not asked
|
||||
coreUpdate: text("core_update"), // the version Nextcloud says is available ("newer" when it does not say which); null: none, or not asked
|
||||
cronState: text("cron_state"), // Nextcloud's own "Cron last run" check: success, info, warning or error; null: not known (no login, or before Nextcloud 28)
|
||||
cronText: text("cron_text"), // what that check said ("Last background job execution ran 3 hours ago. Something seems wrong.")
|
||||
setupIssues: text("setup_issues"), // JSON: name, severity, text — the other setup checks that report an error or a warning; null: could not be asked
|
||||
updatedAt: text("updated_at").notNull(), // ISO
|
||||
});
|
||||
|
||||
|
||||
@@ -6,8 +6,13 @@
|
||||
* upgrading** (either way nobody can sign in). With an administrator's app password the *Monitoring* ("serverinfo") app is also asked: how much room is left for the data, how many
|
||||
* users and files, who was active, and how many app updates wait. Nothing is sent to Nextcloud but GET requests for these.
|
||||
*
|
||||
* Endpoints and field names are from Nextcloud's published API (`/status.php`, `/ocs/v2.php/apps/serverinfo/api/v1/info`) and have not been checked against a live server here;
|
||||
* every field is read defensively. Whether background jobs (cron) are running is *not* read: the API this app uses does not say.
|
||||
* With the same login, **Nextcloud's own setup checks** are asked too (`GET /settings/setupcheck`, Nextcloud 28 and later; no administrator is needed): the one called *Cron last run* says whether the
|
||||
* **background jobs are running** (Nextcloud itself calls it an error when the last run was more than an hour ago), and any other check that reports an error or a warning is kept to show.
|
||||
* The serverinfo app does not report cron — its source has no such field — which is why this is a separate request.
|
||||
*
|
||||
* Endpoints and field names are from Nextcloud's published API and source (`/status.php`, `/ocs/v2.php/apps/serverinfo/api/v1/info`, `/settings/setupcheck`, whose results are
|
||||
* { category: { checkClass: { name, severity, description } } }) and have not been checked against a live server here; every field is read defensively. Before Nextcloud 28 there is no such check,
|
||||
* and cron is simply not known.
|
||||
*/
|
||||
import { requestJson } from "../httpJson.js";
|
||||
import { withDiagLogging } from "../../services/diagLog.js";
|
||||
@@ -33,8 +38,25 @@ export interface NextcloudRead {
|
||||
active24h: number | null;
|
||||
/** How many apps have an update waiting; null when not asked. */
|
||||
appUpdates: number | null;
|
||||
/** The version Nextcloud itself says is available, or "newer" when it says one is but not which; null when none is or it was not asked. */
|
||||
coreUpdate: string | null;
|
||||
/** Nextcloud's own "Cron last run" check; null when it could not be asked (no login, or before Nextcloud 28). */
|
||||
cron: { state: SetupSeverity; text: string | null } | null;
|
||||
/** The other setup checks that report an error or a warning, at most MAX_ISSUES; null when the checks could not be asked. */
|
||||
setupIssues: SetupIssue[] | null;
|
||||
}
|
||||
|
||||
export type SetupSeverity = "success" | "info" | "warning" | "error";
|
||||
|
||||
export interface SetupIssue {
|
||||
/** The check's own name for itself, as Nextcloud shows it (in the language it is set to). */
|
||||
name: string;
|
||||
severity: "warning" | "error";
|
||||
text: string | null;
|
||||
}
|
||||
|
||||
export const MAX_ISSUES = 40;
|
||||
|
||||
export interface NextcloudAdapter {
|
||||
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
|
||||
read(): Promise<NextcloudRead>;
|
||||
@@ -53,14 +75,16 @@ export function parseStatus(raw: unknown): Pick<NextcloudRead, "version" | "main
|
||||
}
|
||||
|
||||
/** Pure: what the monitoring app says (`ocs.data`). */
|
||||
export function parseServerInfo(raw: unknown): Pick<NextcloudRead, "freeBytes" | "users" | "files" | "active24h" | "appUpdates"> | null {
|
||||
export function parseServerInfo(raw: unknown): Pick<NextcloudRead, "freeBytes" | "users" | "files" | "active24h" | "appUpdates" | "coreUpdate"> | null {
|
||||
const data = obj(obj(raw).ocs).data;
|
||||
if (data === undefined || data === null || typeof data !== "object") return null;
|
||||
const nc = obj(obj(data).nextcloud);
|
||||
const system = obj(nc.system);
|
||||
const apps = obj(system.apps);
|
||||
const listed = Object.keys(obj(apps.app_updates)).length;
|
||||
const update = obj(system.update);
|
||||
return {
|
||||
coreUpdate: flag(update.available) ? (str(update.available_version) ?? "newer") : null,
|
||||
freeBytes: num(system.freespace),
|
||||
users: num(obj(nc.storage).num_users),
|
||||
files: num(obj(nc.storage).num_files),
|
||||
@@ -69,6 +93,44 @@ export function parseServerInfo(raw: unknown): Pick<NextcloudRead, "freeBytes" |
|
||||
};
|
||||
}
|
||||
|
||||
const SEVERITIES = new Set(["success", "info", "warning", "error"]);
|
||||
|
||||
/** Nextcloud writes a link as a {placeholder} in the text; it is not wanted here, and what is left is tidied. */
|
||||
const plain = (v: unknown): string | null => {
|
||||
const t = str(v);
|
||||
if (!t) return null;
|
||||
const cleaned = t.replace(/\{[A-Za-z0-9_]+\}/g, "").replace(/\s+/g, " ").replace(/\s+([.,;:])/g, "$1").replace(/([.!?])\s*\.+/g, "$1").trim();
|
||||
return cleaned === "" ? null : cleaned;
|
||||
};
|
||||
|
||||
/**
|
||||
* Pure: the setup checks, { category: { checkClass: { name, severity, description } } } (sometimes inside an OCS envelope). Returns Nextcloud's Cron check on its own, and the other
|
||||
* checks that say error or warning; null when the answer is not that shape at all (which is not the same as every check passing).
|
||||
*/
|
||||
export function parseSetupChecks(raw: unknown): { cron: NextcloudRead["cron"]; issues: SetupIssue[] } | null {
|
||||
if (raw === null || typeof raw !== "object" || Array.isArray(raw)) return null;
|
||||
let root = obj(raw);
|
||||
if (root.ocs !== undefined) root = obj(obj(root.ocs).data);
|
||||
else if (root.data !== undefined && Object.keys(root).length <= 2) root = obj(root.data);
|
||||
const categories = Object.values(root).filter((v) => v !== null && typeof v === "object" && !Array.isArray(v));
|
||||
if (categories.length === 0) return Object.keys(root).length === 0 ? { cron: null, issues: [] } : null;
|
||||
let cron: NextcloudRead["cron"] = null;
|
||||
const issues: SetupIssue[] = [];
|
||||
for (const category of categories) {
|
||||
for (const [checkClass, value] of Object.entries(obj(category))) {
|
||||
const r = obj(value);
|
||||
const severity = str(r.severity)?.toLowerCase();
|
||||
if (!severity || !SEVERITIES.has(severity)) continue;
|
||||
if (/(^|\\)CronInfo$/i.test(checkClass)) {
|
||||
cron = { state: severity as SetupSeverity, text: plain(r.description) };
|
||||
continue;
|
||||
}
|
||||
if ((severity === "error" || severity === "warning") && issues.length < MAX_ISSUES) issues.push({ name: str(r.name) ?? checkClass.split("\\").pop() ?? checkClass, severity, text: plain(r.description) });
|
||||
}
|
||||
}
|
||||
return { cron, issues };
|
||||
}
|
||||
|
||||
export function createNextcloudAdapter(config: NextcloudConfig): NextcloudAdapter {
|
||||
const base = String(config.url ?? "").replace(/\/+$/, "");
|
||||
const insecure = config.insecure === true;
|
||||
@@ -87,14 +149,33 @@ export function createNextcloudAdapter(config: NextcloudConfig): NextcloudAdapte
|
||||
const status = parseStatus(await get("/status.php"));
|
||||
if (!status.installed) throw new Error("That Nextcloud says it is not installed yet");
|
||||
let detail: ReturnType<typeof parseServerInfo> = null;
|
||||
let checks: ReturnType<typeof parseSetupChecks> = null;
|
||||
if (login) {
|
||||
try {
|
||||
detail = parseServerInfo(await get("/ocs/v2.php/apps/serverinfo/api/v1/info?format=json", login));
|
||||
} catch {
|
||||
detail = null; // not an administrator, or the Monitoring app is off: the rest is still worth having
|
||||
}
|
||||
try {
|
||||
checks = parseSetupChecks(await get("/settings/setupcheck", login));
|
||||
} catch {
|
||||
checks = null; // before Nextcloud 28 there is no such page: cron is then simply not known
|
||||
}
|
||||
}
|
||||
return { version: status.version, maintenance: status.maintenance, needsUpgrade: status.needsUpgrade, detail: detail !== null, freeBytes: detail?.freeBytes ?? null, users: detail?.users ?? null, files: detail?.files ?? null, active24h: detail?.active24h ?? null, appUpdates: detail?.appUpdates ?? null };
|
||||
return {
|
||||
version: status.version,
|
||||
maintenance: status.maintenance,
|
||||
needsUpgrade: status.needsUpgrade,
|
||||
detail: detail !== null,
|
||||
freeBytes: detail?.freeBytes ?? null,
|
||||
users: detail?.users ?? null,
|
||||
files: detail?.files ?? null,
|
||||
active24h: detail?.active24h ?? null,
|
||||
appUpdates: detail?.appUpdates ?? null,
|
||||
coreUpdate: detail?.coreUpdate ?? null,
|
||||
cron: checks?.cron ?? null,
|
||||
setupIssues: checks ? checks.issues : null,
|
||||
};
|
||||
}
|
||||
|
||||
async function ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }> {
|
||||
|
||||
@@ -4,7 +4,8 @@ import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { AckError, MAX_HOURS, MAX_NOTE } from "../services/alertAcks.js";
|
||||
import { SnoozeError } from "../services/alertSnoozes.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { acknowledgeAlerts, BulkInputError, changeAssets, changeServerTag, MAX_BULK, restoreAcks, restoreAssets, restoreServerParents, restoreSnoozes, setServerParents, setServerTags, silenceTargets, snoozeAlerts, targetsOfAlerts, undoSilence, unacknowledgeAlerts, unsnoozeAlerts } from "../services/bulkActions.js";
|
||||
import { acknowledgeAlerts, BulkInputError, changeAssets, closeAlerts, reopenAlerts, changeServerTag, MAX_BULK, restoreAcks, restoreAssets, restoreServerParents, restoreSnoozes, setServerParents, setServerTags, silenceTargets, snoozeAlerts, targetsOfAlerts, undoSilence, unacknowledgeAlerts, unsnoozeAlerts } from "../services/bulkActions.js";
|
||||
import { addNote, openIncidentIds } from "../services/incidents.js";
|
||||
import { InvalidTagError } from "../services/serverTags.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
@@ -109,7 +110,10 @@ const alertsBody = z.discriminatedUnion("action", [
|
||||
/** When they should come back, as a date and time with its zone. */
|
||||
z.object({ action: z.literal("snooze"), alertIds: stringIds, until: z.string().min(1).max(40) }),
|
||||
z.object({ action: z.literal("unsnooze"), alertIds: stringIds }),
|
||||
z.object({ action: z.literal("restore_snoozes"), items: z.array(z.object({ alertId: z.string().min(1).max(600), previous: snoozeRecord.nullable() })).min(1).max(MAX_BULK) }),
|
||||
/** "Dealt with, or accepted": hidden with no end, until each gets worse. The note is written on each one's incident. */
|
||||
z.object({ action: z.literal("close"), alertIds: stringIds, note: z.string().max(500).nullable().optional() }),
|
||||
z.object({ action: z.literal("reopen"), alertIds: stringIds }),
|
||||
z.object({ action: z.literal("restore_snoozes"), items: z.array(z.object({ alertId: z.string().min(1).max(600), previous: snoozeRecord.nullable() })).min(1).max(MAX_BULK), closed: z.boolean().optional() }),
|
||||
z.object({ action: z.literal("acknowledge"), alertIds: stringIds, note: z.string().max(MAX_NOTE).default(""), assigneeId: z.number().int().positive().nullable().default(null), hours: z.number().int().min(1).max(MAX_HOURS).nullable().default(null) }),
|
||||
z.object({ action: z.literal("unacknowledge"), alertIds: stringIds }),
|
||||
z.object({ action: z.literal("restore_acks"), items: z.array(z.object({ alertId: z.string().min(1).max(600), previous: previousAck })).min(1).max(MAX_BULK) }),
|
||||
@@ -151,9 +155,43 @@ alertsBulkRouter.post("/", asyncHandler(async (req, res) => {
|
||||
await recordAudit({ actor, category: "alert", action: "bulk_unsnooze", detail: { changed: r.changed, alerts: detailIds(body.alertIds) } });
|
||||
return res.json(r);
|
||||
}
|
||||
case "close": {
|
||||
const note = (body.note ?? "").trim() === "" ? null : (body.note ?? "").trim();
|
||||
const who = nameOf(req);
|
||||
const r = await closeAlerts(body.alertIds, note, who);
|
||||
// Each one's incident says it was closed, by whom and why, as when one is closed alone.
|
||||
const open = await openIncidentIds((r.undo?.items ?? []).map((i) => i.alertId));
|
||||
const together = r.changed > 1 ? ` (closed together with ${r.changed - 1} other${r.changed === 2 ? "" : "s"})` : "";
|
||||
for (const item of r.undo?.items ?? []) {
|
||||
const incidentId = open.get(item.alertId);
|
||||
if (incidentId) await addNote(incidentId, who, (note ? "Closed: " + note : "Closed without a note.") + together);
|
||||
}
|
||||
await recordAudit({ actor, category: "alert", action: "bulk_close", detail: { changed: r.changed, note, alerts: detailIds(body.alertIds) } });
|
||||
return res.json(r);
|
||||
}
|
||||
case "reopen": {
|
||||
const who = nameOf(req);
|
||||
const r = await reopenAlerts(body.alertIds);
|
||||
const open = await openIncidentIds((r.undo?.items ?? []).map((i) => i.alertId));
|
||||
for (const item of r.undo?.items ?? []) {
|
||||
const incidentId = open.get(item.alertId);
|
||||
if (incidentId) await addNote(incidentId, who, "Opened again.");
|
||||
}
|
||||
await recordAudit({ actor, category: "alert", action: "bulk_reopen", detail: { changed: r.changed, alerts: detailIds(body.alertIds) } });
|
||||
return res.json(r);
|
||||
}
|
||||
case "restore_snoozes": {
|
||||
const changed = await restoreSnoozes(body.items);
|
||||
await recordAudit({ actor, category: "alert", action: "bulk_undo", detail: { changed, what: "snooze" } });
|
||||
if (body.closed) {
|
||||
// The undo of closing several: those that are open again say so on their incidents.
|
||||
const reopened = body.items.filter((i) => i.previous?.kind !== "closed").map((i) => i.alertId);
|
||||
const open = await openIncidentIds(reopened);
|
||||
for (const alertId of reopened) {
|
||||
const incidentId = open.get(alertId);
|
||||
if (incidentId) await addNote(incidentId, nameOf(req), "Opened again (closing it was undone).");
|
||||
}
|
||||
}
|
||||
await recordAudit({ actor, category: "alert", action: "bulk_undo", detail: { changed, what: body.closed ? "close" : "snooze" } });
|
||||
return res.json({ changed, skipped: [], undo: null });
|
||||
}
|
||||
case "silence": {
|
||||
|
||||
@@ -3,7 +3,7 @@ import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, nextcloudState } from "../db/schema.js";
|
||||
import { requireAuth } from "../auth/middleware.js";
|
||||
import { isNextcloudFresh, nextcloudProblems, nextcloudSyncStatuses } from "../services/nextcloudSync.js";
|
||||
import { isNextcloudFresh, nextcloudProblems, nextcloudSyncStatuses, setupIssuesOf } from "../services/nextcloudSync.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const nextcloudRouter = Router();
|
||||
@@ -40,6 +40,9 @@ nextcloudRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
files: state.files,
|
||||
active24h: state.active24h,
|
||||
appUpdates: state.appUpdates,
|
||||
coreUpdate: state.coreUpdate,
|
||||
cron: state.cronState ? { state: state.cronState, text: state.cronText } : null,
|
||||
setupIssues: setupIssuesOf(state),
|
||||
updatedAt: state.updatedAt,
|
||||
}
|
||||
: null,
|
||||
|
||||
@@ -121,7 +121,7 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
|
||||
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
|
||||
m = /^duplicati-(?:failed|stale):(\d+)/.exec(key);
|
||||
if (m) return { type: "duplicati", id: Number(m[1]), category: "backup" };
|
||||
m = /^nextcloud-(?:maintenance|upgrade|disk):(\d+)/.exec(key);
|
||||
m = /^nextcloud-(?:maintenance|upgrade|disk|cron|checks):(\d+)/.exec(key);
|
||||
if (m) return { type: "nextcloud", id: Number(m[1]), category: key.startsWith("nextcloud-disk") ? "disk" : "services" };
|
||||
m = /^netbox-(?:workers):(\d+)/.exec(key);
|
||||
if (m) return { type: "netbox", id: Number(m[1]), category: "services" };
|
||||
|
||||
@@ -8,7 +8,7 @@ import { db } from "../db/client.js";
|
||||
import { assets, maintenanceWindows, servers } from "../db/schema.js";
|
||||
import { invalidateAlertsCache, getAlerts } from "./alerts.js";
|
||||
import { clearAck, restoreAck, setAck, type AckInfo, type SetAckInput } from "./alertAcks.js";
|
||||
import { clearSnooze, readSnooze, restoreSnooze, setSnooze, type SnoozeRecord } from "./alertSnoozes.js";
|
||||
import { clearSnooze, readSnooze, restoreSnooze, setClosed, setSnooze, type SnoozeRecord } from "./alertSnoozes.js";
|
||||
import { keyOfAlertId } from "./flapGuard.js";
|
||||
import { impactOf } from "./dependencies.js";
|
||||
import { isRealDate } from "./inventory.js";
|
||||
@@ -237,6 +237,44 @@ export async function restoreAcks(items: AcksUndo["items"]): Promise<number> {
|
||||
export interface SnoozesUndo {
|
||||
action: "restore_snoozes";
|
||||
items: { alertId: string; previous: SnoozeRecord | null }[];
|
||||
/** The undo of closing several: the ones it opens again say so on their incidents. */
|
||||
closed?: boolean;
|
||||
}
|
||||
|
||||
/** Closes each alert the same way (see setClosed): hidden with no end, until it gets worse. Alerts that are no longer there are skipped. */
|
||||
export async function closeAlerts(alertIds: string[], note: string | null, actor: string, now: Date = new Date()): Promise<BulkResult<SnoozesUndo>> {
|
||||
const present = new Map((await getAlerts(false)).alerts.map((a) => [a.id, a]));
|
||||
const skipped: Skipped[] = [];
|
||||
const items: SnoozesUndo["items"] = [];
|
||||
for (const alertId of unique(alertIds)) {
|
||||
const alert = present.get(alertId);
|
||||
if (!alert) {
|
||||
skipped.push({ id: alertId, name: alertId, reason: "it isn't there any more" });
|
||||
continue;
|
||||
}
|
||||
const previous = await readSnooze(alertId);
|
||||
await setClosed({ alertId, conditionKey: keyOfAlertId(alert.id, alert.category), message: alert.message, severity: alert.severity, note, actor }, now);
|
||||
items.push({ alertId, previous });
|
||||
}
|
||||
if (items.length > 0) invalidateAlertsCache();
|
||||
return { changed: items.length, skipped, undo: items.length > 0 ? { action: "restore_snoozes", items, closed: true } : null };
|
||||
}
|
||||
|
||||
/** Opens each closed alert again. One that is only snoozed (or neither) is left as it is. */
|
||||
export async function reopenAlerts(alertIds: string[]): Promise<BulkResult<SnoozesUndo>> {
|
||||
const skipped: Skipped[] = [];
|
||||
const items: SnoozesUndo["items"] = [];
|
||||
for (const alertId of unique(alertIds)) {
|
||||
const previous = await readSnooze(alertId);
|
||||
if (!previous || previous.kind !== "closed") {
|
||||
skipped.push({ id: alertId, name: alertId, reason: "it wasn't closed" });
|
||||
continue;
|
||||
}
|
||||
await clearSnooze(alertId);
|
||||
items.push({ alertId, previous });
|
||||
}
|
||||
if (items.length > 0) invalidateAlertsCache();
|
||||
return { changed: items.length, skipped, undo: items.length > 0 ? { action: "restore_snoozes", items } : null };
|
||||
}
|
||||
|
||||
/** Snoozes each alert until the same time (an earlier snooze is replaced). Alerts that are no longer there are skipped. A time that is not allowed refuses the whole thing, before anything is changed. */
|
||||
@@ -264,8 +302,8 @@ export async function unsnoozeAlerts(alertIds: string[]): Promise<BulkResult<Sno
|
||||
const items: SnoozesUndo["items"] = [];
|
||||
for (const alertId of unique(alertIds)) {
|
||||
const previous = await readSnooze(alertId);
|
||||
if (!previous) {
|
||||
skipped.push({ id: alertId, name: alertId, reason: "it wasn't snoozed" });
|
||||
if (!previous || previous.kind === "closed") {
|
||||
skipped.push({ id: alertId, name: alertId, reason: previous ? "it is closed, not snoozed — use Open again" : "it wasn't snoozed" });
|
||||
continue;
|
||||
}
|
||||
await clearSnooze(alertId);
|
||||
|
||||
@@ -183,7 +183,11 @@ export function describeChange(category: string, action: string, detail: Record<
|
||||
case "note/photo_delete":
|
||||
return { title: `A photo was removed from a note${name ? ` on ${name}` : ""}`, tone: "neutral", detail: null };
|
||||
case "alert/bulk_undo":
|
||||
return { title: detail.what === "snooze" ? "Snoozing several alerts at once was undone" : "Acknowledging several alerts at once was undone", tone: "neutral", detail: null };
|
||||
return { title: detail.what === "snooze" ? "Snoozing several alerts at once was undone" : detail.what === "close" ? "Closing several alerts at once was undone" : "Acknowledging several alerts at once was undone", tone: "neutral", detail: null };
|
||||
case "alert/bulk_close":
|
||||
return { title: `${typeof detail.changed === "number" ? detail.changed : "Several"} alert${detail.changed === 1 ? "" : "s"} closed at once`, tone: "neutral", detail: str(detail.note) };
|
||||
case "alert/bulk_reopen":
|
||||
return { title: `${typeof detail.changed === "number" ? detail.changed : "Several"} closed alert${detail.changed === 1 ? "" : "s"} opened again at once`, tone: "neutral", detail: null };
|
||||
case "alert/bulk_snooze":
|
||||
return { title: `${typeof detail.changed === "number" ? detail.changed : "Several"} alert${detail.changed === 1 ? "" : "s"} snoozed at once`, tone: "neutral", detail: str(detail.until) ? `until ${str(detail.until)!.slice(0, 16).replace("T", " ")} UTC` : null };
|
||||
case "alert/bulk_unsnooze":
|
||||
|
||||
@@ -33,7 +33,7 @@ export function subjectOfConditionKey(key: string): string | null {
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^jellyfin-(?:restart|task|logins):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^nextcloud-(?:maintenance|upgrade|disk):(\d+)/.exec(key);
|
||||
m = /^nextcloud-(?:maintenance|upgrade|disk|cron|checks):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^netbox-(?:workers):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
|
||||
@@ -78,6 +78,7 @@ async function addNextcloudMetrics(out: Exposition, now: Date) {
|
||||
out.add("homelab_nextcloud_maintenance_mode", "gauge", "1 if Nextcloud is in maintenance mode, else 0.", s.maintenance ? 1 : 0, labels);
|
||||
if (s.freeBytes !== null) out.add("homelab_nextcloud_free_bytes", "gauge", "Bytes free where Nextcloud keeps its data.", s.freeBytes, labels);
|
||||
if (s.active24h !== null) out.add("homelab_nextcloud_active_users_24h", "gauge", "Users active in the last 24 hours.", s.active24h, labels);
|
||||
if (s.cronState !== null) out.add("homelab_nextcloud_cron_ok", "gauge", "1 if Nextcloud's own check says its background jobs ran within the last hour, else 0; absent when it could not be asked.", s.cronState === "error" ? 0 : 1, labels);
|
||||
if (s.appUpdates !== null) out.add("homelab_nextcloud_app_updates", "gauge", "How many apps have an update waiting.", s.appUpdates, labels);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, nextcloudState } from "../db/schema.js";
|
||||
import { createNextcloudAdapter, type NextcloudRead } from "../integrations/nextcloud/adapter.js";
|
||||
import { createNextcloudAdapter, type NextcloudRead, type SetupIssue } from "../integrations/nextcloud/adapter.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
@@ -44,6 +44,17 @@ export const isNextcloudFresh = (status: NextcloudSyncStatus | undefined, nowMs:
|
||||
|
||||
export type StateRow = typeof nextcloudState.$inferSelect;
|
||||
|
||||
/** The other setup checks that report an error or a warning; null when the checks could not be asked (not the same as there being none). */
|
||||
export const setupIssuesOf = (row: Pick<StateRow, "setupIssues">): SetupIssue[] | null => {
|
||||
if (row.setupIssues === null) return null;
|
||||
try {
|
||||
const v = JSON.parse(row.setupIssues);
|
||||
return Array.isArray(v) ? (v as SetupIssue[]) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
/** Writes down one read, replacing the last. */
|
||||
export async function storeRead(integrationId: number, r: NextcloudRead, now: Date = new Date()): Promise<void> {
|
||||
const values = {
|
||||
@@ -56,6 +67,10 @@ export async function storeRead(integrationId: number, r: NextcloudRead, now: Da
|
||||
files: r.files,
|
||||
active24h: r.active24h,
|
||||
appUpdates: r.appUpdates,
|
||||
coreUpdate: r.coreUpdate,
|
||||
cronState: r.cron?.state ?? null,
|
||||
cronText: r.cron?.text ?? null,
|
||||
setupIssues: r.setupIssues && JSON.stringify(r.setupIssues),
|
||||
updatedAt: now.toISOString(),
|
||||
};
|
||||
await db.insert(nextcloudState).values({ integrationId, ...values }).onConflictDoUpdate({ target: nextcloudState.integrationId, set: values });
|
||||
@@ -112,13 +127,24 @@ export interface NextcloudProblem {
|
||||
const gb = (bytes: number) => `${(bytes / 1024 ** 3).toFixed(bytes >= 100 * 1024 ** 3 ? 0 : 1)} GB`;
|
||||
|
||||
/** Pure: what is wrong with one Nextcloud, from what is stored for it. */
|
||||
export function nextcloudProblems(id: number, name: string, s: Pick<StateRow, "maintenance" | "needsUpgrade" | "freeBytes">): NextcloudProblem[] {
|
||||
export function nextcloudProblems(id: number, name: string, s: Pick<StateRow, "maintenance" | "needsUpgrade" | "freeBytes"> & Partial<Pick<StateRow, "cronState" | "cronText" | "setupIssues">>): NextcloudProblem[] {
|
||||
const out: NextcloudProblem[] = [];
|
||||
if (s.needsUpgrade) out.push({ key: `nextcloud-upgrade:${id}`, severity: "critical", message: `${name}: its database needs upgrading — nobody can use it until "occ upgrade" has been run` });
|
||||
if (s.maintenance) out.push({ key: `nextcloud-maintenance:${id}`, severity: "warning", message: `${name} is in maintenance mode — nobody can sign in ("occ maintenance:mode --off" ends it)` });
|
||||
if (s.freeBytes !== null && s.freeBytes < FREE_WARN_BYTES) {
|
||||
out.push({ key: `nextcloud-disk:${id}`, severity: s.freeBytes < FREE_CRITICAL_BYTES ? "critical" : "warning", message: `${name}: only ${gb(s.freeBytes)} is left where the data is kept` });
|
||||
}
|
||||
// Nextcloud's own "Cron last run" check says error when the last background job ran more than an hour ago: nothing it does in the background (notifications, clean-up, indexing, sharing
|
||||
// expiry) is being done. Not known (no login, or before Nextcloud 28) is not a problem.
|
||||
if (s.cronState === "error") {
|
||||
out.push({ key: `nextcloud-cron:${id}`, severity: "warning", message: `${name}: its background jobs are not running — ${s.cronText ?? "the last run was more than an hour ago"}` });
|
||||
}
|
||||
// The rest of what Nextcloud's own checks call an error (a warning is only shown on the page).
|
||||
const errors = (setupIssuesOf({ setupIssues: s.setupIssues ?? null }) ?? []).filter((i) => i.severity === "error");
|
||||
if (errors.length > 0) {
|
||||
const names = errors.slice(0, 3).map((i) => i.name).join("; ") + (errors.length > 3 ? ` and ${errors.length - 3} more` : "");
|
||||
out.push({ key: `nextcloud-checks:${id}`, severity: "warning", message: `${name}: Nextcloud's own checks report ${errors.length === 1 ? "an error" : `${errors.length} errors`} — ${names}${errors.length === 1 && errors[0].text ? ` (${errors[0].text})` : ""}` });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
|
||||
@@ -204,3 +204,99 @@ describe("until it gets worse", () => {
|
||||
assert.deepEqual(reminders.map((r) => [r.key, r.reason]), [["k:x5", "ended"]]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("closing several at once", () => {
|
||||
const notesOf = async (alertId: string) => {
|
||||
const incidentId = (await openIncidentIds([alertId])).get(alertId)!;
|
||||
return (await getIncident(incidentId))!.notes.map((n) => n.body);
|
||||
};
|
||||
const clear = async () => {
|
||||
await db.delete(alertSnoozes);
|
||||
};
|
||||
|
||||
test("several are closed with one reason; each incident says so; the one that has gone is skipped; they are held back", async () => {
|
||||
await clear();
|
||||
const a = await idOfHeartbeat("jobA");
|
||||
const b = await idOfHeartbeat("jobB");
|
||||
const r = await call("POST", "/bulk", { action: "close", alertIds: [a.id, b.id, "monitoring:nothing:1"], note: " Both retired with the old host. " });
|
||||
assert.equal(r.status, 200);
|
||||
assert.equal(r.body.changed, 2);
|
||||
assert.deepEqual(r.body.skipped.map((x: { reason: string }) => x.reason), ["it isn't there any more"]);
|
||||
const rows = await db.select().from(alertSnoozes);
|
||||
assert.deepEqual(rows.map((x) => [x.kind, x.note, x.snoozedBy]), [["closed", "Both retired with the old host.", "Olga"], ["closed", "Both retired with the old host.", "Olga"]]);
|
||||
assert.deepEqual(rows.map((x) => x.severity), [a.severity, b.severity]);
|
||||
assert.ok((await notesOf(a.id)).includes("Closed: Both retired with the old host. (closed together with 1 other)"));
|
||||
invalidateAlertsCache();
|
||||
const report = await getAlerts(true);
|
||||
assert.ok([a.id, b.id].every((id) => report.alerts.find((x) => x.id === id)?.closed?.by === "Olga"));
|
||||
await call("POST", "/bulk", r.body.undo);
|
||||
});
|
||||
|
||||
test("undo opens them again and says so on each incident; an alert that was closed before the bulk keeps its own closing and note", async () => {
|
||||
await clear();
|
||||
const a = await idOfHeartbeat("jobA");
|
||||
const b = await idOfHeartbeat("jobB");
|
||||
await call("PUT", "/close", { alertId: a.id, note: "mine from before" });
|
||||
const r = await call("POST", "/bulk", { action: "close", alertIds: [a.id, b.id], note: "bulk" });
|
||||
assert.equal(r.body.changed, 2);
|
||||
assert.equal(r.body.undo.closed, true);
|
||||
assert.equal((await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, a.id)))[0].note, "bulk", "replaced");
|
||||
const undone = (id: string) => notesOf(id).then((n) => n.filter((x) => x === "Opened again (closing it was undone).").length);
|
||||
const [beforeA, beforeB] = [await undone(a.id), await undone(b.id)];
|
||||
await call("POST", "/bulk", r.body.undo);
|
||||
const rows = await db.select().from(alertSnoozes);
|
||||
assert.deepEqual(rows.map((x) => [x.alertId, x.note]), [[a.id, "mine from before"]], "a is closed as it was; b is open again");
|
||||
assert.equal(await undone(b.id), beforeB + 1, "b is open again and says so");
|
||||
assert.equal(await undone(a.id), beforeA, "a was not opened by the undo, so says nothing");
|
||||
await clear();
|
||||
});
|
||||
|
||||
test("open again: only closed alerts are opened, one that is merely snoozed is left as it is, and the incident says so; undo closes them again", async () => {
|
||||
await clear();
|
||||
const a = await idOfHeartbeat("jobA");
|
||||
const b = await idOfHeartbeat("jobB");
|
||||
await call("PUT", "/close", { alertId: a.id, note: "accepted" });
|
||||
await call("PUT", "/snooze", { alertId: b.id, until: new Date(Date.now() + 3_600_000).toISOString() });
|
||||
const r = await call("POST", "/bulk", { action: "reopen", alertIds: [a.id, b.id] });
|
||||
assert.equal(r.body.changed, 1);
|
||||
assert.deepEqual(r.body.skipped.map((x: { reason: string }) => x.reason), ["it wasn't closed"]);
|
||||
assert.deepEqual((await db.select().from(alertSnoozes)).map((x) => [x.alertId, x.kind]), [[b.id, "snooze"]]);
|
||||
assert.equal((await notesOf(a.id)).at(-1), "Opened again.");
|
||||
await call("POST", "/bulk", r.body.undo);
|
||||
const back = (await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, a.id)))[0];
|
||||
assert.deepEqual([back.kind, back.note], ["closed", "accepted"]);
|
||||
await clear();
|
||||
});
|
||||
|
||||
test("waking snoozed alerts leaves a closed one alone and says to use Open again", async () => {
|
||||
await clear();
|
||||
const a = await idOfHeartbeat("jobA");
|
||||
await call("PUT", "/close", { alertId: a.id });
|
||||
const r = await call("POST", "/bulk", { action: "unsnooze", alertIds: [a.id] });
|
||||
assert.equal(r.body.changed, 0);
|
||||
assert.match(r.body.skipped[0].reason, /closed, not snoozed — use Open again/);
|
||||
assert.equal((await db.select().from(alertSnoozes)).length, 1, "still closed");
|
||||
await clear();
|
||||
});
|
||||
|
||||
test("a note that is too long is refused and nothing is closed; a viewer cannot; both are in the audit log and the change log", async () => {
|
||||
await clear();
|
||||
const a = await idOfHeartbeat("jobA");
|
||||
const long = await call("POST", "/bulk", { action: "close", alertIds: [a.id], note: "x".repeat(501) });
|
||||
assert.equal(long.status, 400);
|
||||
assert.equal((await db.select().from(alertSnoozes)).length, 0);
|
||||
who = "viewer";
|
||||
assert.equal((await call("POST", "/bulk", { action: "close", alertIds: [a.id] })).status, 403);
|
||||
assert.equal((await call("POST", "/bulk", { action: "reopen", alertIds: [a.id] })).status, 403);
|
||||
who = "operator";
|
||||
await call("POST", "/bulk", { action: "close", alertIds: [a.id], note: "n" });
|
||||
await call("POST", "/bulk", { action: "reopen", alertIds: [a.id] });
|
||||
const actions = (await db.select().from(auditLog)).map((e) => `${e.category}/${e.action}`);
|
||||
assert.ok(actions.includes("alert/bulk_close") && actions.includes("alert/bulk_reopen"));
|
||||
const titles = (await getChangeLog({ days: 1, limit: 400 })).entries.map((e) => e.title);
|
||||
assert.ok(titles.some((t) => /alerts? closed at once$/.test(t)), "close");
|
||||
assert.ok(titles.some((t) => /closed alerts? opened again at once$/.test(t)), "reopen");
|
||||
assert.ok(titles.some((t) => /^Closing several alerts at once was undone$/.test(t)), "undo");
|
||||
await clear();
|
||||
});
|
||||
});
|
||||
@@ -36,9 +36,9 @@ await runMigrations();
|
||||
|
||||
const MIN = 60_000;
|
||||
type Read = import("../src/integrations/nextcloud/adapter.js").NextcloudRead;
|
||||
const read = (extra: Partial<Read> = {}): Read => ({ version: "31.0.2", maintenance: false, needsUpgrade: false, detail: true, freeBytes: 500e9, users: 4, files: 120000, active24h: 2, appUpdates: 1, ...extra });
|
||||
const read = (extra: Partial<Read> = {}): Read => ({ version: "31.0.2", maintenance: false, needsUpgrade: false, detail: true, freeBytes: 500e9, users: 4, files: 120000, active24h: 2, appUpdates: 1, coreUpdate: null, cron: { state: "success", text: "Last background job execution ran 4 minutes ago." }, setupIssues: [], ...extra });
|
||||
|
||||
const INFO = { ocs: { meta: { status: "ok" }, data: { nextcloud: { system: { version: "31.0.2.1", freespace: 412e9, apps: { num_installed: 40, num_updates_available: 3, app_updates: { calendar: "5.1.0" } } }, storage: { num_users: 4, num_files: 120000 } }, activeUsers: { last5minutes: 1, last24hours: 2 } } } };
|
||||
const INFO = { ocs: { meta: { status: "ok" }, data: { nextcloud: { system: { version: "31.0.2.1", freespace: 412e9, apps: { num_installed: 40, num_updates_available: 3, app_updates: { calendar: "5.1.0" } }, update: { lastupdatedat: 1, available: true, available_version: "31.0.5" } }, storage: { num_users: 4, num_files: 120000 } }, activeUsers: { last5minutes: 1, last24hours: 2 } } } };
|
||||
|
||||
describe("what Nextcloud's API says", () => {
|
||||
test("status.php: the version, maintenance mode and a database that wants upgrading", () => {
|
||||
@@ -50,16 +50,61 @@ describe("what Nextcloud's API says", () => {
|
||||
});
|
||||
|
||||
test("the monitoring app: room left, users, files, who was active and the app updates", () => {
|
||||
assert.deepEqual(adapter.parseServerInfo(INFO), { freeBytes: 412e9, users: 4, files: 120000, active24h: 2, appUpdates: 3 });
|
||||
assert.deepEqual(adapter.parseServerInfo(INFO), { freeBytes: 412e9, users: 4, files: 120000, active24h: 2, appUpdates: 3, coreUpdate: "31.0.5" });
|
||||
assert.equal(adapter.parseServerInfo({ ocs: { meta: {} } }), null, "no data is not an empty answer");
|
||||
assert.equal(adapter.parseServerInfo({ ocs: { data: { nextcloud: { system: { apps: { app_updates: { a: "1", b: "2" } } } } } } })?.appUpdates, 2, "counted from the list when no count is given");
|
||||
assert.equal(adapter.parseServerInfo({ ocs: { data: { nextcloud: { system: {} } } } })?.appUpdates, null);
|
||||
});
|
||||
|
||||
test("a core update: its version when Nextcloud says it, 'newer' when it says only that there is one, nothing when there is none", () => {
|
||||
const withUpdate = (update: unknown) => adapter.parseServerInfo({ ocs: { data: { nextcloud: { system: { update } } } } })?.coreUpdate;
|
||||
assert.equal(withUpdate({ available: true, available_version: "31.0.5" }), "31.0.5");
|
||||
assert.equal(withUpdate({ available: true }), "newer");
|
||||
assert.equal(withUpdate({ available: false, available_version: "31.0.5" }), null);
|
||||
assert.equal(withUpdate(undefined), null);
|
||||
});
|
||||
|
||||
describe("Nextcloud's own setup checks", () => {
|
||||
const CRON = "OCA\\Settings\\SetupChecks\\CronInfo";
|
||||
const checks = (cron: { severity: string; description: string }) => ({
|
||||
system: {
|
||||
[CRON]: { name: "Cron last run", ...cron, descriptionParameters: { link: {} }, linkToDoc: null },
|
||||
"OCA\\Settings\\SetupChecks\\PhpMemoryLimit": { name: "PHP memory limit", severity: "success", description: "ok" },
|
||||
"OCA\\Settings\\SetupChecks\\DatabaseHasMissingIndices": { name: "Database missing indices", severity: "error", description: "Missing optional index \"x\" in table \"y\". {link}" },
|
||||
},
|
||||
security: { "OCA\\Settings\\SetupChecks\\SecurityHeaders": { name: "HTTP headers", severity: "warning", description: "Some headers are not set." } },
|
||||
});
|
||||
|
||||
test("the Cron check is found by its class, not by its name (which is in the language Nextcloud is set to); the others that say error or warning are kept; successes are not", () => {
|
||||
const r = adapter.parseSetupChecks(checks({ severity: "error", description: "Last background job execution ran 3 hours ago. Something seems wrong. {link}." }))!;
|
||||
assert.deepEqual(r.cron, { state: "error", text: "Last background job execution ran 3 hours ago. Something seems wrong." });
|
||||
assert.deepEqual(r.issues.map((i) => [i.name, i.severity, i.text]), [["Database missing indices", "error", 'Missing optional index "x" in table "y".'], ["HTTP headers", "warning", "Some headers are not set."]]);
|
||||
});
|
||||
|
||||
test("a Cron check that is fine is fine", () => {
|
||||
const r = adapter.parseSetupChecks(checks({ severity: "success", description: "Last background job execution ran 4 minutes ago." }))!;
|
||||
assert.deepEqual(r.cron, { state: "success", text: "Last background job execution ran 4 minutes ago." });
|
||||
});
|
||||
|
||||
test("inside an OCS envelope it is the same; no checks at all is no problem; something that is not checks is not read", () => {
|
||||
assert.equal(adapter.parseSetupChecks({ ocs: { data: checks({ severity: "success", description: "ok" }) } })?.cron?.state, "success");
|
||||
assert.deepEqual(adapter.parseSetupChecks({}), { cron: null, issues: [] });
|
||||
assert.equal(adapter.parseSetupChecks({ message: "nope" }), null);
|
||||
assert.equal(adapter.parseSetupChecks("html"), null);
|
||||
});
|
||||
|
||||
test("an unknown severity is skipped, and no more than the limit are kept", () => {
|
||||
const many = Object.fromEntries(Array.from({ length: 60 }, (_, i) => [`A\\Check${i}`, { name: `Check ${i}`, severity: "warning", description: null }]));
|
||||
assert.equal(adapter.parseSetupChecks({ system: { ...many, "A\\Odd": { name: "Odd", severity: "fantastic" } } })?.issues.length, adapter.MAX_ISSUES);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the adapter against a Nextcloud that answers", () => {
|
||||
let server: Server;
|
||||
let url = "";
|
||||
let infoStatus = 200;
|
||||
let setupStatus = 200;
|
||||
let cronSeverity = "error";
|
||||
let installed = true;
|
||||
const seen: { path: string; auth: string | undefined; ocs: string | undefined }[] = [];
|
||||
before(async () => {
|
||||
@@ -72,6 +117,7 @@ describe("what Nextcloud's API says", () => {
|
||||
};
|
||||
if (path === "/status.php") return send(200, { installed, maintenance: true, needsDbUpgrade: false, versionstring: "31.0.2" });
|
||||
if (path === "/ocs/v2.php/apps/serverinfo/api/v1/info") return infoStatus === 200 ? send(200, INFO) : send(infoStatus, {});
|
||||
if (path === "/settings/setupcheck") return setupStatus === 200 ? send(200, { system: { "OCA\\Settings\\SetupChecks\\CronInfo": { name: "Cron last run", severity: cronSeverity, description: "Last background job execution ran 3 hours ago. Something seems wrong. {link}." } } }) : send(setupStatus, {});
|
||||
send(404, {});
|
||||
});
|
||||
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
@@ -95,6 +141,23 @@ describe("what Nextcloud's API says", () => {
|
||||
assert.equal(info.ocs, "true");
|
||||
});
|
||||
|
||||
test("with a login it also asks Nextcloud's setup checks and reads the Cron check and the core update", async () => {
|
||||
seen.length = 0;
|
||||
const r = await adapter.createNextcloudAdapter({ url, username: "admin", appPassword: "pw" }).read();
|
||||
assert.deepEqual(r.cron, { state: "error", text: "Last background job execution ran 3 hours ago. Something seems wrong." });
|
||||
assert.equal(r.coreUpdate, "31.0.5");
|
||||
assert.deepEqual(r.setupIssues, []);
|
||||
assert.equal(seen.find((x) => x.path === "/settings/setupcheck")?.auth, `Basic ${Buffer.from("admin:pw").toString("base64")}`);
|
||||
assert.equal((await adapter.createNextcloudAdapter({ url }).read()).cron, null, "without a login it is not asked");
|
||||
});
|
||||
|
||||
test("a Nextcloud from before 28 has no such page: cron and the checks are not known, and the rest is read as before", async () => {
|
||||
setupStatus = 404;
|
||||
const r = await adapter.createNextcloudAdapter({ url, username: "admin", appPassword: "pw" }).read();
|
||||
assert.deepEqual([r.cron, r.setupIssues, r.detail, r.freeBytes], [null, null, true, 412e9]);
|
||||
setupStatus = 200;
|
||||
});
|
||||
|
||||
test("a login that is refused (or an app that is off) costs only the detail, not the status", async () => {
|
||||
infoStatus = 401;
|
||||
const r = await adapter.createNextcloudAdapter({ url, username: "admin", appPassword: "wrong" }).read();
|
||||
@@ -127,6 +190,12 @@ describe("what is stored", () => {
|
||||
await nc.storeRead(id, read({ detail: false, freeBytes: null, users: null, files: null, active24h: null, appUpdates: null }), new Date());
|
||||
const s = await state();
|
||||
assert.deepEqual([s.detail, s.freeBytes, s.appUpdates], [false, null, null]);
|
||||
await nc.storeRead(id, read({ cron: { state: "error", text: "ran 3 hours ago" }, coreUpdate: "31.0.5", setupIssues: [{ name: "Index", severity: "error", text: "missing" }] }), new Date());
|
||||
const t = await state();
|
||||
assert.deepEqual([t.cronState, t.cronText, t.coreUpdate, nc.setupIssuesOf(t)?.map((i) => i.name)], ["error", "ran 3 hours ago", "31.0.5", ["Index"]]);
|
||||
await nc.storeRead(id, read({ cron: null, setupIssues: null }), new Date());
|
||||
const u = await state();
|
||||
assert.deepEqual([u.cronState, nc.setupIssuesOf(u)], [null, null], "not asked is null, not fine and not empty");
|
||||
});
|
||||
|
||||
test("a read that failed is said, and what was stored is left alone", async () => {
|
||||
@@ -158,6 +227,25 @@ describe("what is wrong", () => {
|
||||
assert.match(p[1].message, /occ maintenance:mode --off/);
|
||||
});
|
||||
|
||||
test("background jobs: Nextcloud's own Cron check saying error is a warning, with what it said; success, anything else and 'not known' are nothing", () => {
|
||||
const cron = (state: string | null, text: string | null = "Last background job execution ran 3 hours ago. Something seems wrong.") => nc.nextcloudProblems(1, "Cloud", { ...row(), cronState: state, cronText: text });
|
||||
assert.deepEqual(cron("error").map((p) => [p.key, p.severity]), [["nextcloud-cron:1", "warning"]]);
|
||||
assert.match(cron("error")[0].message, /Cloud: its background jobs are not running — Last background job execution ran 3 hours ago\. Something seems wrong\./);
|
||||
assert.match(cron("error", null)[0].message, /the last run was more than an hour ago/);
|
||||
for (const state of ["success", "info", "warning", null]) assert.deepEqual(cron(state), [], String(state));
|
||||
});
|
||||
|
||||
test("Nextcloud's other checks that say error are one alert naming them (with the text when there is one); warnings and checks that could not be asked are not alerts", () => {
|
||||
const issues = (list: { name: string; severity: string; text: string | null }[] | null) => nc.nextcloudProblems(1, "Cloud", { ...row(), setupIssues: list && JSON.stringify(list) });
|
||||
assert.deepEqual(issues([{ name: "Index", severity: "warning", text: null }]), []);
|
||||
assert.deepEqual(issues(null), []);
|
||||
const one = issues([{ name: "Database missing indices", severity: "error", text: "Missing optional index." }, { name: "Headers", severity: "warning", text: null }]);
|
||||
assert.deepEqual(one.map((p) => p.key), ["nextcloud-checks:1"]);
|
||||
assert.match(one[0].message, /report an error — Database missing indices \(Missing optional index\.\)$/);
|
||||
const many = issues(["A", "B", "C", "D", "E"].map((name) => ({ name, severity: "error", text: "t" })));
|
||||
assert.match(many[0].message, /report 5 errors — A; B; C and 2 more$/);
|
||||
});
|
||||
|
||||
test("the room for the data: a warning under 20 GB, critical under 5 GB", () => {
|
||||
assert.deepEqual(keys(row({ freeBytes: 20e9 })), []);
|
||||
assert.deepEqual(keys(row({ freeBytes: 19e9 })), ["nextcloud-disk:1=warning"]);
|
||||
@@ -176,7 +264,7 @@ describe("the health pass, the page and /metrics", () => {
|
||||
before(async () => {
|
||||
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
|
||||
ncId = (await db.insert(integrations).values({ type: "nextcloud", name: "Family cloud", baseUrl: "http://10.9.8.2", config: JSON.stringify({ url: "http://10.9.8.2" }) }).returning({ id: integrations.id }))[0].id;
|
||||
await nc.storeRead(ncId, read({ maintenance: true, needsUpgrade: true, freeBytes: 3e9 }), new Date());
|
||||
await nc.storeRead(ncId, read({ maintenance: true, needsUpgrade: true, freeBytes: 3e9, cron: { state: "error", text: "Last background job execution ran 3 hours ago. Something seems wrong." }, coreUpdate: "31.0.5", setupIssues: [{ name: "Database missing indices", severity: "error", text: null }, { name: "HTTP headers", severity: "warning", text: null }] }), new Date());
|
||||
await fresh();
|
||||
const a = express();
|
||||
a.use((req, _res, next) => {
|
||||
@@ -204,7 +292,7 @@ describe("the health pass, the page and /metrics", () => {
|
||||
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
|
||||
const c = list.find((x) => x.key === `nextcloud-upgrade:${ncId}`)!;
|
||||
assert.deepEqual([c.severity, c.source], ["critical", `nextcloud:${ncId}`]);
|
||||
for (const k of ["nextcloud-upgrade", "nextcloud-maintenance", "nextcloud-disk"]) assert.equal(subjectOfConditionKey(`${k}:${ncId}`), `integration:${ncId}`, "maintenance on it silences its alerts");
|
||||
for (const k of ["nextcloud-upgrade", "nextcloud-maintenance", "nextcloud-disk", "nextcloud-cron", "nextcloud-checks"]) assert.equal(subjectOfConditionKey(`${k}:${ncId}`), `integration:${ncId}`, "maintenance on it silences its alerts");
|
||||
await stale();
|
||||
({ snapshot, held } = await collectSnapshot());
|
||||
assert.deepEqual([snapshot.nextcloud ?? [], held.has(`nextcloud:${ncId}`)], [[], true]);
|
||||
@@ -218,13 +306,17 @@ describe("the health pass, the page and /metrics", () => {
|
||||
assert.deepEqual([up.severity, up.category, up.source, up.link, up.target?.type, up.target?.id], ["critical", "services", "Nextcloud", "/nextcloud", "integration", ncId]);
|
||||
assert.equal(report.alerts.find((x) => x.id === `disk:nextcloud-disk:${ncId}`)?.severity, "critical");
|
||||
assert.ok(report.alerts.some((x) => x.id === `services:nextcloud-maintenance:${ncId}`));
|
||||
const cronAlert = report.alerts.find((x) => x.id === `services:nextcloud-cron:${ncId}`)!;
|
||||
assert.deepEqual([cronAlert.severity, cronAlert.category, cronAlert.link], ["warning", "services", "/nextcloud"]);
|
||||
assert.ok(report.alerts.some((x) => x.id === `services:nextcloud-checks:${ncId}`));
|
||||
});
|
||||
|
||||
test("anyone signed in sees each Nextcloud; an old reading says nothing is wrong", async () => {
|
||||
const body = (await (await fetch(base)).json()) as { nextclouds: { id: number; fresh: boolean; state: { maintenance: boolean; freeBytes: number; appUpdates: number } | null; problems: unknown[] }[] };
|
||||
const n = body.nextclouds.find((x) => x.id === ncId)!;
|
||||
assert.deepEqual([n.fresh, n.state?.maintenance, n.state?.freeBytes, n.state?.appUpdates], [true, true, 3e9, 1]);
|
||||
assert.equal(n.problems.length, 3);
|
||||
assert.equal(n.problems.length, 5);
|
||||
assert.deepEqual([n.state?.coreUpdate, n.state?.cron?.state, n.state?.setupIssues?.map((i) => i.severity)], ["31.0.5", "error", ["error", "warning"]]);
|
||||
await stale();
|
||||
const old = (await (await fetch(base)).json()) as typeof body;
|
||||
assert.deepEqual(old.nextclouds.find((x) => x.id === ncId)!.problems, []);
|
||||
@@ -237,6 +329,7 @@ describe("the health pass, the page and /metrics", () => {
|
||||
assert.match(text, /homelab_nextcloud_free_bytes\{name="Family cloud"\} 3000000000|homelab_nextcloud_free_bytes\{name="Family cloud"\} 3e\+9/);
|
||||
assert.match(text, /homelab_nextcloud_active_users_24h\{name="Family cloud"\} 2/);
|
||||
assert.match(text, /homelab_nextcloud_app_updates\{name="Family cloud"\} 1/);
|
||||
assert.match(text, /homelab_nextcloud_cron_ok\{name="Family cloud"\} 0/);
|
||||
await stale();
|
||||
text = await buildMetrics(new Date());
|
||||
assert.doesNotMatch(text, /homelab_nextcloud/);
|
||||
|
||||
Reference in new issue
Block a user