Nextcloud cron and Nextcloud's own checks; bulk close and open again
Nextcloud: its own "Cron last run" setup check (GET /settings/setupcheck, Nextcloud 28+, found by class name) raises an alert when background jobs have stopped; other checks that report an error are one alert, warnings are shown on the page. The core version Nextcloud says is available is shown too. Before 28, cron and the checks are unknown, never "fine". Alerts page: bulk Close... (one reason, written on each incident, undoable) and Open again. Bulk Wake now leaves closed alerts alone. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5ae61a1ece
commit
65214f6f0f
25 files changed
+8108
-35
No files matched your search
@@ -121,7 +121,7 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
|
||||
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
|
||||
m = /^duplicati-(?:failed|stale):(\d+)/.exec(key);
|
||||
if (m) return { type: "duplicati", id: Number(m[1]), category: "backup" };
|
||||
m = /^nextcloud-(?:maintenance|upgrade|disk):(\d+)/.exec(key);
|
||||
m = /^nextcloud-(?:maintenance|upgrade|disk|cron|checks):(\d+)/.exec(key);
|
||||
if (m) return { type: "nextcloud", id: Number(m[1]), category: key.startsWith("nextcloud-disk") ? "disk" : "services" };
|
||||
m = /^netbox-(?:workers):(\d+)/.exec(key);
|
||||
if (m) return { type: "netbox", id: Number(m[1]), category: "services" };
|
||||
|
||||
@@ -8,7 +8,7 @@ import { db } from "../db/client.js";
|
||||
import { assets, maintenanceWindows, servers } from "../db/schema.js";
|
||||
import { invalidateAlertsCache, getAlerts } from "./alerts.js";
|
||||
import { clearAck, restoreAck, setAck, type AckInfo, type SetAckInput } from "./alertAcks.js";
|
||||
import { clearSnooze, readSnooze, restoreSnooze, setSnooze, type SnoozeRecord } from "./alertSnoozes.js";
|
||||
import { clearSnooze, readSnooze, restoreSnooze, setClosed, setSnooze, type SnoozeRecord } from "./alertSnoozes.js";
|
||||
import { keyOfAlertId } from "./flapGuard.js";
|
||||
import { impactOf } from "./dependencies.js";
|
||||
import { isRealDate } from "./inventory.js";
|
||||
@@ -237,6 +237,44 @@ export async function restoreAcks(items: AcksUndo["items"]): Promise<number> {
|
||||
export interface SnoozesUndo {
|
||||
action: "restore_snoozes";
|
||||
items: { alertId: string; previous: SnoozeRecord | null }[];
|
||||
/** The undo of closing several: the ones it opens again say so on their incidents. */
|
||||
closed?: boolean;
|
||||
}
|
||||
|
||||
/** Closes each alert the same way (see setClosed): hidden with no end, until it gets worse. Alerts that are no longer there are skipped. */
|
||||
export async function closeAlerts(alertIds: string[], note: string | null, actor: string, now: Date = new Date()): Promise<BulkResult<SnoozesUndo>> {
|
||||
const present = new Map((await getAlerts(false)).alerts.map((a) => [a.id, a]));
|
||||
const skipped: Skipped[] = [];
|
||||
const items: SnoozesUndo["items"] = [];
|
||||
for (const alertId of unique(alertIds)) {
|
||||
const alert = present.get(alertId);
|
||||
if (!alert) {
|
||||
skipped.push({ id: alertId, name: alertId, reason: "it isn't there any more" });
|
||||
continue;
|
||||
}
|
||||
const previous = await readSnooze(alertId);
|
||||
await setClosed({ alertId, conditionKey: keyOfAlertId(alert.id, alert.category), message: alert.message, severity: alert.severity, note, actor }, now);
|
||||
items.push({ alertId, previous });
|
||||
}
|
||||
if (items.length > 0) invalidateAlertsCache();
|
||||
return { changed: items.length, skipped, undo: items.length > 0 ? { action: "restore_snoozes", items, closed: true } : null };
|
||||
}
|
||||
|
||||
/** Opens each closed alert again. One that is only snoozed (or neither) is left as it is. */
|
||||
export async function reopenAlerts(alertIds: string[]): Promise<BulkResult<SnoozesUndo>> {
|
||||
const skipped: Skipped[] = [];
|
||||
const items: SnoozesUndo["items"] = [];
|
||||
for (const alertId of unique(alertIds)) {
|
||||
const previous = await readSnooze(alertId);
|
||||
if (!previous || previous.kind !== "closed") {
|
||||
skipped.push({ id: alertId, name: alertId, reason: "it wasn't closed" });
|
||||
continue;
|
||||
}
|
||||
await clearSnooze(alertId);
|
||||
items.push({ alertId, previous });
|
||||
}
|
||||
if (items.length > 0) invalidateAlertsCache();
|
||||
return { changed: items.length, skipped, undo: items.length > 0 ? { action: "restore_snoozes", items } : null };
|
||||
}
|
||||
|
||||
/** Snoozes each alert until the same time (an earlier snooze is replaced). Alerts that are no longer there are skipped. A time that is not allowed refuses the whole thing, before anything is changed. */
|
||||
@@ -264,8 +302,8 @@ export async function unsnoozeAlerts(alertIds: string[]): Promise<BulkResult<Sno
|
||||
const items: SnoozesUndo["items"] = [];
|
||||
for (const alertId of unique(alertIds)) {
|
||||
const previous = await readSnooze(alertId);
|
||||
if (!previous) {
|
||||
skipped.push({ id: alertId, name: alertId, reason: "it wasn't snoozed" });
|
||||
if (!previous || previous.kind === "closed") {
|
||||
skipped.push({ id: alertId, name: alertId, reason: previous ? "it is closed, not snoozed — use Open again" : "it wasn't snoozed" });
|
||||
continue;
|
||||
}
|
||||
await clearSnooze(alertId);
|
||||
|
||||
@@ -183,7 +183,11 @@ export function describeChange(category: string, action: string, detail: Record<
|
||||
case "note/photo_delete":
|
||||
return { title: `A photo was removed from a note${name ? ` on ${name}` : ""}`, tone: "neutral", detail: null };
|
||||
case "alert/bulk_undo":
|
||||
return { title: detail.what === "snooze" ? "Snoozing several alerts at once was undone" : "Acknowledging several alerts at once was undone", tone: "neutral", detail: null };
|
||||
return { title: detail.what === "snooze" ? "Snoozing several alerts at once was undone" : detail.what === "close" ? "Closing several alerts at once was undone" : "Acknowledging several alerts at once was undone", tone: "neutral", detail: null };
|
||||
case "alert/bulk_close":
|
||||
return { title: `${typeof detail.changed === "number" ? detail.changed : "Several"} alert${detail.changed === 1 ? "" : "s"} closed at once`, tone: "neutral", detail: str(detail.note) };
|
||||
case "alert/bulk_reopen":
|
||||
return { title: `${typeof detail.changed === "number" ? detail.changed : "Several"} closed alert${detail.changed === 1 ? "" : "s"} opened again at once`, tone: "neutral", detail: null };
|
||||
case "alert/bulk_snooze":
|
||||
return { title: `${typeof detail.changed === "number" ? detail.changed : "Several"} alert${detail.changed === 1 ? "" : "s"} snoozed at once`, tone: "neutral", detail: str(detail.until) ? `until ${str(detail.until)!.slice(0, 16).replace("T", " ")} UTC` : null };
|
||||
case "alert/bulk_unsnooze":
|
||||
|
||||
@@ -33,7 +33,7 @@ export function subjectOfConditionKey(key: string): string | null {
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^jellyfin-(?:restart|task|logins):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^nextcloud-(?:maintenance|upgrade|disk):(\d+)/.exec(key);
|
||||
m = /^nextcloud-(?:maintenance|upgrade|disk|cron|checks):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^netbox-(?:workers):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
|
||||
@@ -78,6 +78,7 @@ async function addNextcloudMetrics(out: Exposition, now: Date) {
|
||||
out.add("homelab_nextcloud_maintenance_mode", "gauge", "1 if Nextcloud is in maintenance mode, else 0.", s.maintenance ? 1 : 0, labels);
|
||||
if (s.freeBytes !== null) out.add("homelab_nextcloud_free_bytes", "gauge", "Bytes free where Nextcloud keeps its data.", s.freeBytes, labels);
|
||||
if (s.active24h !== null) out.add("homelab_nextcloud_active_users_24h", "gauge", "Users active in the last 24 hours.", s.active24h, labels);
|
||||
if (s.cronState !== null) out.add("homelab_nextcloud_cron_ok", "gauge", "1 if Nextcloud's own check says its background jobs ran within the last hour, else 0; absent when it could not be asked.", s.cronState === "error" ? 0 : 1, labels);
|
||||
if (s.appUpdates !== null) out.add("homelab_nextcloud_app_updates", "gauge", "How many apps have an update waiting.", s.appUpdates, labels);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, nextcloudState } from "../db/schema.js";
|
||||
import { createNextcloudAdapter, type NextcloudRead } from "../integrations/nextcloud/adapter.js";
|
||||
import { createNextcloudAdapter, type NextcloudRead, type SetupIssue } from "../integrations/nextcloud/adapter.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
@@ -44,6 +44,17 @@ export const isNextcloudFresh = (status: NextcloudSyncStatus | undefined, nowMs:
|
||||
|
||||
export type StateRow = typeof nextcloudState.$inferSelect;
|
||||
|
||||
/** The other setup checks that report an error or a warning; null when the checks could not be asked (not the same as there being none). */
|
||||
export const setupIssuesOf = (row: Pick<StateRow, "setupIssues">): SetupIssue[] | null => {
|
||||
if (row.setupIssues === null) return null;
|
||||
try {
|
||||
const v = JSON.parse(row.setupIssues);
|
||||
return Array.isArray(v) ? (v as SetupIssue[]) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
/** Writes down one read, replacing the last. */
|
||||
export async function storeRead(integrationId: number, r: NextcloudRead, now: Date = new Date()): Promise<void> {
|
||||
const values = {
|
||||
@@ -56,6 +67,10 @@ export async function storeRead(integrationId: number, r: NextcloudRead, now: Da
|
||||
files: r.files,
|
||||
active24h: r.active24h,
|
||||
appUpdates: r.appUpdates,
|
||||
coreUpdate: r.coreUpdate,
|
||||
cronState: r.cron?.state ?? null,
|
||||
cronText: r.cron?.text ?? null,
|
||||
setupIssues: r.setupIssues && JSON.stringify(r.setupIssues),
|
||||
updatedAt: now.toISOString(),
|
||||
};
|
||||
await db.insert(nextcloudState).values({ integrationId, ...values }).onConflictDoUpdate({ target: nextcloudState.integrationId, set: values });
|
||||
@@ -112,13 +127,24 @@ export interface NextcloudProblem {
|
||||
const gb = (bytes: number) => `${(bytes / 1024 ** 3).toFixed(bytes >= 100 * 1024 ** 3 ? 0 : 1)} GB`;
|
||||
|
||||
/** Pure: what is wrong with one Nextcloud, from what is stored for it. */
|
||||
export function nextcloudProblems(id: number, name: string, s: Pick<StateRow, "maintenance" | "needsUpgrade" | "freeBytes">): NextcloudProblem[] {
|
||||
export function nextcloudProblems(id: number, name: string, s: Pick<StateRow, "maintenance" | "needsUpgrade" | "freeBytes"> & Partial<Pick<StateRow, "cronState" | "cronText" | "setupIssues">>): NextcloudProblem[] {
|
||||
const out: NextcloudProblem[] = [];
|
||||
if (s.needsUpgrade) out.push({ key: `nextcloud-upgrade:${id}`, severity: "critical", message: `${name}: its database needs upgrading — nobody can use it until "occ upgrade" has been run` });
|
||||
if (s.maintenance) out.push({ key: `nextcloud-maintenance:${id}`, severity: "warning", message: `${name} is in maintenance mode — nobody can sign in ("occ maintenance:mode --off" ends it)` });
|
||||
if (s.freeBytes !== null && s.freeBytes < FREE_WARN_BYTES) {
|
||||
out.push({ key: `nextcloud-disk:${id}`, severity: s.freeBytes < FREE_CRITICAL_BYTES ? "critical" : "warning", message: `${name}: only ${gb(s.freeBytes)} is left where the data is kept` });
|
||||
}
|
||||
// Nextcloud's own "Cron last run" check says error when the last background job ran more than an hour ago: nothing it does in the background (notifications, clean-up, indexing, sharing
|
||||
// expiry) is being done. Not known (no login, or before Nextcloud 28) is not a problem.
|
||||
if (s.cronState === "error") {
|
||||
out.push({ key: `nextcloud-cron:${id}`, severity: "warning", message: `${name}: its background jobs are not running — ${s.cronText ?? "the last run was more than an hour ago"}` });
|
||||
}
|
||||
// The rest of what Nextcloud's own checks call an error (a warning is only shown on the page).
|
||||
const errors = (setupIssuesOf({ setupIssues: s.setupIssues ?? null }) ?? []).filter((i) => i.severity === "error");
|
||||
if (errors.length > 0) {
|
||||
const names = errors.slice(0, 3).map((i) => i.name).join("; ") + (errors.length > 3 ? ` and ${errors.length - 3} more` : "");
|
||||
out.push({ key: `nextcloud-checks:${id}`, severity: "warning", message: `${name}: Nextcloud's own checks report ${errors.length === 1 ? "an error" : `${errors.length} errors`} — ${names}${errors.length === 1 && errors[0].text ? ` (${errors[0].text})` : ""}` });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user