Acknowledge and hand over alerts, incident timelines, and built-in database backups
- Alerts can be acknowledged (note, assignee, duration); acknowledged alerts are left out of what needs attention (sidebar, dashboard strip, cards) but stay raised and counted by /metrics - Incidents: each time an alert is raised and goes away is recorded (alert_acks, alert_incidents, incident_notes, migration 0032), with a timeline of acks, maintenance, changes, update runs, related alerts and notes; Incidents page and a digest section - Backups of Homelab Manager's own database: daily checked VACUUM INTO copies, retention, download, check, upload, delete, and restore staged and applied at start-up before the database opens (pre-restore copy kept); alerts and /metrics for stale or failing backups - Docs: README, BACKUP, DATABASE, NOTIFICATIONS, ROLES, API; tests Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
7a5b623444
commit
2744a84719
41 files changed
+6573
-26
No files matched your search
@@ -0,0 +1,133 @@
|
||||
/**
|
||||
* "I'm on it": an acknowledgement says that somebody has seen an alert, may say who is dealing with it and why, and may run out after a
|
||||
* while. It changes nothing about the problem — the alert is still raised, still counted by /metrics and still on the wall display — but
|
||||
* the people looking at the app can tell what nobody has looked at yet from what somebody has, and the sidebar and dashboard count only the
|
||||
* former.
|
||||
*
|
||||
* An acknowledgement belongs to the alert's id (the same stable key the Alerts page keys rows on). It goes away on its own once the alert
|
||||
* has been gone for a while, so the same problem coming back later is a new alert that nobody has looked at, not one that is "still being
|
||||
* dealt with" from last month.
|
||||
*/
|
||||
import { and, eq, inArray, lt } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { alertAcks, users } from "../db/schema.js";
|
||||
import type { Alert } from "./alertTypes.js";
|
||||
|
||||
export interface AckInfo {
|
||||
note: string;
|
||||
assignee: { id: number | null; name: string } | null;
|
||||
ackedBy: string;
|
||||
ackedAt: string;
|
||||
/** ISO — when it runs out; null means it lasts until the alert is gone. */
|
||||
until: string | null;
|
||||
/** Still in force: has no end, or the end hasn't come. */
|
||||
active: boolean;
|
||||
}
|
||||
|
||||
/** An alert that has been absent this long loses its acknowledgement (a blip of a few minutes doesn't). */
|
||||
export const GONE_AFTER_MS = 30 * 60_000;
|
||||
export const MAX_NOTE = 500;
|
||||
export const MAX_HOURS = 24 * 30;
|
||||
|
||||
export class AckError extends Error {
|
||||
constructor(
|
||||
readonly status: number,
|
||||
readonly code: string,
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
const shape = (r: typeof alertAcks.$inferSelect, now: Date): AckInfo => ({
|
||||
note: r.note,
|
||||
assignee: r.assigneeName ? { id: r.assigneeUserId, name: r.assigneeName } : null,
|
||||
ackedBy: r.ackedBy,
|
||||
ackedAt: r.ackedAt,
|
||||
until: r.until,
|
||||
active: r.until === null || Date.parse(r.until) > now.getTime(),
|
||||
});
|
||||
|
||||
export async function loadAcks(now: Date = new Date()): Promise<Map<string, AckInfo>> {
|
||||
const rows = await db.select().from(alertAcks);
|
||||
return new Map(rows.map((r) => [r.alertKey, shape(r, now)]));
|
||||
}
|
||||
|
||||
export async function loadAck(alertKey: string, now: Date = new Date()): Promise<AckInfo | null> {
|
||||
const [row] = await db.select().from(alertAcks).where(eq(alertAcks.alertKey, alertKey)).limit(1);
|
||||
return row ? shape(row, now) : null;
|
||||
}
|
||||
|
||||
export interface SetAckInput {
|
||||
alertId: string;
|
||||
note: string;
|
||||
/** A user to hand it to; null for nobody in particular. */
|
||||
assigneeId: number | null;
|
||||
/** How long the acknowledgement lasts; null means until the alert is gone. */
|
||||
hours: number | null;
|
||||
actor: string;
|
||||
}
|
||||
|
||||
/** Who an alert can be handed to: people who can act on it. */
|
||||
export async function assignableUsers(): Promise<{ id: number; name: string; role: string }[]> {
|
||||
const rows = await db.select().from(users).where(inArray(users.role, ["admin", "operator"]));
|
||||
return rows.map((u) => ({ id: u.id, name: u.name ?? u.email ?? u.oidcSub, role: u.role })).sort((a, b) => a.name.localeCompare(b.name));
|
||||
}
|
||||
|
||||
/** Saves an acknowledgement, replacing an earlier one on the same alert. Returns what changed, for the audit log. */
|
||||
export async function setAck(input: SetAckInput, now: Date = new Date()): Promise<{ ack: AckInfo; before: AckInfo | null }> {
|
||||
const note = input.note.trim();
|
||||
if (note.length > MAX_NOTE) throw new AckError(400, "note_too_long", `Keep the note under ${MAX_NOTE} characters.`);
|
||||
if (input.hours !== null && !(Number.isInteger(input.hours) && input.hours >= 1 && input.hours <= MAX_HOURS)) throw new AckError(400, "invalid_duration", "Choose how long it lasts, up to thirty days — or until the alert is gone.");
|
||||
|
||||
let assigneeName: string | null = null;
|
||||
if (input.assigneeId !== null) {
|
||||
const [u] = await db.select().from(users).where(eq(users.id, input.assigneeId)).limit(1);
|
||||
if (!u || (u.role !== "admin" && u.role !== "operator")) throw new AckError(400, "invalid_assignee", "An alert can be handed to an administrator or an operator.");
|
||||
assigneeName = u.name ?? u.email ?? u.oidcSub;
|
||||
}
|
||||
|
||||
const before = await loadAck(input.alertId, now);
|
||||
const until = input.hours === null ? null : new Date(now.getTime() + input.hours * 3600_000).toISOString();
|
||||
const values = { note, assigneeUserId: input.assigneeId, assigneeName, ackedBy: input.actor, ackedAt: now.toISOString(), until, lastSeenAt: now.toISOString() };
|
||||
await db.insert(alertAcks).values({ alertKey: input.alertId, ...values }).onConflictDoUpdate({ target: alertAcks.alertKey, set: values });
|
||||
return { ack: (await loadAck(input.alertId, now))!, before };
|
||||
}
|
||||
|
||||
export async function clearAck(alertId: string): Promise<AckInfo | null> {
|
||||
const before = await loadAck(alertId);
|
||||
if (!before) return null;
|
||||
await db.delete(alertAcks).where(eq(alertAcks.alertKey, alertId));
|
||||
return before;
|
||||
}
|
||||
|
||||
/**
|
||||
* Called after every collection of the alerts: the acknowledgements of alerts that are there are marked as seen, and those of alerts that
|
||||
* have been gone for half an hour are dropped. Returns how many were dropped.
|
||||
*/
|
||||
export async function reconcileAcks(presentIds: Set<string>, now: Date = new Date()): Promise<number> {
|
||||
const rows = await db.select().from(alertAcks);
|
||||
const seen = rows.filter((r) => presentIds.has(r.alertKey)).map((r) => r.id);
|
||||
if (seen.length > 0) await db.update(alertAcks).set({ lastSeenAt: now.toISOString() }).where(inArray(alertAcks.id, seen));
|
||||
const cutoff = new Date(now.getTime() - GONE_AFTER_MS).toISOString();
|
||||
const gone = rows.filter((r) => !presentIds.has(r.alertKey) && r.lastSeenAt < cutoff).map((r) => r.id);
|
||||
if (gone.length > 0) await db.delete(alertAcks).where(and(inArray(alertAcks.id, gone), lt(alertAcks.lastSeenAt, cutoff)));
|
||||
return gone.length;
|
||||
}
|
||||
|
||||
/** What still needs somebody to look: the alerts that aren't silenced and that nobody has acknowledged (or whose acknowledgement has run out). */
|
||||
export function attentionCounts(alerts: Pick<Alert, "id" | "severity" | "silenced">[], acks: Map<string, AckInfo>): { critical: number; warning: number; acknowledged: number } {
|
||||
let critical = 0;
|
||||
let warning = 0;
|
||||
let acknowledged = 0;
|
||||
for (const a of alerts) {
|
||||
if (a.silenced) continue;
|
||||
if (acks.get(a.id)?.active) {
|
||||
acknowledged++;
|
||||
continue;
|
||||
}
|
||||
if (a.severity === "critical") critical++;
|
||||
else if (a.severity === "warning") warning++;
|
||||
}
|
||||
return { critical, warning, acknowledged };
|
||||
}
|
||||
@@ -15,6 +15,9 @@ import { integrations, secrets, servers } from "../db/schema.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
|
||||
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
|
||||
import { recordFromReport } from "./incidents.js";
|
||||
import { reconcileAcks } from "./alertAcks.js";
|
||||
import { lastAttempt as lastSelfBackup, newestSnapshot, STALE_AFTER_HOURS as SELF_BACKUP_STALE_HOURS } from "./selfBackup.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createUptimeKumaAdapter } from "../integrations/uptimekuma/adapter.js";
|
||||
import { createOsTicketAdapter } from "../integrations/osticket/adapter.js";
|
||||
@@ -53,6 +56,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
|
||||
gitea: { label: "Gitea", link: "/gitea" },
|
||||
dockhand: { label: "Docker", link: "/docker" },
|
||||
tailscale: { label: "Tailscale", link: "/tailscale" },
|
||||
homelab: { label: "Homelab Manager", link: "/settings/backup" },
|
||||
pbs: { label: "Proxmox Backup", link: "/pbs" },
|
||||
uptimekuma: { label: "Uptime Kuma", link: "/uptime-kuma" },
|
||||
osticket: { label: "osTicket", link: "/osticket" },
|
||||
@@ -262,6 +266,29 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
}
|
||||
}),
|
||||
|
||||
// Homelab Manager's own backups: if they are switched on, there should be a recent one. This is what turns a backup that has quietly
|
||||
// stopped working — a full disk, a folder that went read-only — into something somebody hears about before they need it.
|
||||
check("Homelab Manager's own backups", async () => {
|
||||
const { selfBackup } = await getSettings();
|
||||
if (!selfBackup.enabled) return;
|
||||
const newest = newestSnapshot();
|
||||
const last = await lastSelfBackup();
|
||||
const failing = last && !last.ok ? ` The last attempt failed: ${last.error}` : "";
|
||||
if (!newest) {
|
||||
if (failing) push({ category: "backup", severity: "warning", type: "homelab", message: `Homelab Manager has no backup of its own yet.${failing}`, key: "self-backup" });
|
||||
return;
|
||||
}
|
||||
const ageHours = (Date.now() - Date.parse(newest.at)) / 3600_000;
|
||||
if (ageHours <= SELF_BACKUP_STALE_HOURS) return;
|
||||
push({
|
||||
category: "backup",
|
||||
severity: ageHours >= 24 * 7 ? "critical" : "warning",
|
||||
type: "homelab",
|
||||
message: `The newest backup of Homelab Manager's own database is ${ageHours < 72 ? `${Math.round(ageHours)} h` : `${Math.floor(ageHours / 24)} days`} old.${failing}`,
|
||||
key: "self-backup",
|
||||
});
|
||||
}),
|
||||
|
||||
check("Backup verification", async () => {
|
||||
const { problems, sourceFailures } = await collectPbsProblems({ skipSilenced: false });
|
||||
for (const p of problems) {
|
||||
@@ -475,6 +502,8 @@ export async function getAlerts(force: boolean): Promise<AlertsReport & { cached
|
||||
inFlight ??= collectAlerts()
|
||||
.then((report) => {
|
||||
cache = { at: Date.now(), report };
|
||||
// Remembered for the incident record and the acknowledgements — in the background, and never allowed to hold up or break the answer.
|
||||
void recordFromReport(report).then(() => reconcileAcks(new Set(report.alerts.map((a) => a.id)))).catch((err) => console.error("[alerts] couldn't update the incident record:", err));
|
||||
return report;
|
||||
})
|
||||
.finally(() => {
|
||||
|
||||
@@ -0,0 +1,326 @@
|
||||
/**
|
||||
* Incidents: each time an alert is raised and later goes away is one incident, with the moment it began and the moment it ended — so
|
||||
* "what happened on Tuesday night?" has an answer after the alert itself is long gone. Around each one the page puts together everything the
|
||||
* app knows about that time: who acknowledged it and why, the maintenance windows, what was changed on the server, the update runs, the
|
||||
* other alerts that came with it, and the notes people wrote.
|
||||
*
|
||||
* The record is made from the alerts as they are collected (and a check every five minutes so it doesn't depend on anybody looking), by
|
||||
* comparing what is there now with what was open. An alert that isn't there is only called resolved after it has been missing for a couple
|
||||
* of minutes in a row — much longer when a check couldn't run, since "not there" then may only mean "couldn't look" — and never in the first
|
||||
* minutes after the app starts, when server alerts are held back on purpose.
|
||||
*/
|
||||
import { and, asc, desc, eq, gte, inArray, isNotNull, isNull, lte, ne, or } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { alertIncidents, auditLog, incidentNotes, maintenanceWindows, updateRuns } from "../db/schema.js";
|
||||
import type { Alert } from "./alertTypes.js";
|
||||
import { describeAudit, auditTimeToIso, type ActivityTone } from "./serverActivity.js";
|
||||
import { startupGraceRemainingMs } from "./healthMonitor.js";
|
||||
import { loadAck, type AckInfo } from "./alertAcks.js";
|
||||
|
||||
export type IncidentRow = typeof alertIncidents.$inferSelect;
|
||||
|
||||
/** How long an alert must be missing before the incident is called over. */
|
||||
export const MISSING_GRACE_MS = 2 * 60_000;
|
||||
/** The same, when a check couldn't run on the last look: not there may then only mean couldn't look. */
|
||||
export const MISSING_GRACE_BLIND_MS = 30 * 60_000;
|
||||
/** Resolved incidents are kept this long. */
|
||||
export const KEEP_DAYS = 400;
|
||||
/** Other alerts raised this close to an incident, about the same thing, are shown with it. */
|
||||
export const RELATED_WINDOW_MS = 15 * 60_000;
|
||||
|
||||
const RANK: Record<string, number> = { info: 0, warning: 1, critical: 2 };
|
||||
|
||||
export interface SnapshotInput {
|
||||
alerts: Alert[];
|
||||
/** The checks that couldn't run on this look. */
|
||||
blind: number;
|
||||
generatedAt?: string;
|
||||
}
|
||||
|
||||
export interface SnapshotResult {
|
||||
opened: number[];
|
||||
resolved: number[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Brings the incidents in line with the alerts as they are now. Information-only alerts aren't incidents. Safe to call as often as likes:
|
||||
* the same look twice changes nothing.
|
||||
*/
|
||||
export async function recordAlertSnapshot(input: SnapshotInput, now: Date = new Date(), options: { inGrace?: boolean } = {}): Promise<SnapshotResult> {
|
||||
const iso = now.toISOString();
|
||||
const present = new Map(input.alerts.filter((a) => a.severity !== "info").map((a) => [a.id, a]));
|
||||
const open = await db.select().from(alertIncidents).where(isNull(alertIncidents.resolvedAt));
|
||||
const openByKey = new Map(open.map((i) => [i.alertKey, i]));
|
||||
const result: SnapshotResult = { opened: [], resolved: [] };
|
||||
|
||||
for (const [key, a] of present) {
|
||||
const existing = openByKey.get(key);
|
||||
if (existing) {
|
||||
const peak = (RANK[a.severity] ?? 0) > (RANK[existing.peakSeverity] ?? 0) ? a.severity : existing.peakSeverity;
|
||||
await db
|
||||
.update(alertIncidents)
|
||||
.set({ severity: a.severity, peakSeverity: peak, message: a.message, causedBy: a.causedBy?.name ?? null, lastSeenAt: iso, missingSince: null })
|
||||
.where(eq(alertIncidents.id, existing.id));
|
||||
} else {
|
||||
const [row] = await db
|
||||
.insert(alertIncidents)
|
||||
.values({
|
||||
alertKey: key,
|
||||
severity: a.severity,
|
||||
peakSeverity: a.severity,
|
||||
category: a.category,
|
||||
source: a.source,
|
||||
message: a.message,
|
||||
link: a.link,
|
||||
targetType: a.target?.type ?? null,
|
||||
targetId: a.target?.id ?? null,
|
||||
targetName: a.target?.name ?? null,
|
||||
causedBy: a.causedBy?.name ?? null,
|
||||
openedAt: iso,
|
||||
lastSeenAt: iso,
|
||||
})
|
||||
.returning({ id: alertIncidents.id });
|
||||
result.opened.push(row.id);
|
||||
}
|
||||
}
|
||||
|
||||
// What was open and isn't there now. Nothing is called over while the app is still settling after a start, when server alerts are held back.
|
||||
const grace = input.blind > 0 ? MISSING_GRACE_BLIND_MS : MISSING_GRACE_MS;
|
||||
for (const inc of open) {
|
||||
if (present.has(inc.alertKey)) continue;
|
||||
if (options.inGrace) continue;
|
||||
if (inc.missingSince === null) {
|
||||
await db.update(alertIncidents).set({ missingSince: iso }).where(eq(alertIncidents.id, inc.id));
|
||||
} else if (now.getTime() - Date.parse(inc.missingSince) >= grace) {
|
||||
// Over since the first look it was missing, not since this one.
|
||||
await db.update(alertIncidents).set({ resolvedAt: inc.missingSince, missingSince: null }).where(eq(alertIncidents.id, inc.id));
|
||||
result.resolved.push(inc.id);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Called by the collection of the alerts: records what it found, and never lets a problem here stop the alerts from being served. */
|
||||
export async function recordFromReport(report: { alerts: Alert[]; couldntCheck: unknown[] }, now: Date = new Date()): Promise<void> {
|
||||
try {
|
||||
await recordAlertSnapshot({ alerts: report.alerts, blind: report.couldntCheck.length }, now, { inGrace: startupGraceRemainingMs(now.getTime()) > 0 });
|
||||
} catch (err) {
|
||||
console.error("[incidents] couldn't record the alerts:", err);
|
||||
}
|
||||
}
|
||||
|
||||
/** Resolved incidents older than the keeping time are removed (with their notes). */
|
||||
export async function pruneIncidents(now: Date = new Date()): Promise<number> {
|
||||
const cutoff = new Date(now.getTime() - KEEP_DAYS * 86_400_000).toISOString();
|
||||
const old = await db.select({ id: alertIncidents.id }).from(alertIncidents).where(and(isNotNull(alertIncidents.resolvedAt), lte(alertIncidents.resolvedAt, cutoff)));
|
||||
if (old.length === 0) return 0;
|
||||
await db.delete(alertIncidents).where(inArray(alertIncidents.id, old.map((o) => o.id)));
|
||||
return old.length;
|
||||
}
|
||||
|
||||
// ─── Reading them ───────────────────────────────────────────────────────────
|
||||
|
||||
export interface IncidentSummary {
|
||||
id: number;
|
||||
alertKey: string;
|
||||
severity: string;
|
||||
peakSeverity: string;
|
||||
category: string;
|
||||
source: string;
|
||||
message: string;
|
||||
link: string | null;
|
||||
target: { type: string; id: number; name: string } | null;
|
||||
causedBy: string | null;
|
||||
openedAt: string;
|
||||
resolvedAt: string | null;
|
||||
/** In milliseconds: until it was resolved, or until now while it is open. */
|
||||
durationMs: number;
|
||||
open: boolean;
|
||||
notes: number;
|
||||
}
|
||||
|
||||
const summarize = (r: IncidentRow, notes: number, now: Date): IncidentSummary => ({
|
||||
id: r.id,
|
||||
alertKey: r.alertKey,
|
||||
severity: r.severity,
|
||||
peakSeverity: r.peakSeverity,
|
||||
category: r.category,
|
||||
source: r.source,
|
||||
message: r.message,
|
||||
link: r.link,
|
||||
target: r.targetType && r.targetId !== null ? { type: r.targetType, id: r.targetId, name: r.targetName ?? `${r.targetType} #${r.targetId}` } : null,
|
||||
causedBy: r.causedBy,
|
||||
openedAt: r.openedAt,
|
||||
resolvedAt: r.resolvedAt,
|
||||
durationMs: Math.max(0, (r.resolvedAt ? Date.parse(r.resolvedAt) : now.getTime()) - Date.parse(r.openedAt)),
|
||||
open: r.resolvedAt === null,
|
||||
notes,
|
||||
});
|
||||
|
||||
/** Open incidents first (newest first), then the resolved ones, newest first. */
|
||||
export async function listIncidents(options: { limit?: number; now?: Date } = {}): Promise<IncidentSummary[]> {
|
||||
const now = options.now ?? new Date();
|
||||
const limit = options.limit ?? 300;
|
||||
const open = await db.select().from(alertIncidents).where(isNull(alertIncidents.resolvedAt)).orderBy(desc(alertIncidents.openedAt));
|
||||
const closed = await db.select().from(alertIncidents).where(isNotNull(alertIncidents.resolvedAt)).orderBy(desc(alertIncidents.openedAt)).limit(limit);
|
||||
const rows = [...open, ...closed];
|
||||
const noteRows = rows.length > 0 ? await db.select({ incidentId: incidentNotes.incidentId }).from(incidentNotes).where(inArray(incidentNotes.incidentId, rows.map((r) => r.id))) : [];
|
||||
const counts = new Map<number, number>();
|
||||
for (const n of noteRows) counts.set(n.incidentId, (counts.get(n.incidentId) ?? 0) + 1);
|
||||
return rows.map((r) => summarize(r, counts.get(r.id) ?? 0, now));
|
||||
}
|
||||
|
||||
/** The open incident for each alert id, so the Alerts page can link to it. */
|
||||
export async function openIncidentIds(alertKeys: string[]): Promise<Map<string, number>> {
|
||||
if (alertKeys.length === 0) return new Map();
|
||||
const rows = await db.select({ id: alertIncidents.id, key: alertIncidents.alertKey }).from(alertIncidents).where(and(isNull(alertIncidents.resolvedAt), inArray(alertIncidents.alertKey, alertKeys)));
|
||||
return new Map(rows.map((r) => [r.key, r.id]));
|
||||
}
|
||||
|
||||
export type TimelineKind = "alert" | "ack" | "maintenance" | "change" | "health" | "run" | "related" | "note";
|
||||
|
||||
export interface TimelineEvent {
|
||||
/** ISO. */
|
||||
at: string;
|
||||
kind: TimelineKind;
|
||||
tone: ActivityTone;
|
||||
title: string;
|
||||
detail: string | null;
|
||||
actor: string | null;
|
||||
/** For a related alert or a note: what it points at. */
|
||||
incidentId?: number;
|
||||
noteId?: number;
|
||||
}
|
||||
|
||||
export interface IncidentDetail {
|
||||
incident: IncidentSummary;
|
||||
/** The acknowledgement in force now, if the alert is still open. */
|
||||
ack: AckInfo | null;
|
||||
timeline: TimelineEvent[];
|
||||
notes: { id: number; author: string; body: string; at: string }[];
|
||||
}
|
||||
|
||||
function parse(detail: string | null): Record<string, unknown> {
|
||||
if (!detail) return {};
|
||||
try {
|
||||
const v = JSON.parse(detail);
|
||||
return v && typeof v === "object" ? (v as Record<string, unknown>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
const words = (ms: number) => {
|
||||
const m = Math.round(ms / 60_000);
|
||||
if (m < 90) return `${Math.max(1, m)} min`;
|
||||
const h = ms / 3600_000;
|
||||
return h < 48 ? `${Math.round(h * 10) / 10} h` : `${Math.round(h / 24)} days`;
|
||||
};
|
||||
|
||||
/** Everything the app knows about one incident, in the order it happened. */
|
||||
export async function getIncident(id: number, now: Date = new Date()): Promise<IncidentDetail | null> {
|
||||
const [row] = await db.select().from(alertIncidents).where(eq(alertIncidents.id, id)).limit(1);
|
||||
if (!row) return null;
|
||||
const notes = await db.select().from(incidentNotes).where(eq(incidentNotes.incidentId, id)).orderBy(asc(incidentNotes.createdAt));
|
||||
const incident = summarize(row, notes.length, now);
|
||||
|
||||
const from = Date.parse(row.openedAt);
|
||||
const to = row.resolvedAt ? Date.parse(row.resolvedAt) : now.getTime();
|
||||
const fromIso = new Date(from - 30 * 60_000).toISOString();
|
||||
const toIso = new Date(to + 15 * 60_000).toISOString();
|
||||
const events: TimelineEvent[] = [];
|
||||
|
||||
events.push({ at: row.openedAt, kind: "alert", tone: row.peakSeverity === "critical" ? "bad" : "info", title: `Alert raised (${row.severity})`, detail: row.message, actor: null });
|
||||
if (row.peakSeverity !== row.severity) {
|
||||
events.push({ at: row.resolvedAt ?? row.lastSeenAt, kind: "alert", tone: "neutral", title: `At its worst it was ${row.peakSeverity}`, detail: null, actor: null });
|
||||
}
|
||||
if (row.causedBy) events.push({ at: row.openedAt, kind: "alert", tone: "info", title: `Held back: caused by ${row.causedBy}`, detail: "Folded under the alert for the machine it runs on.", actor: null });
|
||||
if (row.resolvedAt) events.push({ at: row.resolvedAt, kind: "alert", tone: "good", title: "Resolved", detail: `It lasted ${words(to - from)}.`, actor: null });
|
||||
|
||||
// Acknowledgements and hand-overs of this alert, from the audit log.
|
||||
const audits = await db.select().from(auditLog).where(and(gte(auditLog.createdAt, fromIso.slice(0, 19).replace("T", " ")), lte(auditLog.createdAt, toIso.slice(0, 19).replace("T", " ")))).orderBy(asc(auditLog.createdAt));
|
||||
for (const a of audits) {
|
||||
const d = parse(a.detail);
|
||||
if (a.category === "alert") {
|
||||
if (d.alertId !== row.alertKey) continue;
|
||||
const described = describeAudit(a.category, a.action, d);
|
||||
events.push({ at: auditTimeToIso(a.createdAt), kind: "ack", tone: described.tone, title: described.title, detail: described.detail, actor: a.actorLabel });
|
||||
continue;
|
||||
}
|
||||
// Things done to, or noticed about, the thing the alert is about.
|
||||
if (row.targetType && row.targetId !== null && a.targetType === row.targetType && a.targetId === String(row.targetId) && a.category !== "maintenance") {
|
||||
const described = describeAudit(a.category, a.action, d);
|
||||
events.push({ at: auditTimeToIso(a.createdAt), kind: described.kind === "health" ? "health" : "change", tone: described.tone, title: described.title, detail: described.detail, actor: a.actorLabel });
|
||||
}
|
||||
}
|
||||
|
||||
if (row.targetType && row.targetId !== null) {
|
||||
const windows = await db
|
||||
.select()
|
||||
.from(maintenanceWindows)
|
||||
.where(and(eq(maintenanceWindows.targetType, row.targetType as "server"), eq(maintenanceWindows.targetId, row.targetId), lte(maintenanceWindows.startedAt, new Date(to).toISOString()), gte(maintenanceWindows.endsAt, new Date(from).toISOString())));
|
||||
for (const w of windows) {
|
||||
if (w.endsAt <= w.startedAt) continue; // withdrawn before it began
|
||||
events.push({ at: w.startedAt, kind: "maintenance", tone: "info", title: "Maintenance began — alerts silenced", detail: w.reason, actor: w.createdBy });
|
||||
if (w.endsAt <= toIso) events.push({ at: w.endsAt, kind: "maintenance", tone: "neutral", title: "Maintenance ended", detail: null, actor: null });
|
||||
}
|
||||
if (row.targetType === "server") {
|
||||
const runs = await db.select().from(updateRuns).where(and(eq(updateRuns.serverId, row.targetId), lte(updateRuns.startedAt, new Date(to).toISOString()), or(isNull(updateRuns.finishedAt), gte(updateRuns.finishedAt, new Date(from).toISOString()))!));
|
||||
for (const r of runs) {
|
||||
events.push({ at: r.startedAt, kind: "run", tone: "info", title: `Update run started: ${r.templateName}`, detail: `by ${r.startedBy ?? "someone"}`, actor: r.startedBy });
|
||||
if (r.finishedAt) events.push({ at: r.finishedAt, kind: "run", tone: r.status === "done" ? "good" : "bad", title: r.status === "done" ? "Update run finished" : r.status === "cancelled" ? "Update run stopped being followed" : "Update run didn't finish", detail: r.message, actor: null });
|
||||
}
|
||||
}
|
||||
|
||||
// Other alerts about the same thing that began around the same time.
|
||||
const related = await db
|
||||
.select()
|
||||
.from(alertIncidents)
|
||||
.where(and(eq(alertIncidents.targetType, row.targetType), eq(alertIncidents.targetId, row.targetId), ne(alertIncidents.id, row.id), gte(alertIncidents.openedAt, new Date(from - RELATED_WINDOW_MS).toISOString()), lte(alertIncidents.openedAt, new Date(Math.max(to, from) + RELATED_WINDOW_MS).toISOString())));
|
||||
for (const r of related) events.push({ at: r.openedAt, kind: "related", tone: "info", title: `Another alert was raised: ${r.category} (${r.severity})`, detail: r.message, actor: null, incidentId: r.id });
|
||||
}
|
||||
|
||||
for (const n of notes) events.push({ at: n.createdAt, kind: "note", tone: "neutral", title: `${n.author} wrote a note`, detail: n.body, actor: n.author, noteId: n.id });
|
||||
|
||||
events.sort((a, b) => a.at.localeCompare(b.at));
|
||||
return { incident, ack: row.resolvedAt ? null : await loadAck(row.alertKey, now), timeline: events, notes: notes.map((n) => ({ id: n.id, author: n.author, body: n.body, at: n.createdAt })) };
|
||||
}
|
||||
|
||||
export const MAX_NOTE = 4000;
|
||||
|
||||
export async function addNote(incidentId: number, author: string, body: string, now: Date = new Date()): Promise<{ id: number } | null> {
|
||||
const [inc] = await db.select({ id: alertIncidents.id }).from(alertIncidents).where(eq(alertIncidents.id, incidentId)).limit(1);
|
||||
if (!inc) return null;
|
||||
const [row] = await db.insert(incidentNotes).values({ incidentId, author, body: body.trim(), createdAt: now.toISOString() }).returning({ id: incidentNotes.id });
|
||||
return row;
|
||||
}
|
||||
|
||||
export async function removeNote(incidentId: number, noteId: number, actor: { name: string; isAdmin: boolean }): Promise<"removed" | "not_found" | "forbidden"> {
|
||||
const [n] = await db.select().from(incidentNotes).where(and(eq(incidentNotes.id, noteId), eq(incidentNotes.incidentId, incidentId))).limit(1);
|
||||
if (!n) return "not_found";
|
||||
if (n.author !== actor.name && !actor.isAdmin) return "forbidden";
|
||||
await db.delete(incidentNotes).where(eq(incidentNotes.id, noteId));
|
||||
return "removed";
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
/** A look at the alerts every five minutes, so incidents are recorded whether or not anybody has the page open. Once a day it tidies up old ones. */
|
||||
export function initIncidentRecorder(lookAtAlerts: () => Promise<unknown>): void {
|
||||
if (timer) return;
|
||||
let lastPrune = 0;
|
||||
const tick = async () => {
|
||||
try {
|
||||
await lookAtAlerts();
|
||||
if (Date.now() - lastPrune > 24 * 3600_000) {
|
||||
lastPrune = Date.now();
|
||||
await pruneIncidents();
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[incidents] the five-minute look failed:", err);
|
||||
}
|
||||
};
|
||||
setTimeout(() => void tick(), 60_000).unref();
|
||||
timer = setInterval(() => void tick(), 5 * 60_000);
|
||||
timer.unref();
|
||||
}
|
||||
@@ -13,6 +13,9 @@ import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
import { parseHostHealth, poolIsFine } from "./hostHealth.js";
|
||||
import { parseThresholds } from "./serverThresholds.js";
|
||||
import { getCachedBackupCoverage } from "./backupCoverage.js";
|
||||
import { lastAttempt as lastSelfBackup, listSnapshots } from "./selfBackup.js";
|
||||
import { loadAcks, attentionCounts } from "./alertAcks.js";
|
||||
import { listIncidents } from "./incidents.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { computeSecretStatus } from "./secretStatus.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
@@ -115,6 +118,18 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
console.error("[metrics] couldn't read the uptime:", err);
|
||||
}
|
||||
|
||||
// ── Homelab Manager's own backups ──
|
||||
try {
|
||||
const snapshots = listSnapshots();
|
||||
out.add("homelab_selfbackup_snapshots", "gauge", "How many backups of Homelab Manager's own database are kept.", snapshots.length);
|
||||
const newest = snapshots.find((s) => s.kind === "scheduled" || s.kind === "manual");
|
||||
if (newest) out.add("homelab_selfbackup_last_success_timestamp_seconds", "gauge", "When the newest backup of Homelab Manager's own database was made, as a Unix time.", Math.floor(Date.parse(newest.at) / 1000));
|
||||
const last = await lastSelfBackup();
|
||||
if (last) out.add("homelab_selfbackup_last_attempt_ok", "gauge", "1 if the last attempt to back up Homelab Manager's own database worked, 0 if it failed.", last.ok ? 1 : 0);
|
||||
} catch (err) {
|
||||
console.error("[metrics] couldn't read the backups folder:", err);
|
||||
}
|
||||
|
||||
// ── Alerts ──
|
||||
try {
|
||||
const report = await getAlerts(false);
|
||||
@@ -127,6 +142,9 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
}
|
||||
}
|
||||
for (const severity of ["critical", "warning", "info"] as const) out.add("homelab_alerts", "gauge", "Active alerts by severity (not counting silenced ones).", report.counts[severity], { severity });
|
||||
// Somebody has said they are dealing with these (still raised, and still counted above).
|
||||
out.add("homelab_alerts_acknowledged", "gauge", "Alerts somebody has acknowledged and that are not silenced by maintenance.", attentionCounts(report.alerts, await loadAcks(now)).acknowledged);
|
||||
out.add("homelab_incidents_open", "gauge", "Incidents that have begun and not yet ended.", (await listIncidents({ limit: 1, now })).filter((i) => i.open).length);
|
||||
out.add("homelab_alerts_silenced", "gauge", "Alerts held back by a maintenance window.", report.counts.silenced);
|
||||
out.add("homelab_alerts_grouped", "gauge", "Alerts folded under a server that is down.", report.counts.grouped);
|
||||
} catch (err) {
|
||||
|
||||
@@ -0,0 +1,323 @@
|
||||
/**
|
||||
* Backups of Homelab Manager itself. Everything it knows is one SQLite file, so a backup is a consistent copy of that file (`VACUUM INTO`,
|
||||
* which is safe while the app is working) kept in the data folder's `backups/`: taken every day, the newest few kept, each one checked as
|
||||
* it is made, and downloadable, uploadable and restorable from Settings → Backup.
|
||||
*
|
||||
* What a backup is not: it does not hold `CREDENTIALS_ENCRYPTION_KEY`, which decrypts the credentials stored in it, and it isn't off the
|
||||
* machine until someone copies it there. Both are said where backups are shown.
|
||||
*
|
||||
* A restore can't swap a database that is open, so it is staged (checked, then copied beside the database) and takes effect when the app
|
||||
* next starts — see db/applyRestore.ts, which does it before anything opens the file.
|
||||
*/
|
||||
import { createClient } from "@libsql/client";
|
||||
import { copyFileSync, chmodSync, existsSync, mkdirSync, mkdtempSync, openSync, readdirSync, readFileSync, readSync, closeSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { client, dbPath } from "../db/client.js";
|
||||
import { PENDING_FILE, PENDING_NOTE, RESULT_NOTE, type RestoreResult } from "../db/restoreFiles.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export type SnapshotKind = "scheduled" | "manual" | "pre-restore" | "uploaded";
|
||||
|
||||
export interface SnapshotInfo {
|
||||
name: string;
|
||||
kind: SnapshotKind;
|
||||
sizeBytes: number;
|
||||
/** ISO — when the file was written. */
|
||||
at: string;
|
||||
}
|
||||
|
||||
export interface SnapshotCheck {
|
||||
ok: boolean;
|
||||
/** SQLite's own verdict ("ok", or what it found). */
|
||||
integrity: string | null;
|
||||
/** How many schema changes it has had; goes up with each release that changes the database. */
|
||||
migrations: number | null;
|
||||
/** How many this version of the app knows. */
|
||||
appMigrations: number;
|
||||
/** Made by a newer version of the app than this one: it can't be restored here. */
|
||||
newerThanApp: boolean;
|
||||
counts: { users: number; servers: number; integrations: number } | null;
|
||||
/** Why it isn't usable, when it isn't. */
|
||||
problem: string | null;
|
||||
}
|
||||
|
||||
const NAME = /^homelab-manager-\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}(?:-(manual|pre-restore|uploaded))?(?:-\d+)?\.sqlite$/;
|
||||
export const isSnapshotName = (name: string) => NAME.test(name);
|
||||
|
||||
const dataDir = () => dirname(dbPath);
|
||||
export const backupDir = () => join(dataDir(), "backups");
|
||||
|
||||
const stamp = (d: Date) => d.toISOString().slice(0, 19).replace(/[:T]/g, "-");
|
||||
|
||||
function kindOf(name: string): SnapshotKind {
|
||||
const m = NAME.exec(name);
|
||||
return (m?.[1] as SnapshotKind | undefined) ?? "scheduled";
|
||||
}
|
||||
|
||||
/** How many schema changes this version of the app has (the entries in its migration journal). */
|
||||
export function appMigrationCount(): number {
|
||||
try {
|
||||
const journal = JSON.parse(readFileSync(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "drizzle", "meta", "_journal.json"), "utf8")) as { entries?: unknown[] };
|
||||
return journal.entries?.length ?? 0;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
function hasSqliteHeader(file: string): boolean {
|
||||
try {
|
||||
const fd = openSync(file, "r");
|
||||
try {
|
||||
const buf = Buffer.alloc(16);
|
||||
readSync(fd, buf, 0, 16, 0);
|
||||
return buf.toString("latin1") === "SQLite format 3\0";
|
||||
} finally {
|
||||
closeSync(fd);
|
||||
}
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** A scratch folder for looking at a copy of a backup, and the way to be rid of it (best effort: some systems hold an opened database until the process ends). */
|
||||
function scratch(): { file: string; done: () => void } {
|
||||
const folder = mkdtempSync(join(tmpdir(), "hlm-backup-"));
|
||||
return {
|
||||
file: join(folder, "db.sqlite"),
|
||||
done: () => {
|
||||
try {
|
||||
rmSync(folder, { recursive: true, force: true });
|
||||
} catch {
|
||||
// left for the system's own tidying of its temporary folder
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Looks at a backup: does SQLite think it is sound, is it one of ours, and is it from this version or an older one? It looks at a copy in a
|
||||
* temporary folder, never at the file itself, so a backup in the backups folder is never held open — it can always be deleted, anywhere.
|
||||
*/
|
||||
export async function checkSnapshotFile(file: string): Promise<SnapshotCheck> {
|
||||
if (!existsSync(file)) return checkOpened(file);
|
||||
if (!hasSqliteHeader(file)) return checkOpened(file);
|
||||
const tmp = scratch();
|
||||
try {
|
||||
copyFileSync(file, tmp.file);
|
||||
return await checkOpened(tmp.file);
|
||||
} finally {
|
||||
tmp.done();
|
||||
}
|
||||
}
|
||||
|
||||
/** The check itself, on a file nobody else cares about the handle of. */
|
||||
async function checkOpened(file: string): Promise<SnapshotCheck> {
|
||||
const appMigrations = appMigrationCount();
|
||||
const fail = (problem: string, extra: Partial<SnapshotCheck> = {}): SnapshotCheck => ({ ok: false, integrity: null, migrations: null, appMigrations, newerThanApp: false, counts: null, problem, ...extra });
|
||||
if (!existsSync(file)) return fail("The file isn't there.");
|
||||
if (!hasSqliteHeader(file)) return fail("That isn't a database file.");
|
||||
const c = createClient({ url: `file:${file}` });
|
||||
try {
|
||||
const one = async (sql: string) => Object.values((await c.execute(sql)).rows[0] ?? {})[0];
|
||||
const integrity = String(await one("PRAGMA integrity_check"));
|
||||
const tables = new Set((await c.execute("select name from sqlite_master where type = 'table'")).rows.map((r) => String(r.name)));
|
||||
for (const needed of ["users", "servers", "integrations", "settings", "__drizzle_migrations"]) {
|
||||
if (!tables.has(needed)) return fail(`It isn't a Homelab Manager database (no "${needed}" table).`, { integrity });
|
||||
}
|
||||
const migrations = Number(await one("select count(*) from __drizzle_migrations"));
|
||||
const counts = { users: Number(await one("select count(*) from users")), servers: Number(await one("select count(*) from servers")), integrations: Number(await one("select count(*) from integrations")) };
|
||||
const newerThanApp = appMigrations > 0 && migrations > appMigrations;
|
||||
if (integrity !== "ok") return fail(`SQLite found problems in it: ${integrity}`, { integrity, migrations, counts, newerThanApp });
|
||||
if (newerThanApp) return fail("It was made by a newer version of Homelab Manager than this one, and an older version can't use it. Update first.", { integrity, migrations, counts, newerThanApp });
|
||||
return { ok: true, integrity, migrations, appMigrations, newerThanApp: false, counts, problem: null };
|
||||
} catch (err) {
|
||||
return fail(`It couldn't be read as a database: ${err instanceof Error ? err.message : String(err)}`);
|
||||
} finally {
|
||||
c.close();
|
||||
}
|
||||
}
|
||||
|
||||
/** A name for a new file that isn't taken. */
|
||||
function freshName(kind: SnapshotKind, now: Date): string {
|
||||
const base = `homelab-manager-${stamp(now)}${kind === "scheduled" ? "" : `-${kind}`}`;
|
||||
let name = `${base}.sqlite`;
|
||||
for (let n = 2; existsSync(join(backupDir(), name)); n++) name = `${base}-${n}.sqlite`;
|
||||
return name;
|
||||
}
|
||||
|
||||
export interface LastAttempt {
|
||||
at: string;
|
||||
ok: boolean;
|
||||
kind: SnapshotKind;
|
||||
error: string | null;
|
||||
name: string | null;
|
||||
}
|
||||
const ATTEMPT_FLAG = "selfBackupLastAttempt";
|
||||
|
||||
export async function lastAttempt(): Promise<LastAttempt | null> {
|
||||
try {
|
||||
const v = await getInternalFlag(ATTEMPT_FLAG);
|
||||
return v ? (JSON.parse(v) as LastAttempt) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Writes a consistent copy of the database into the backups folder and checks it. Throws, saying why, if it can't — and records the attempt either way. */
|
||||
export async function createSnapshot(kind: SnapshotKind = "manual", now: Date = new Date()): Promise<{ info: SnapshotInfo; check: SnapshotCheck }> {
|
||||
const tmp = scratch();
|
||||
try {
|
||||
mkdirSync(backupDir(), { recursive: true });
|
||||
const name = freshName(kind, now);
|
||||
const file = join(backupDir(), name);
|
||||
// Made and checked in a temporary folder first, and only then put in the backups folder: what is there is always whole and sound.
|
||||
await client.execute(`VACUUM INTO '${tmp.file.replace(/'/g, "''")}'`);
|
||||
const check = await checkOpened(tmp.file);
|
||||
if (!check.ok) throw new Error(`The copy didn't check out, so it was thrown away: ${check.problem}`);
|
||||
copyFileSync(tmp.file, file);
|
||||
try {
|
||||
chmodSync(file, 0o600);
|
||||
} catch {
|
||||
// not every file system has modes
|
||||
}
|
||||
const info: SnapshotInfo = { name, kind, sizeBytes: statSync(file).size, at: now.toISOString() };
|
||||
await setInternalFlag(ATTEMPT_FLAG, JSON.stringify({ at: now.toISOString(), ok: true, kind, error: null, name } satisfies LastAttempt));
|
||||
return { info, check };
|
||||
} catch (err) {
|
||||
const error = err instanceof Error ? err.message : String(err);
|
||||
await setInternalFlag(ATTEMPT_FLAG, JSON.stringify({ at: now.toISOString(), ok: false, kind, error, name: null } satisfies LastAttempt));
|
||||
throw err;
|
||||
} finally {
|
||||
tmp.done();
|
||||
}
|
||||
}
|
||||
|
||||
export function listSnapshots(): SnapshotInfo[] {
|
||||
if (!existsSync(backupDir())) return [];
|
||||
return readdirSync(backupDir())
|
||||
.filter(isSnapshotName)
|
||||
.map((name) => {
|
||||
const st = statSync(join(backupDir(), name));
|
||||
return { name, kind: kindOf(name), sizeBytes: st.size, at: st.mtime.toISOString() };
|
||||
})
|
||||
.sort((a, b) => b.at.localeCompare(a.at) || b.name.localeCompare(a.name));
|
||||
}
|
||||
|
||||
/** What is kept: the newest `keep` scheduled backups, and a few of each other kind (a restore's "before" copy is worth keeping longer than a day). */
|
||||
export const KEEP_OTHER: Record<Exclude<SnapshotKind, "scheduled">, number> = { manual: 10, "pre-restore": 3, uploaded: 3 };
|
||||
|
||||
export function pruneSnapshots(keep: number): string[] {
|
||||
const removed: string[] = [];
|
||||
const all = listSnapshots();
|
||||
const limitFor = (kind: SnapshotKind) => (kind === "scheduled" ? Math.max(1, keep) : KEEP_OTHER[kind]);
|
||||
for (const kind of ["scheduled", "manual", "pre-restore", "uploaded"] as const) {
|
||||
for (const s of all.filter((x) => x.kind === kind).slice(limitFor(kind))) {
|
||||
rmSync(join(backupDir(), s.name), { force: true });
|
||||
removed.push(s.name);
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
export function snapshotPath(name: string): string | null {
|
||||
if (!isSnapshotName(name)) return null;
|
||||
const file = join(backupDir(), name);
|
||||
return existsSync(file) ? file : null;
|
||||
}
|
||||
|
||||
export function deleteSnapshot(name: string): boolean {
|
||||
const file = snapshotPath(name);
|
||||
if (!file) return false;
|
||||
rmSync(file, { force: true });
|
||||
return true;
|
||||
}
|
||||
|
||||
/** A file somebody uploaded, kept as a backup if it checks out (and thrown away if not). */
|
||||
export async function importUploaded(bytes: Buffer, now: Date = new Date()): Promise<{ info: SnapshotInfo; check: SnapshotCheck }> {
|
||||
const tmp = scratch();
|
||||
try {
|
||||
writeFileSync(tmp.file, bytes);
|
||||
const check = await checkOpened(tmp.file);
|
||||
if (!check.ok) throw new Error(check.problem ?? "That file isn't usable.");
|
||||
mkdirSync(backupDir(), { recursive: true });
|
||||
const name = freshName("uploaded", now);
|
||||
const file = join(backupDir(), name);
|
||||
copyFileSync(tmp.file, file);
|
||||
try {
|
||||
chmodSync(file, 0o600);
|
||||
} catch {
|
||||
// not every file system has modes
|
||||
}
|
||||
return { info: { name, kind: "uploaded", sizeBytes: statSync(file).size, at: now.toISOString() }, check };
|
||||
} finally {
|
||||
tmp.done();
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Restoring ──────────────────────────────────────────────────────────────
|
||||
|
||||
export interface StagedRestore {
|
||||
from: string;
|
||||
stagedAt: string;
|
||||
by: string | null;
|
||||
}
|
||||
|
||||
export function stagedRestore(): StagedRestore | null {
|
||||
try {
|
||||
if (!existsSync(join(dataDir(), PENDING_FILE))) return null;
|
||||
const note = JSON.parse(readFileSync(join(dataDir(), PENDING_NOTE), "utf8")) as { from?: string; stagedAt?: string; by?: string | null };
|
||||
return { from: note.from ?? "a backup", stagedAt: note.stagedAt ?? "", by: note.by ?? null };
|
||||
} catch {
|
||||
return existsSync(join(dataDir(), PENDING_FILE)) ? { from: "a backup", stagedAt: "", by: null } : null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Checks a backup and stages it to replace the database the next time the app starts. Throws, saying why, if it can't be used. */
|
||||
export async function stageRestore(name: string, by: string | null, now: Date = new Date()): Promise<SnapshotCheck> {
|
||||
const file = snapshotPath(name);
|
||||
if (!file) throw new Error("There is no backup with that name.");
|
||||
const check = await checkSnapshotFile(file);
|
||||
if (!check.ok) throw new Error(check.problem ?? "That backup isn't usable.");
|
||||
const pending = join(dataDir(), PENDING_FILE);
|
||||
const tmp = `${pending}.copying`;
|
||||
copyFileSync(file, tmp);
|
||||
renameSync(tmp, pending); // whole, or not there at all: the start-up step never sees half a file
|
||||
try {
|
||||
chmodSync(pending, 0o600);
|
||||
} catch {
|
||||
// not every file system has modes
|
||||
}
|
||||
writeFileSync(join(dataDir(), PENDING_NOTE), JSON.stringify({ from: name, stagedAt: now.toISOString(), by }));
|
||||
return check;
|
||||
}
|
||||
|
||||
export function cancelStagedRestore(): boolean {
|
||||
const had = existsSync(join(dataDir(), PENDING_FILE));
|
||||
rmSync(join(dataDir(), PENDING_FILE), { force: true });
|
||||
rmSync(join(dataDir(), PENDING_NOTE), { force: true });
|
||||
return had;
|
||||
}
|
||||
|
||||
/** What the last restore did, once, for the page to say. */
|
||||
export function lastRestore(): RestoreResult | null {
|
||||
try {
|
||||
return JSON.parse(readFileSync(join(dataDir(), RESULT_NOTE), "utf8")) as RestoreResult;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function dismissLastRestore(): void {
|
||||
rmSync(join(dataDir(), RESULT_NOTE), { force: true });
|
||||
}
|
||||
|
||||
// ─── How current the backups are ────────────────────────────────────────────
|
||||
|
||||
/** Backups older than this are an alert: a day's slack on top of the daily schedule. */
|
||||
export const STALE_AFTER_HOURS = 50;
|
||||
|
||||
export function newestSnapshot(): SnapshotInfo | null {
|
||||
return listSnapshots().find((s) => s.kind === "scheduled" || s.kind === "manual") ?? null;
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
import schedule from "node-schedule";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { createSnapshot, lastAttempt, newestSnapshot, pruneSnapshots } from "./selfBackup.js";
|
||||
import { notify } from "./notify.js";
|
||||
|
||||
let currentJob: schedule.Job | null = null;
|
||||
|
||||
/** The cron rule for "every day at HH:MM". */
|
||||
export function cronForBackup(time: string): string {
|
||||
const [h, m] = time.split(":").map(Number);
|
||||
return `${Number.isFinite(m) ? m : 30} ${Number.isFinite(h) ? h : 3} * * *`;
|
||||
}
|
||||
|
||||
/** One scheduled backup: take it, then drop the oldest beyond how many are kept. Says so through the channels if it fails. */
|
||||
export async function runScheduledBackup(now: Date = new Date()): Promise<{ name: string } | null> {
|
||||
const { selfBackup, notifications } = await getSettings();
|
||||
try {
|
||||
const { info } = await createSnapshot("scheduled", now);
|
||||
pruneSnapshots(selfBackup.keep);
|
||||
return { name: info.name };
|
||||
} catch (err) {
|
||||
console.error("[selfBackup] the scheduled backup failed:", err instanceof Error ? err.message : err);
|
||||
if (notifications.healthAlerts !== false) {
|
||||
await notify("Homelab Manager — its own backup failed", `The scheduled backup of Homelab Manager's database didn't work: ${err instanceof Error ? err.message : String(err)}`).catch(() => {});
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** The next time it will run, or null when it is off. */
|
||||
export function nextBackupRun(): Date | null {
|
||||
const next = currentJob?.nextInvocation();
|
||||
return next ? new Date(next.getTime()) : null;
|
||||
}
|
||||
|
||||
/** (Re)schedules the daily backup per the current settings. Call again after they change. */
|
||||
export async function scheduleSelfBackup(): Promise<void> {
|
||||
if (currentJob) {
|
||||
currentJob.cancel();
|
||||
currentJob = null;
|
||||
}
|
||||
const { selfBackup, notifications } = await getSettings();
|
||||
if (!selfBackup.enabled) {
|
||||
console.log("[selfBackup] daily backups are off, nothing scheduled");
|
||||
return;
|
||||
}
|
||||
currentJob = schedule.scheduleJob({ rule: cronForBackup(selfBackup.time), tz: notifications.timezone }, () => {
|
||||
void runScheduledBackup();
|
||||
});
|
||||
console.log(`Daily backup scheduled at ${selfBackup.time} (${notifications.timezone})`);
|
||||
}
|
||||
|
||||
/** Schedules it, and takes one soon if the app has been off through its time (or has never taken one). */
|
||||
export async function initSelfBackupScheduler(): Promise<void> {
|
||||
await scheduleSelfBackup();
|
||||
const { selfBackup } = await getSettings();
|
||||
if (!selfBackup.enabled) return;
|
||||
const newest = newestSnapshot();
|
||||
const overdue = !newest || Date.now() - Date.parse(newest.at) > 26 * 3600_000;
|
||||
// Only retry a failing one at the next scheduled time, not at every restart.
|
||||
const last = await lastAttempt();
|
||||
if (overdue && !(last && !last.ok && Date.now() - Date.parse(last.at) < 6 * 3600_000)) {
|
||||
setTimeout(() => void runScheduledBackup(), 30_000).unref();
|
||||
}
|
||||
}
|
||||
@@ -72,6 +72,17 @@ export function describeAudit(category: string, action: string, detail: Record<s
|
||||
return { title: "Admin link removed", tone: "neutral", kind: "change", detail: label };
|
||||
}
|
||||
}
|
||||
if (category === "alert") {
|
||||
const note = typeof detail.note === "string" && detail.note ? detail.note : null;
|
||||
const assignee = typeof detail.assignee === "string" ? detail.assignee : null;
|
||||
const lasts = typeof detail.hours === "number" ? `for ${detail.hours >= 48 ? `${Math.round(detail.hours / 24)} days` : `${detail.hours} h`}` : "until the alert is gone";
|
||||
switch (action) {
|
||||
case "ack":
|
||||
return { title: assignee ? `Acknowledged, handed to ${assignee}` : "Acknowledged", tone: "info", kind: "change", detail: [note, lasts].filter(Boolean).join(" · ") };
|
||||
case "unack":
|
||||
return { title: "Acknowledgement taken back", tone: "neutral", kind: "change", detail: null };
|
||||
}
|
||||
}
|
||||
if (category === "update_run") {
|
||||
const template = typeof detail.template === "string" ? detail.template : null;
|
||||
switch (action) {
|
||||
|
||||
@@ -129,6 +129,15 @@ export interface AgentSettings {
|
||||
autoUpdate: boolean;
|
||||
}
|
||||
|
||||
/** The daily backup of Homelab Manager's own database (see services/selfBackup.ts). On unless an administrator turns it off. */
|
||||
export interface SelfBackupSettings {
|
||||
enabled: boolean;
|
||||
/** "HH:MM", in the notification time zone. */
|
||||
time: string;
|
||||
/** How many of the daily backups are kept; the oldest beyond that are deleted. */
|
||||
keep: number;
|
||||
}
|
||||
|
||||
/** What the inventory needs to work out a running cost. Without a price per kWh it shows draw but no cost. */
|
||||
export interface InventorySettings {
|
||||
/** A short name shown after amounts ("SEK", "kr", "$"); may be blank. */
|
||||
@@ -190,6 +199,7 @@ export interface AppSettings {
|
||||
digest: DigestSettings;
|
||||
signIn: SignInSettings;
|
||||
inventory: InventorySettings;
|
||||
selfBackup: SelfBackupSettings;
|
||||
agents: AgentSettings;
|
||||
consistency: ConsistencySettings;
|
||||
nameGenerator: NameGeneratorSettings;
|
||||
@@ -227,6 +237,7 @@ const DEFAULTS: AppSettings = {
|
||||
digest: { enabled: false, day: 1, time: "08:00" },
|
||||
inventory: { currency: "", pricePerKwh: null },
|
||||
agents: { autoUpdate: false },
|
||||
selfBackup: { enabled: true, time: "03:30", keep: 14 },
|
||||
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
|
||||
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48 },
|
||||
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
|
||||
|
||||
@@ -20,6 +20,8 @@ import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { getUptime } from "./uptimeHistory.js";
|
||||
import { attentionCounts, loadAcks } from "./alertAcks.js";
|
||||
import { listIncidents } from "./incidents.js";
|
||||
|
||||
export const DIGEST_DAYS = 7;
|
||||
const DAY_MS = 24 * 3600_000;
|
||||
@@ -90,11 +92,30 @@ export async function buildDigest(now: Date = new Date()): Promise<Digest> {
|
||||
if (report.counts.silenced > 0) held.push(`${report.counts.silenced} silenced by maintenance`);
|
||||
if (report.counts.grouped > 0) held.push(`${report.counts.grouped} folded under a server that is down`);
|
||||
if (held.length > 0) lines.push(`(${held.join(", ")})`);
|
||||
const acknowledged = attentionCounts(report.alerts, await loadAcks(now)).acknowledged;
|
||||
if (acknowledged > 0) lines.push(`(${acknowledged} of them acknowledged — somebody is on it)`);
|
||||
sections.push({ heading: "Right now", lines, attention: active.length > 0 || report.couldntCheck.length > 0 });
|
||||
} catch (err) {
|
||||
console.error("[digest] couldn't read the alerts:", err);
|
||||
}
|
||||
|
||||
// ── Incidents ──
|
||||
try {
|
||||
const since = now.getTime() - DIGEST_DAYS * DAY_MS;
|
||||
const all = await listIncidents({ now });
|
||||
const begun = all.filter((i) => Date.parse(i.openedAt) >= since);
|
||||
const ended = all.filter((i) => i.resolvedAt !== null && Date.parse(i.resolvedAt) >= since);
|
||||
const open = all.filter((i) => i.open);
|
||||
if (begun.length > 0 || ended.length > 0) {
|
||||
const lines = [`${plural(begun.length, "incident")} began, ${ended.length} ended, ${open.length} still open.`];
|
||||
const longest = [...ended].sort((a, b) => b.durationMs - a.durationMs).slice(0, 3);
|
||||
lines.push(...longest.map((i) => `- ${i.message} — lasted ${spanWords(i.durationMs / 60_000)}`));
|
||||
sections.push({ heading: "Incidents", lines, attention: open.length > 0 });
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[digest] couldn't read the incidents:", err);
|
||||
}
|
||||
|
||||
// ── Uptime ──
|
||||
try {
|
||||
const uptime = await getUptime(now, DIGEST_DAYS);
|
||||
|
||||
Reference in new issue
Block a user