Add a weekly digest and report which version of the agent each server runs
- Weekly digest through the notification channels: alerts now, a week of uptime, what happened, disks heading for full, expiring secrets and domains, updates waiting, planned maintenance. Off until switched on in Settings -> Notifications; Preview and Send it now there. Setting key 'digest'. - Agents say their version in every report (Linux and Windows scripts are version 2); the server compares it with the scripts it serves and marks agents that are behind on the Servers, server and Updates pages. servers.agent_version, migration 0026. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3da5c639b1
commit
aa94d84f6c
28 files changed
+3179
-19
No files matched your search
@@ -0,0 +1 @@
|
||||
ALTER TABLE `servers` ADD `agent_version` integer;
|
||||
File diff suppressed because it is too large.
Load diff
@@ -183,6 +183,13 @@
|
||||
"when": 1792191757544,
|
||||
"tag": "0025_clean_the_hand",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 26,
|
||||
"version": "6",
|
||||
"when": 1792258587758,
|
||||
"tag": "0026_reflective_northstar",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -291,6 +291,10 @@ export const servers = sqliteTable("servers", {
|
||||
pendingUpdates: text("pending_updates"),
|
||||
// When updates were first seen waiting (and none were before) — how long the oldest has been waiting. Null when none are.
|
||||
updatesPendingSince: text("updates_pending_since"),
|
||||
|
||||
// Which version of the agent script is running on it, as the agent says in each report (see agent/*/report-tasks.*). Null until it says:
|
||||
// an agent that predates the number being reported is version 1.
|
||||
agentVersion: integer("agent_version"),
|
||||
});
|
||||
|
||||
export const scheduledTasks = sqliteTable("scheduled_tasks", {
|
||||
|
||||
@@ -10,6 +10,7 @@ import { initDockerUpdateScheduler } from "./services/dockerUpdateScheduler.js";
|
||||
import { initProxmoxBackupScheduler } from "./services/proxmoxBackupScheduler.js";
|
||||
import { initPbsVerificationScheduler } from "./services/pbsVerificationScheduler.js";
|
||||
import { initQuietHoursScheduler } from "./services/quietHoursScheduler.js";
|
||||
import { initWeeklyDigestScheduler } from "./services/weeklyDigestScheduler.js";
|
||||
import { initHealthScheduler } from "./services/healthScheduler.js";
|
||||
import { initAlertTicketScheduler } from "./services/alertTicketScheduler.js";
|
||||
import { initKumaMaintenanceScheduler } from "./services/kumaMaintenanceScheduler.js";
|
||||
@@ -40,6 +41,7 @@ await initDockerUpdateScheduler();
|
||||
await initProxmoxBackupScheduler();
|
||||
await initPbsVerificationScheduler();
|
||||
await initQuietHoursScheduler();
|
||||
await initWeeklyDigestScheduler();
|
||||
await initHealthScheduler();
|
||||
await initAlertTicketScheduler();
|
||||
await initKumaMaintenanceScheduler();
|
||||
|
||||
@@ -65,6 +65,11 @@ const reportSchema = z.object({
|
||||
hostname: text(255).optional(),
|
||||
os_type: text(50).optional(),
|
||||
reported_at: text(50).optional(),
|
||||
// Lenient like the rest: a number that makes no sense is dropped, never allowed to cost the agent its report.
|
||||
agent_version: z
|
||||
.unknown()
|
||||
.optional()
|
||||
.transform((v) => (typeof v === "number" && Number.isInteger(v) && v >= 1 && v <= 100_000 ? v : undefined)),
|
||||
system: systemSchema.nullable().optional(),
|
||||
tasks: z
|
||||
.array(
|
||||
@@ -111,6 +116,7 @@ agentReportRouter.post("/", asyncHandler(authenticateAgent), express.json({ limi
|
||||
await syncServerTasks(server.id, {
|
||||
hostname: parsed.data.hostname,
|
||||
osType: parsed.data.os_type,
|
||||
agentVersion: parsed.data.agent_version,
|
||||
system: parsed.data.system,
|
||||
tasks: parsed.data.tasks.map((t) => ({
|
||||
scheduleType: t.schedule_type,
|
||||
|
||||
@@ -8,6 +8,7 @@ import { generateApiToken } from "../services/tokens.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { loadParentRows, resolveParents, wouldLoop } from "../services/serverParents.js";
|
||||
import { parsePendingUpdates } from "../services/pendingUpdates.js";
|
||||
import { latestAgentVersions } from "../services/agentVersions.js";
|
||||
import { getServerMetrics, RETENTION_DAYS } from "../services/serverMetrics.js";
|
||||
import { getServerActivity } from "../services/serverActivity.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
@@ -42,6 +43,8 @@ serversRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const rows = await db.query.servers.findMany({ orderBy: (s, { asc }) => [asc(s.name)] });
|
||||
res.json({
|
||||
servers: rows.map(publicServer),
|
||||
// What a new install would be, so the page can say which agents are behind.
|
||||
agentLatest: latestAgentVersions(),
|
||||
});
|
||||
}));
|
||||
|
||||
@@ -199,13 +202,15 @@ serversRouter.get("/updates", asyncHandler(async (_req, res) => {
|
||||
lastSeenAt: servers.lastSeenAt,
|
||||
pendingUpdates: servers.pendingUpdates,
|
||||
updatesPendingSince: servers.updatesPendingSince,
|
||||
agentVersion: servers.agentVersion,
|
||||
})
|
||||
.from(servers)
|
||||
.orderBy(servers.name);
|
||||
res.json({
|
||||
servers: rows
|
||||
.filter((r) => r.lastSeenAt !== null)
|
||||
.map((r) => ({ id: r.id, name: r.name, osType: r.osType, lastSeenAt: r.lastSeenAt, updates: parsePendingUpdates(r.pendingUpdates), pendingSince: r.updatesPendingSince })),
|
||||
.map((r) => ({ id: r.id, name: r.name, osType: r.osType, lastSeenAt: r.lastSeenAt, updates: parsePendingUpdates(r.pendingUpdates), pendingSince: r.updatesPendingSince, agentVersion: r.agentVersion })),
|
||||
agentLatest: latestAgentVersions(),
|
||||
});
|
||||
}));
|
||||
|
||||
@@ -356,7 +361,7 @@ serversRouter.get("/:id/detail", asyncHandler(async (req, res) => {
|
||||
children: [...links2.entries()].filter(([, l]) => l.parentId === id).map(([childId, l]) => ({ id: childId, name: nameOf.get(childId) ?? "", auto: l.auto })).sort((a, b) => a.name.localeCompare(b.name)),
|
||||
};
|
||||
|
||||
res.json({ server: { ...serverOut, tags: parseTags(rawTags), pendingUpdates: parsePendingUpdates(rawUpdates) }, ipAddresses, hardware, dnsMatches, links, relations });
|
||||
res.json({ server: { ...serverOut, tags: parseTags(rawTags), pendingUpdates: parsePendingUpdates(rawUpdates) }, ipAddresses, hardware, dnsMatches, links, relations, agentLatest: latestAgentVersions() });
|
||||
}));
|
||||
|
||||
const tagsSchema = z.object({ tags: z.array(z.string().max(100)).max(50) });
|
||||
|
||||
@@ -12,6 +12,9 @@ import { schedulePbsVerificationCheck } from "../services/pbsVerificationSchedul
|
||||
import { scheduleLogRetentionPurge } from "../services/logRetentionScheduler.js";
|
||||
import { purgeOldLogs } from "../services/logRetention.js";
|
||||
import { scheduleQuietHoursFlush } from "../services/quietHoursScheduler.js";
|
||||
import { scheduleWeeklyDigest } from "../services/weeklyDigestScheduler.js";
|
||||
import { buildDigest, sendDigest } from "../services/weeklyDigest.js";
|
||||
import { enabledChannelCount } from "../services/notify.js";
|
||||
import { testGotify, testNtfy, testSmtp, testWebhook, flushQuietHoursQueue } from "../services/notify.js";
|
||||
import { listQueuedNotifications } from "../services/notificationQueue.js";
|
||||
import { buildExportPayload, encryptExport, decryptExport, applyImportPayload, type EncryptedExportFile } from "../services/configBackup.js";
|
||||
@@ -99,6 +102,10 @@ const updateSchema = z.object({
|
||||
.object({ enabled: z.boolean(), start: z.string().regex(/^\d{2}:\d{2}$/), end: z.string().regex(/^\d{2}:\d{2}$/) })
|
||||
.partial()
|
||||
.optional(),
|
||||
digest: z
|
||||
.object({ enabled: z.boolean(), day: z.number().int().min(0).max(6), time: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/) })
|
||||
.partial()
|
||||
.optional(),
|
||||
});
|
||||
|
||||
settingsRouter.put("/", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
@@ -123,6 +130,9 @@ settingsRouter.put("/", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
if (parsed.data.quietHours || parsed.data.notifications) {
|
||||
await scheduleQuietHoursFlush();
|
||||
}
|
||||
if (parsed.data.digest || parsed.data.notifications) {
|
||||
await scheduleWeeklyDigest();
|
||||
}
|
||||
|
||||
await recordAudit({
|
||||
actor: req.currentUser!,
|
||||
@@ -203,6 +213,23 @@ settingsRouter.post("/purge-logs", requireRole("admin"), asyncHandler(async (req
|
||||
res.json(result);
|
||||
}));
|
||||
|
||||
// What the weekly digest would say right now, so it can be read before it is switched on.
|
||||
settingsRouter.get("/digest/preview", requireRole("admin"), asyncHandler(async (_req, res) => {
|
||||
const digest = await buildDigest();
|
||||
res.json({ title: digest.title, message: digest.message, quiet: digest.quiet, channels: await enabledChannelCount() });
|
||||
}));
|
||||
|
||||
// Sends it now, through whatever channels are switched on, at whatever time it is.
|
||||
settingsRouter.post("/digest/send", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const channels = await enabledChannelCount();
|
||||
if (channels === 0) {
|
||||
return res.status(400).json({ error: "no_channel", message: "No notification channel is switched on, so there is nowhere to send it. Set one up above first." });
|
||||
}
|
||||
const digest = await sendDigest({ immediately: true });
|
||||
await recordAudit({ actor: req.currentUser!, category: "settings", action: "send_digest", detail: { channels, quiet: digest.quiet } });
|
||||
res.json({ sent: true, channels });
|
||||
}));
|
||||
|
||||
settingsRouter.get("/quiet-hours-queue", requireRole("admin"), asyncHandler(async (_req, res) => {
|
||||
const items = await listQueuedNotifications();
|
||||
res.json({ count: items.length, items });
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
/**
|
||||
* Which version of the agent each server runs, and which version this Homelab Manager would give a new install. An agent is a copy of
|
||||
* a script that was served at install time, so after the app is updated the agents out on the servers can be behind. Each script carries
|
||||
* its version as a number (`AGENT_VERSION=2` / `$script:AgentVersion = 2`), says it in every report, and the "latest" is read from the very
|
||||
* scripts this app serves — so the two can never disagree about what "current" means.
|
||||
*
|
||||
* 1 — before the number was reported (so an agent that says nothing is version 1)
|
||||
* 2 — reports pending updates, and its own version
|
||||
*/
|
||||
import { readFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
export type AgentOs = "linux" | "windows";
|
||||
export interface LatestAgentVersions {
|
||||
linux: number | null;
|
||||
windows: number | null;
|
||||
}
|
||||
|
||||
// server/src/services → the repository's agent folder (the same place the app serves /agent from; in the image it sits beside server/dist).
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
export const AGENT_DIR = join(here, "..", "..", "..", "agent");
|
||||
|
||||
const FILES: Record<AgentOs, string> = { linux: join("linux", "report-tasks.sh"), windows: join("windows", "report-tasks.ps1") };
|
||||
const PATTERNS: Record<AgentOs, RegExp> = {
|
||||
linux: /^AGENT_VERSION=(\d+)\s*$/m,
|
||||
windows: /^\$script:AgentVersion\s*=\s*(\d+)\s*$/m,
|
||||
};
|
||||
|
||||
/** The version a script says it is, or null if it doesn't say. */
|
||||
export function parseAgentVersion(text: string, os: AgentOs): number | null {
|
||||
const m = PATTERNS[os].exec(text.replace(/\r\n/g, "\n"));
|
||||
return m ? Number(m[1]) : null;
|
||||
}
|
||||
|
||||
let cached: LatestAgentVersions | null = null;
|
||||
|
||||
/** What the scripts served from here say. Read once (they don't change while the app runs); null for one that can't be read. */
|
||||
export function latestAgentVersions(dir: string = AGENT_DIR): LatestAgentVersions {
|
||||
if (cached && dir === AGENT_DIR) return cached;
|
||||
const read = (os: AgentOs): number | null => {
|
||||
try {
|
||||
return parseAgentVersion(readFileSync(join(dir, FILES[os]), "utf8"), os);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
const result = { linux: read("linux"), windows: read("windows") };
|
||||
if (dir === AGENT_DIR) cached = result;
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a server's agent is behind. Only a server that reports through an agent can be: one that never has (a Proxmox guest tracked
|
||||
* through its host, say) has none to update. An agent that reports no version is version 1.
|
||||
*/
|
||||
export function isAgentOutdated(row: { osType: string; lastSeenAt: string | null; agentVersion: number | null }, latest: LatestAgentVersions): boolean {
|
||||
if (!row.lastSeenAt) return false;
|
||||
const newest = row.osType === "windows" ? latest.windows : row.osType === "linux" ? latest.linux : null;
|
||||
if (newest === null) return false;
|
||||
return (row.agentVersion ?? 1) < newest;
|
||||
}
|
||||
@@ -104,6 +104,22 @@ async function isQuietHoursActive(): Promise<boolean> {
|
||||
return startMin < endMin ? nowMin >= startMin && nowMin < endMin : nowMin >= startMin || nowMin < endMin;
|
||||
}
|
||||
|
||||
/** Sends at once through every channel that is switched on, whatever the time — for something a person just asked for. */
|
||||
export async function notifyNow(title: string, message: string): Promise<void> {
|
||||
await sendToChannels(title, message);
|
||||
}
|
||||
|
||||
/** How many channels are switched on and filled in enough to send, so a "send it now" can say when there is nowhere to send to. */
|
||||
export async function enabledChannelCount(): Promise<number> {
|
||||
const { gotify, ntfy, smtp, webhook } = await getSettings();
|
||||
return [
|
||||
gotify.enabled && gotify.url && gotify.token,
|
||||
ntfy.enabled && ntfy.url && ntfy.topic,
|
||||
smtp.enabled && smtp.host && smtp.to && smtp.from,
|
||||
webhook.enabled && webhook.url,
|
||||
].filter(Boolean).length;
|
||||
}
|
||||
|
||||
export async function notify(title: string, message: string): Promise<void> {
|
||||
if (await isQuietHoursActive()) {
|
||||
await enqueueNotification(title, message);
|
||||
|
||||
@@ -99,6 +99,15 @@ export interface HealthCheckSettings {
|
||||
diskForecastDays: number;
|
||||
}
|
||||
|
||||
/** The weekly digest: one message a week through the notification channels. Off until an admin switches it on under Settings → Notifications. */
|
||||
export interface DigestSettings {
|
||||
enabled: boolean;
|
||||
/** 0 = Sunday … 6 = Saturday, in notifications.timezone. */
|
||||
day: number;
|
||||
/** "HH:MM", in notifications.timezone. */
|
||||
time: string;
|
||||
}
|
||||
|
||||
export interface ConsistencySettings {
|
||||
/** CIDR ranges and single addresses the consistency report ignores entirely — e.g. Docker networks that repeat on many hosts. */
|
||||
excludedRanges: string[];
|
||||
@@ -150,6 +159,7 @@ export interface AppSettings {
|
||||
logRetention: LogRetentionSettings;
|
||||
quietHours: QuietHoursSettings;
|
||||
healthChecks: HealthCheckSettings;
|
||||
digest: DigestSettings;
|
||||
consistency: ConsistencySettings;
|
||||
nameGenerator: NameGeneratorSettings;
|
||||
alertTickets: AlertTicketSettings;
|
||||
@@ -182,6 +192,7 @@ const DEFAULTS: AppSettings = {
|
||||
display: { dateFormat: "ymd", timeFormat: "24h", pageSize: 20 },
|
||||
logRetention: { enabled: false, retentionDays: 90, intervalHours: 24 },
|
||||
quietHours: { enabled: false, start: "22:00", end: "07:00" },
|
||||
digest: { enabled: false, day: 1, time: "08:00" },
|
||||
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14 },
|
||||
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
|
||||
// Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN.
|
||||
|
||||
@@ -29,6 +29,8 @@ export interface AgentReport {
|
||||
hostname?: string;
|
||||
/** What the agent says it runs on. Only "linux" and "windows" are recorded; anything else is ignored. */
|
||||
osType?: string;
|
||||
/** Which version of the agent script is running. Absent from an agent that predates the number. */
|
||||
agentVersion?: number;
|
||||
system?: IncomingSystemInfo | null;
|
||||
tasks: IncomingTask[];
|
||||
}
|
||||
@@ -109,6 +111,7 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
|
||||
// Only stored when the agent reported it, so an agent that predates this (or couldn't tell this time) doesn't wipe what it said before.
|
||||
...(system?.updates && before ? nextUpdateState(before, system.updates, now) : {}),
|
||||
...(report.hostname ? { hostname: report.hostname } : {}),
|
||||
...(report.agentVersion !== undefined ? { agentVersion: report.agentVersion } : {}),
|
||||
// An agent knows what it runs on better than whoever registered the server did.
|
||||
...(report.osType === "linux" || report.osType === "windows" ? { osType: report.osType } : {}),
|
||||
...(system?.ip_addresses ? { ipAddresses: JSON.stringify(system.ip_addresses) } : {}),
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
/**
|
||||
* The weekly digest: one message, sent through the notification channels, that says how the week went and what is coming — alerts now,
|
||||
* what was down and for how long, what happened, disks heading for full, things about to expire, updates waiting, planned maintenance.
|
||||
* Everything in it is already known to the app (the alerts, the uptime record, the audit log, the agents' reports); this only gathers it
|
||||
* into a few lines per topic and leaves out any topic with nothing to say. Plain text, so it reads in a phone notification and an email alike.
|
||||
*/
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, secrets, servers } from "../db/schema.js";
|
||||
import { env } from "../env.js";
|
||||
import { getAlerts } from "./alerts.js";
|
||||
import { getChangesSince } from "./changesSince.js";
|
||||
import { collectDomainAlerts } from "./domainMonitor.js";
|
||||
import { describeTarget, listUpcomingWindows } from "./maintenance.js";
|
||||
import { notify, notifyNow } from "./notify.js";
|
||||
import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
import { computeSecretStatus } from "./secretStatus.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { getUptime } from "./uptimeHistory.js";
|
||||
|
||||
export const DIGEST_DAYS = 7;
|
||||
const DAY_MS = 24 * 3600_000;
|
||||
const MAX_LINES = 8;
|
||||
|
||||
export interface Digest {
|
||||
title: string;
|
||||
message: string;
|
||||
/** Nothing in it needs anyone: the quiet week. */
|
||||
quiet: boolean;
|
||||
}
|
||||
|
||||
interface Section {
|
||||
heading: string;
|
||||
lines: string[];
|
||||
/** A section that only reassures ("every server stayed up") doesn't make the week un-quiet. */
|
||||
attention: boolean;
|
||||
}
|
||||
|
||||
/** "2 h 40 min", "35 min", "3 d 4 h". */
|
||||
export function spanWords(minutes: number): string {
|
||||
const m = Math.max(1, Math.round(minutes));
|
||||
if (m < 60) return `${m} min`;
|
||||
const h = Math.floor(m / 60);
|
||||
if (h < 24) return m % 60 && h < 10 ? `${h} h ${m % 60} min` : `${h} h`;
|
||||
const d = Math.floor(h / 24);
|
||||
return h % 24 ? `${d} d ${h % 24} h` : `${d} d`;
|
||||
}
|
||||
|
||||
const plural = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
|
||||
|
||||
function listWithMore(lines: string[], max = MAX_LINES): string[] {
|
||||
return lines.length > max ? [...lines.slice(0, max), `…and ${lines.length - max} more`] : lines;
|
||||
}
|
||||
|
||||
function whenWords(date: Date, timezone: string): string {
|
||||
return new Intl.DateTimeFormat("en-GB", { timeZone: timezone, weekday: "short", day: "numeric", month: "short", hour: "2-digit", minute: "2-digit", hourCycle: "h23" }).format(date);
|
||||
}
|
||||
|
||||
function dayWords(date: Date, timezone: string): string {
|
||||
return new Intl.DateTimeFormat("en-GB", { timeZone: timezone, weekday: "short", day: "numeric", month: "short", year: "numeric" }).format(date);
|
||||
}
|
||||
|
||||
export async function buildDigest(now: Date = new Date()): Promise<Digest> {
|
||||
const settings = await getSettings();
|
||||
const tz = settings.notifications.timezone || "UTC";
|
||||
const sections: Section[] = [];
|
||||
|
||||
const serverRows = await db
|
||||
.select({ id: servers.id, name: servers.name, lastSeenAt: servers.lastSeenAt, pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince })
|
||||
.from(servers);
|
||||
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
|
||||
|
||||
// ── Right now ──
|
||||
try {
|
||||
const report = await getAlerts(false);
|
||||
const active = report.alerts.filter((a) => !a.silenced && a.severity !== "info");
|
||||
const lines: string[] = [];
|
||||
if (active.length === 0) lines.push(report.couldntCheck.length > 0 ? `No active alerts, but ${plural(report.couldntCheck.length, "check")} couldn't run.` : "No active alerts.");
|
||||
else {
|
||||
lines.push(`${report.counts.critical} critical, ${report.counts.warning} warning`);
|
||||
// Things about to expire have a section of their own, with dates; they are counted here but not listed twice.
|
||||
const listed = active.filter((a) => a.category !== "expiry");
|
||||
lines.push(...listWithMore(listed.map((a) => `- ${a.message}`), 5));
|
||||
if (listed.length < active.length) lines.push(`(${plural(active.length - listed.length, "more is", "more are")} about things expiring, below)`);
|
||||
}
|
||||
const held: string[] = [];
|
||||
if (report.counts.silenced > 0) held.push(`${report.counts.silenced} silenced by maintenance`);
|
||||
if (report.counts.grouped > 0) held.push(`${report.counts.grouped} folded under a server that is down`);
|
||||
if (held.length > 0) lines.push(`(${held.join(", ")})`);
|
||||
sections.push({ heading: "Right now", lines, attention: active.length > 0 || report.couldntCheck.length > 0 });
|
||||
} catch (err) {
|
||||
console.error("[digest] couldn't read the alerts:", err);
|
||||
}
|
||||
|
||||
// ── Uptime ──
|
||||
try {
|
||||
const uptime = await getUptime(now, DIGEST_DAYS);
|
||||
const integrationName = new Map((await db.select({ id: integrations.id, name: integrations.name }).from(integrations)).map((i) => [i.id, i.name]));
|
||||
const rows: { name: string; down: number; percent: number }[] = [];
|
||||
let measured = 0;
|
||||
for (const [id, series] of Object.entries(uptime.servers)) {
|
||||
measured++;
|
||||
const down = series.buckets.reduce((n, b) => n + b.d, 0);
|
||||
if (down > 0) rows.push({ name: serverName.get(Number(id)) ?? `server ${id}`, down, percent: series.percent });
|
||||
}
|
||||
for (const [id, series] of Object.entries(uptime.integrations)) {
|
||||
measured++;
|
||||
const down = series.buckets.reduce((n, b) => n + b.d, 0);
|
||||
if (down > 0) rows.push({ name: integrationName.get(Number(id)) ?? `integration ${id}`, down, percent: series.percent });
|
||||
}
|
||||
if (measured > 0) {
|
||||
rows.sort((a, b) => b.down - a.down);
|
||||
sections.push({
|
||||
heading: `Uptime, last ${DIGEST_DAYS} days`,
|
||||
lines: rows.length === 0 ? ["Everything stayed up."] : listWithMore(rows.map((r) => `- ${r.name}: ${r.percent}% (${spanWords(r.down)} down)`)),
|
||||
attention: rows.length > 0,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[digest] couldn't read the uptime:", err);
|
||||
}
|
||||
|
||||
// ── What happened ──
|
||||
try {
|
||||
const changes = await getChangesSince({ since: new Date(now.getTime() - DIGEST_DAYS * DAY_MS), now, actorId: 0, canSeeAudit: false });
|
||||
if (changes.events.length > 0) {
|
||||
sections.push({ heading: "What happened", lines: listWithMore(changes.events.map((e) => `- ${e.title}`)), attention: changes.events.some((e) => e.tone === "bad") });
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[digest] couldn't read what happened:", err);
|
||||
}
|
||||
|
||||
// ── Disks heading for full ──
|
||||
try {
|
||||
const { diskForecastDays, diskUsagePercent } = settings.healthChecks;
|
||||
if (diskForecastDays > 0) {
|
||||
const lines: string[] = [];
|
||||
for (const s of serverRows) {
|
||||
for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), diskForecastDays, diskUsagePercent)) {
|
||||
lines.push(`- ${s.name}: ${f.mount} is ${Math.round(f.latest)}% full and filling by about ${f.perDay.toFixed(1)} points a day — full in about ${plural(f.daysToFull, "day")}`);
|
||||
}
|
||||
}
|
||||
if (lines.length > 0) sections.push({ heading: "Disks heading for full", lines: listWithMore(lines), attention: true });
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[digest] couldn't read the disk forecasts:", err);
|
||||
}
|
||||
|
||||
// ── About to expire ──
|
||||
try {
|
||||
const expiring: { daysLeft: number; text: string }[] = [];
|
||||
for (const s of await db.select().from(secrets)) {
|
||||
const status = computeSecretStatus(s.expiryDate, s.warnDays, now);
|
||||
if (status.status === "expired") expiring.push({ daysLeft: status.daysLeft, text: `Secret "${s.name}" expired ${plural(Math.abs(status.daysLeft), "day")} ago` });
|
||||
else if (status.status === "expiring") expiring.push({ daysLeft: status.daysLeft, text: `Secret "${s.name}" expires in ${plural(status.daysLeft, "day")} (${s.expiryDate})` });
|
||||
}
|
||||
for (const d of (await collectDomainAlerts(now)).expiring) {
|
||||
expiring.push({ daysLeft: d.daysLeft, text: d.daysLeft < 0 ? `Domain ${d.name} expired ${plural(Math.abs(d.daysLeft), "day")} ago` : `Domain ${d.name} expires in ${plural(d.daysLeft, "day")} (${d.expiresAt})` });
|
||||
}
|
||||
if (expiring.length > 0) {
|
||||
expiring.sort((a, b) => a.daysLeft - b.daysLeft);
|
||||
sections.push({ heading: "Expiring", lines: listWithMore(expiring.map((e) => `- ${e.text}`)), attention: true });
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[digest] couldn't read what is expiring:", err);
|
||||
}
|
||||
|
||||
// ── Updates waiting ──
|
||||
try {
|
||||
const waiting = serverRows
|
||||
.map((s) => ({ name: s.name, updates: parsePendingUpdates(s.pendingUpdates), since: s.updatesPendingSince }))
|
||||
.filter((s) => s.updates && s.updates.count > 0);
|
||||
if (waiting.length > 0) {
|
||||
const security = waiting.reduce((n, s) => n + (s.updates!.security ?? 0), 0);
|
||||
const restarts = waiting.filter((s) => s.updates!.rebootRequired).length;
|
||||
const oldest = waiting.filter((s) => s.since).sort((a, b) => Date.parse(a.since!) - Date.parse(b.since!))[0];
|
||||
const lines = [`${plural(waiting.length, "server")} ${waiting.length === 1 ? "has" : "have"} updates waiting${security > 0 ? ` (${plural(security, "security fix", "security fixes")} among them)` : ""}.`];
|
||||
if (restarts > 0) lines.push(`${plural(restarts, "server")} ${restarts === 1 ? "needs" : "need"} a restart.`);
|
||||
if (oldest) lines.push(`Longest wait: ${oldest.name}, ${spanWords((now.getTime() - Date.parse(oldest.since!)) / 60_000)}.`);
|
||||
const withSecurity = waiting.filter((s) => (s.updates!.security ?? 0) > 0).sort((a, b) => (b.updates!.security ?? 0) - (a.updates!.security ?? 0));
|
||||
lines.push(...listWithMore(withSecurity.map((s) => `- ${s.name}: ${s.updates!.count} waiting, ${s.updates!.security} security`), 5));
|
||||
sections.push({ heading: "Updates waiting", lines, attention: security > 0 });
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[digest] couldn't read the pending updates:", err);
|
||||
}
|
||||
|
||||
// ── Planned ──
|
||||
try {
|
||||
const horizon = now.getTime() + DIGEST_DAYS * DAY_MS;
|
||||
const groups = new Map<string, { start: string; end: string; reason: string | null; names: string[] }>();
|
||||
for (const w of await listUpcomingWindows(now)) {
|
||||
if (Date.parse(w.startedAt) > horizon) continue;
|
||||
const target = await describeTarget(w.targetType, w.targetId);
|
||||
if (!target) continue;
|
||||
const key = `${w.startedAt}|${w.endsAt}|${w.reason ?? ""}`;
|
||||
const g = groups.get(key) ?? { start: w.startedAt, end: w.endsAt, reason: w.reason, names: [] };
|
||||
g.names.push(target.name);
|
||||
groups.set(key, g);
|
||||
}
|
||||
if (groups.size > 0) {
|
||||
const lines = [...groups.values()]
|
||||
.sort((a, b) => a.start.localeCompare(b.start))
|
||||
.map((g) => {
|
||||
const sorted = [...g.names].sort((a, b) => a.localeCompare(b));
|
||||
const names = sorted.length <= 3 ? sorted.join(", ") : `${sorted.slice(0, 3).join(", ")} and ${sorted.length - 3} more`;
|
||||
return `- ${whenWords(new Date(g.start), tz)}, ${spanWords((Date.parse(g.end) - Date.parse(g.start)) / 60_000)}: ${names}${g.reason ? ` — ${g.reason}` : ""}`;
|
||||
});
|
||||
sections.push({ heading: "Planned maintenance", lines: listWithMore(lines, 6), attention: false });
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[digest] couldn't read the planned maintenance:", err);
|
||||
}
|
||||
|
||||
const quiet = !sections.some((s) => s.attention);
|
||||
const envName = env.environment?.name;
|
||||
const title = `Homelab weekly digest${envName ? ` (${envName})` : ""}`;
|
||||
const body = sections.map((s) => `${s.heading.toUpperCase()}\n${s.lines.join("\n")}`).join("\n\n");
|
||||
const message = `${dayWords(now, tz)}\n${quiet ? "A quiet week: nothing needs attention.\n" : ""}\n${body}`.trim();
|
||||
return { title, message, quiet };
|
||||
}
|
||||
|
||||
/** Builds the digest and sends it through the channels — held with other notifications during quiet hours, unless `immediately`. */
|
||||
export async function sendDigest(options: { immediately?: boolean; now?: Date } = {}): Promise<Digest> {
|
||||
const digest = await buildDigest(options.now);
|
||||
if (options.immediately) await notifyNow(digest.title, digest.message);
|
||||
else await notify(digest.title, digest.message);
|
||||
return digest;
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import schedule from "node-schedule";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { sendDigest } from "./weeklyDigest.js";
|
||||
|
||||
let currentJob: schedule.Job | null = null;
|
||||
|
||||
/** The cron rule for "this weekday (0 = Sunday) at HH:MM". */
|
||||
export function cronForDigest(day: number, time: string): string {
|
||||
const [h, m] = time.split(":").map(Number);
|
||||
const d = Number.isInteger(day) && day >= 0 && day <= 6 ? day : 1;
|
||||
return `${Number.isFinite(m) ? m : 0} ${Number.isFinite(h) ? h : 8} * * ${d}`;
|
||||
}
|
||||
|
||||
/** (Re)schedules the weekly digest per the current settings. Call again after they change. */
|
||||
export async function scheduleWeeklyDigest(): Promise<void> {
|
||||
if (currentJob) {
|
||||
currentJob.cancel();
|
||||
currentJob = null;
|
||||
}
|
||||
const { digest, notifications } = await getSettings();
|
||||
if (!digest.enabled) {
|
||||
console.log("[digest] weekly digest is off, nothing scheduled");
|
||||
return;
|
||||
}
|
||||
currentJob = schedule.scheduleJob({ rule: cronForDigest(digest.day, digest.time), tz: notifications.timezone }, () => {
|
||||
sendDigest().catch((err) => console.error("[digest] couldn't send the weekly digest:", err));
|
||||
});
|
||||
console.log(`Weekly digest scheduled: ${cronForDigest(digest.day, digest.time)} (${notifications.timezone})`);
|
||||
}
|
||||
|
||||
export async function initWeeklyDigestScheduler(): Promise<void> {
|
||||
await scheduleWeeklyDigest();
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
/**
|
||||
* Which version of the agent each server runs, and which agents are behind the scripts this app serves.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-agentver-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { parseAgentVersion, latestAgentVersions, isAgentOutdated } = await import("../src/services/agentVersions.js");
|
||||
const { serversRouter } = await import("../src/routes/servers.js");
|
||||
const { agentReportRouter } = await import("../src/routes/agentReport.js");
|
||||
const { hashToken } = await import("../src/services/tokens.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
describe("reading a version out of a script", () => {
|
||||
test("each script says what it is, in the form the server looks for", () => {
|
||||
assert.equal(parseAgentVersion("#!/usr/bin/env bash\nset -e\n\nAGENT_VERSION=7\nENV_FILE=x\n", "linux"), 7);
|
||||
assert.equal(parseAgentVersion("$script:AgentVersion = 12\r\n$script:Other = 1\r\n", "windows"), 12);
|
||||
assert.equal(parseAgentVersion("# AGENT_VERSION=3 in a comment\n", "linux"), null, "only a line that is the assignment counts");
|
||||
assert.equal(parseAgentVersion("no version here", "windows"), null);
|
||||
});
|
||||
|
||||
test("the scripts this app really serves each carry a version", () => {
|
||||
const latest = latestAgentVersions();
|
||||
assert.ok(Number.isInteger(latest.linux) && latest.linux! >= 2, `linux ${latest.linux}`);
|
||||
assert.ok(Number.isInteger(latest.windows) && latest.windows! >= 2, `windows ${latest.windows}`);
|
||||
});
|
||||
|
||||
test("a folder with nothing in it gives no version, not an error", () => {
|
||||
const empty = mkdtempSync(join(tmpdir(), "hm-noagent-"));
|
||||
assert.deepEqual(latestAgentVersions(empty), { linux: null, windows: null });
|
||||
mkdirSync(join(empty, "linux"));
|
||||
writeFileSync(join(empty, "linux", "report-tasks.sh"), "AGENT_VERSION=5\n");
|
||||
assert.deepEqual(latestAgentVersions(empty), { linux: 5, windows: null });
|
||||
rmSync(empty, { recursive: true, force: true });
|
||||
});
|
||||
});
|
||||
|
||||
describe("which agents are behind", () => {
|
||||
const latest = { linux: 3, windows: 2 };
|
||||
const seen = "2026-10-17T00:00:00Z";
|
||||
|
||||
test("an agent that says nothing is version 1, and older than the latest is behind", () => {
|
||||
assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: seen, agentVersion: null }, latest), true);
|
||||
assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: seen, agentVersion: 2 }, latest), true);
|
||||
assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: seen, agentVersion: 3 }, latest), false);
|
||||
assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: seen, agentVersion: 4 }, latest), false, "ahead of this app is not behind");
|
||||
assert.equal(isAgentOutdated({ osType: "windows", lastSeenAt: seen, agentVersion: 2 }, latest), false, "each system has its own latest");
|
||||
});
|
||||
|
||||
test("a server with no agent has nothing to update, and an unknown latest flags nothing", () => {
|
||||
assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: null, agentVersion: null }, latest), false);
|
||||
assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: seen, agentVersion: null }, { linux: null, windows: null }), false);
|
||||
assert.equal(isAgentOutdated({ osType: "plan9", lastSeenAt: seen, agentVersion: null }, latest), false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the version in reports", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let id = 0;
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values({ oidcSub: "admin", name: "Ada", role: "admin" });
|
||||
const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t" }).returning();
|
||||
id = s.id;
|
||||
const a = express();
|
||||
a.use("/api/agent/report", agentReportRouter);
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: "admin" } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/servers", serversRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const x = a.listen(0, "127.0.0.1", () => resolve(x));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const report = (extra: Record<string, unknown>) =>
|
||||
fetch(`${base}/agent/report`, { method: "POST", headers: { "content-type": "application/json", authorization: "Bearer tok" }, body: JSON.stringify({ hostname: "h", os_type: "linux", tasks: [], ...extra }) });
|
||||
|
||||
test("an agent's version is stored; a report without one, or with nonsense, leaves it as it was", async () => {
|
||||
assert.equal((await report({})).status, 202);
|
||||
let [s] = await db.select().from(servers).where(eq(servers.id, id));
|
||||
assert.equal(s.agentVersion, null, "an old agent says nothing");
|
||||
|
||||
assert.equal((await report({ agent_version: 2 })).status, 202);
|
||||
[s] = await db.select().from(servers).where(eq(servers.id, id));
|
||||
assert.equal(s.agentVersion, 2);
|
||||
|
||||
for (const odd of ["two", 0, -1, 1.5, 999999999, null, {}]) {
|
||||
assert.equal((await report({ agent_version: odd })).status, 202, `${JSON.stringify(odd)} must not cost the report`);
|
||||
}
|
||||
[s] = await db.select().from(servers).where(eq(servers.id, id));
|
||||
assert.equal(s.agentVersion, 2);
|
||||
});
|
||||
|
||||
test("the lists say what the latest is, and the server's own row says what it runs", async () => {
|
||||
const list = await (await fetch(`${base}/servers`)).json();
|
||||
assert.ok(list.agentLatest.linux >= 2 && list.agentLatest.windows >= 2);
|
||||
assert.equal(list.servers.find((x: { id: number }) => x.id === id).agentVersion, 2);
|
||||
|
||||
const detail = await (await fetch(`${base}/servers/${id}/detail`)).json();
|
||||
assert.equal(detail.server.agentVersion, 2);
|
||||
assert.ok(detail.agentLatest.linux >= 2);
|
||||
|
||||
const updates = await (await fetch(`${base}/servers/updates`)).json();
|
||||
assert.equal(updates.servers.find((x: { id: number }) => x.id === id).agentVersion, 2);
|
||||
assert.ok(updates.agentLatest.windows >= 2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,202 @@
|
||||
/**
|
||||
* The weekly digest: what it gathers, what it leaves out, and sending it.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import http from "node:http";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-digest-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, secrets, auditLog, maintenanceWindows } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { buildDigest, spanWords } = await import("../src/services/weeklyDigest.js");
|
||||
const { cronForDigest } = await import("../src/services/weeklyDigestScheduler.js");
|
||||
const { settingsRouter } = await import("../src/routes/settings.js");
|
||||
const { updateSettings, getSettings } = await import("../src/services/settingsStore.js");
|
||||
const { invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const HOUR = 3600_000;
|
||||
const DAY = 24 * HOUR;
|
||||
const NOW = new Date(Math.floor(Date.now() / 1000) * 1000);
|
||||
const dbTime = (ms: number) => new Date(ms).toISOString().slice(0, 19).replace("T", " ");
|
||||
|
||||
describe("small pieces", () => {
|
||||
test("spans read as words", () => {
|
||||
assert.equal(spanWords(35), "35 min");
|
||||
assert.equal(spanWords(160), "2 h 40 min");
|
||||
assert.equal(spanWords(60 * 30), "1 d 6 h");
|
||||
assert.equal(spanWords(60 * 9), "9 h");
|
||||
assert.equal(spanWords(60 * 24 * 3 + 240), "3 d 4 h");
|
||||
});
|
||||
|
||||
test("the schedule is the chosen weekday and time", () => {
|
||||
assert.equal(cronForDigest(1, "08:00"), "0 8 * * 1");
|
||||
assert.equal(cronForDigest(0, "21:30"), "30 21 * * 0");
|
||||
assert.equal(cronForDigest(9, "nonsense"), "0 8 * * 1", "a bad value falls back rather than scheduling nonsense");
|
||||
});
|
||||
});
|
||||
|
||||
describe("the quiet week", () => {
|
||||
test("with nothing wrong it says so, and leaves out topics with nothing to say", async () => {
|
||||
const digest = await buildDigest(NOW);
|
||||
assert.equal(digest.quiet, true);
|
||||
assert.match(digest.message, /A quiet week: nothing needs attention\./);
|
||||
assert.match(digest.message, /RIGHT NOW\nNo active alerts\./);
|
||||
for (const heading of ["UPTIME", "WHAT HAPPENED", "DISKS HEADING FOR FULL", "EXPIRING", "UPDATES WAITING", "PLANNED MAINTENANCE"]) {
|
||||
assert.ok(!digest.message.includes(heading), `no ${heading} section when there's nothing in it`);
|
||||
}
|
||||
assert.match(digest.title, /^Homelab weekly digest/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("a week with things in it", () => {
|
||||
let app: http.Server;
|
||||
let base = "";
|
||||
let webhook: http.Server;
|
||||
const sent: { content: string }[] = [];
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values({ oidcSub: "admin", name: "Ada", role: "admin" });
|
||||
const made = await db
|
||||
.insert(servers)
|
||||
.values([
|
||||
{ name: "elsa", apiTokenHash: "a", apiTokenPrefix: "a", createdAt: dbTime(NOW.getTime() - 40 * DAY), lastSeenAt: new Date(NOW.getTime() - 5 * HOUR).toISOString() },
|
||||
{ name: "anna", apiTokenHash: "b", apiTokenPrefix: "b", createdAt: dbTime(NOW.getTime() - 40 * DAY), lastSeenAt: NOW.toISOString(), pendingUpdates: JSON.stringify({ manager: "apt", count: 9, security: 3, rebootRequired: true, reportedAt: NOW.toISOString() }), updatesPendingSince: new Date(NOW.getTime() - 12 * DAY).toISOString() },
|
||||
{ name: "zoe", apiTokenHash: "c", apiTokenPrefix: "c", createdAt: dbTime(NOW.getTime() - 40 * DAY), lastSeenAt: NOW.toISOString(), pendingUpdates: JSON.stringify({ manager: "apt", count: 0, security: 0, rebootRequired: false, reportedAt: NOW.toISOString() }) },
|
||||
])
|
||||
.returning({ id: servers.id });
|
||||
const [elsa, anna] = made.map((m) => m.id);
|
||||
|
||||
// elsa was down for 2 hours three days ago, and has now stopped reporting again.
|
||||
await db.insert(auditLog).values([
|
||||
{ category: "server", action: "went_offline", targetType: "server", targetId: String(elsa), actorLabel: "system", createdAt: dbTime(NOW.getTime() - 3 * DAY) },
|
||||
{ category: "server", action: "back_online", targetType: "server", targetId: String(elsa), actorLabel: "system", createdAt: dbTime(NOW.getTime() - 3 * DAY + 2 * HOUR) },
|
||||
]);
|
||||
|
||||
const date = (days: number) => new Date(NOW.getTime() + days * DAY).toISOString().slice(0, 10);
|
||||
await db.insert(secrets).values([
|
||||
{ name: "VPN key", type: "api_token", expiryDate: date(9), warnDays: 30 },
|
||||
{ name: "Old token", type: "api_token", expiryDate: date(-4), warnDays: 30 },
|
||||
{ name: "Fine for years", type: "api_token", expiryDate: date(900), warnDays: 30 },
|
||||
]);
|
||||
|
||||
await db.insert(maintenanceWindows).values([elsa, anna].map((id) => ({
|
||||
targetType: "server" as const,
|
||||
targetId: id,
|
||||
reason: "Patch night",
|
||||
startedAt: new Date(NOW.getTime() + 2 * DAY).toISOString(),
|
||||
endsAt: new Date(NOW.getTime() + 2 * DAY + 2 * HOUR).toISOString(),
|
||||
createdBy: "Ada",
|
||||
})));
|
||||
|
||||
webhook = await new Promise<http.Server>((resolve) => {
|
||||
const s = http.createServer((req, res) => {
|
||||
let raw = "";
|
||||
req.on("data", (c) => (raw += c));
|
||||
req.on("end", () => {
|
||||
sent.push(JSON.parse(raw));
|
||||
res.end("ok");
|
||||
});
|
||||
});
|
||||
s.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: "admin" } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/settings", settingsRouter);
|
||||
app = await new Promise<http.Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/settings`;
|
||||
invalidateAlertsCache();
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
// Saving the schedule started a cron job, which would keep this process alive.
|
||||
await (await import("node-schedule")).default.gracefulShutdown();
|
||||
app.close();
|
||||
webhook.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
test("it gathers each topic into a few lines", async () => {
|
||||
const digest = await buildDigest(NOW);
|
||||
assert.equal(digest.quiet, false);
|
||||
const m = digest.message;
|
||||
assert.match(m, /RIGHT NOW\n.*critical/, m);
|
||||
assert.ok(!/RIGHT NOW[^]*Secret "Old token"[^]*EXPIRING/.test(m), "an expiring secret is in its own section, not listed twice");
|
||||
assert.match(m, /\(1 more is about things expiring, below\)|\(2 more are about things expiring, below\)/);
|
||||
assert.match(m, /elsa hasn't reported for 5 h/);
|
||||
assert.match(m, /UPTIME, LAST 7 DAYS\n- elsa: .*% \(.*down\)/s, "elsa's downtime is told with a figure");
|
||||
assert.match(m, /WHAT HAPPENED\n- elsa was offline for 2 h/);
|
||||
assert.match(m, /EXPIRING\n- Secret "Old token" expired 4 days ago\n- Secret "VPN key" expires in 9 days/);
|
||||
assert.ok(!m.includes("Fine for years"));
|
||||
assert.match(m, /UPDATES WAITING\n1 server has updates waiting \(3 security fixes among them\)\.\n1 server needs a restart\.\nLongest wait: anna, 12 d\.\n- anna: 9 waiting, 3 security/);
|
||||
assert.match(m, /PLANNED MAINTENANCE\n- .*2 h: anna, elsa — Patch night/);
|
||||
});
|
||||
|
||||
test("preview needs an administrator's route and shows the same text; sending needs somewhere to send to", async () => {
|
||||
const preview = await (await fetch(`${base}/digest/preview`)).json();
|
||||
assert.match(preview.title, /^Homelab weekly digest/);
|
||||
assert.match(preview.message, /UPDATES WAITING/);
|
||||
assert.equal(preview.channels, 0);
|
||||
|
||||
const none = await fetch(`${base}/digest/send`, { method: "POST" });
|
||||
assert.equal(none.status, 400);
|
||||
assert.equal((await none.json()).error, "no_channel");
|
||||
assert.equal(sent.length, 0);
|
||||
});
|
||||
|
||||
test("sending goes out through a switched-on channel at once, and is in the audit log", async () => {
|
||||
const url = `http://127.0.0.1:${(webhook.address() as AddressInfo).port}/hook`;
|
||||
await updateSettings({ webhook: { enabled: true, url }, quietHours: { enabled: true, start: "00:00", end: "23:59" } });
|
||||
const res = await fetch(`${base}/digest/send`, { method: "POST" });
|
||||
assert.equal(res.status, 200);
|
||||
assert.deepEqual(await res.json(), { sent: true, channels: 1 });
|
||||
assert.equal(sent.length, 1, "sent at once even inside quiet hours: a person asked for it now");
|
||||
assert.match(sent[0].content, /^\*\*Homelab weekly digest/);
|
||||
assert.match(sent[0].content, /UPDATES WAITING/);
|
||||
const entries = (await db.select().from(auditLog)).filter((e) => e.action === "send_digest");
|
||||
assert.equal(entries.length, 1);
|
||||
await updateSettings({ quietHours: { enabled: false } });
|
||||
});
|
||||
|
||||
test("the schedule is saved, and a day or time that makes no sense is refused", async () => {
|
||||
const put = (digest: unknown) => fetch(`${base}/`, { method: "PUT", headers: { "content-type": "application/json" }, body: JSON.stringify({ digest }) });
|
||||
assert.equal((await put({ enabled: true, day: 0, time: "21:30" })).status, 200);
|
||||
assert.deepEqual((await getSettings()).digest, { enabled: true, day: 0, time: "21:30" });
|
||||
for (const bad of [{ day: 7 }, { day: -1 }, { day: 1.5 }, { time: "25:00" }, { time: "8am" }, { enabled: "yes" }]) {
|
||||
assert.equal((await put(bad)).status, 400, JSON.stringify(bad));
|
||||
}
|
||||
assert.deepEqual((await getSettings()).digest, { enabled: true, day: 0, time: "21:30" }, "nothing was half-saved");
|
||||
});
|
||||
|
||||
test("it is off until switched on", async () => {
|
||||
const fresh = mkdtempSync(join(tmpdir(), "x-"));
|
||||
rmSync(fresh, { recursive: true, force: true });
|
||||
const { digest } = await getSettings();
|
||||
assert.equal(typeof digest.enabled, "boolean");
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user