Bulk snooze, alert rules judged on report, photos on notes

Bulk snooze: Snooze... and Wake on the ticked alerts, with undo that puts back
each alert's earlier snooze exactly.

Alert rules are judged for a server right after its agent reports, instead of at
the next 15-minute pass; only that server's rule findings change, and the check
is serialized with the pass so neither overwrites the other's state.

Photos on server and device notes: up to 12 each, shrunk in the browser,
accepted by their own bytes (JPEG/PNG/GIF/WebP, 4 MB), kept in the database and
removed with the server or device.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-24 22:53:09 +02:00
1 parent 251a4a9d73
commit a16996d975
27 files changed
+8442 -54

No files matched your search

+16
View File
@@ -0,0 +1,16 @@
CREATE TABLE `note_photos` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`server_id` integer,
`device_id` integer,
`content_type` text NOT NULL,
`data` blob NOT NULL,
`size` integer NOT NULL,
`caption` text,
`created_by` text,
`created_at` text NOT NULL,
FOREIGN KEY (`server_id`) REFERENCES `servers`(`id`) ON UPDATE no action ON DELETE cascade,
FOREIGN KEY (`device_id`) REFERENCES `network_devices`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE INDEX `note_photos_server_idx` ON `note_photos` (`server_id`);--> statement-breakpoint
CREATE INDEX `note_photos_device_idx` ON `note_photos` (`device_id`);
File diff suppressed because it is too large. Load diff
+7
View File
@@ -435,6 +435,13 @@
"when": 1792873347697,
"tag": "0061_reflective_daimon_hellstrom",
"breakpoints": true
},
{
"idx": 62,
"version": "6",
"when": 1792874762673,
"tag": "0062_spotty_viper",
"breakpoints": true
}
]
}
+2
View File
@@ -49,6 +49,7 @@ import { networkRouter } from "./routes/network.js";
import { myWorkRouter } from "./routes/myWork.js";
import { apiTokensRouter } from "./routes/apiTokens.js";
import { notesRouter, todosRouter } from "./routes/todos.js";
import { notePhotosRouter } from "./routes/notePhotos.js";
import { npmRouter } from "./routes/npm.js";
import { alertRulesRouter } from "./routes/alertRules.js";
import { storageRouter } from "./routes/storage.js";
@@ -206,6 +207,7 @@ export function createApp() {
app.use("/api/service-levels", serviceLevelsRouter);
app.use("/api/todos", todosRouter);
app.use("/api/notes", notesRouter);
app.use("/api/note-photos", notePhotosRouter);
app.use("/api/generator", generatorRouter);
app.use("/api/alert-tickets", alertTicketsRouter);
+20 -1
View File
@@ -1,5 +1,5 @@
import { sql } from "drizzle-orm";
import { sqliteTable, text, integer, real, uniqueIndex, index, primaryKey } from "drizzle-orm/sqlite-core";
import { sqliteTable, text, integer, real, blob, uniqueIndex, index, primaryKey } from "drizzle-orm/sqlite-core";
// ─── Users & roles ──────────────────────────────────────────────────────────
@@ -1424,6 +1424,25 @@ export const notes = sqliteTable(
(t) => [uniqueIndex("notes_target_idx").on(t.targetType, t.targetId)],
);
// Photos on a note — the rack as it is, a cable's label, the back of the NAS. Kept in the database itself, so a database backup has them and nothing else needs a volume; each is
// one server's or one device's, with a foreign key that cascades, so removing the server removes its photos. The bytes are exactly what the browser sent (it makes them small first),
// and only a JPEG, PNG, GIF or WebP is accepted, told by what the bytes are and not by what the upload says it is.
export const notePhotos = sqliteTable(
"note_photos",
{
id: integer("id").primaryKey({ autoIncrement: true }),
serverId: integer("server_id").references(() => servers.id, { onDelete: "cascade" }),
deviceId: integer("device_id").references(() => networkDevices.id, { onDelete: "cascade" }),
contentType: text("content_type").notNull(),
data: blob("data", { mode: "buffer" }).notNull(),
size: integer("size").notNull(),
caption: text("caption"),
createdBy: text("created_by"),
createdAt: text("created_at").notNull(), // ISO
},
(t) => [index("note_photos_server_idx").on(t.serverId), index("note_photos_device_idx").on(t.deviceId)],
);
// Things to do: the small jobs of a homelab ("replace the UPS battery", "move the NAS to the new switch"), with a day they are due and optionally a person and
// a thing they are about. Open ones with a due day are in Coming up; the ones handed to someone are in My work.
export const todos = sqliteTable("todos", {
+3
View File
@@ -3,6 +3,7 @@ import { z } from "zod";
import { db } from "../db/client.js";
import { hashToken } from "../services/tokens.js";
import { syncServerTasks } from "../services/taskSync.js";
import { checkRulesFor } from "../services/healthMonitor.js";
import { asyncHandler } from "../utils/asyncHandler.js";
import { getSettings } from "../services/settingsStore.js";
import { latestAgentSha256, latestAgentVersions } from "../services/agentVersions.js";
@@ -206,4 +207,6 @@ agentReportRouter.post("/", asyncHandler(authenticateAgent), express.json({ limi
});
res.status(202).json({ ok: true, taskCount: parsed.data.tasks.length });
// The figures are as new as they will be: judge the alert rules for this server now rather than at the next pass. After the answer, so the agent is not kept waiting.
void checkRulesFor(server.id).catch((err) => console.error("[health] couldn't judge the alert rules after a report:", err instanceof Error ? err.message : err));
}));
+26 -2
View File
@@ -2,8 +2,9 @@ import { Router } from "express";
import { z } from "zod";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { AckError, MAX_HOURS, MAX_NOTE } from "../services/alertAcks.js";
import { SnoozeError } from "../services/alertSnoozes.js";
import { recordAudit } from "../services/audit.js";
import { acknowledgeAlerts, BulkInputError, changeAssets, changeServerTag, MAX_BULK, restoreAcks, restoreAssets, restoreServerParents, setServerParents, setServerTags, silenceTargets, targetsOfAlerts, undoSilence, unacknowledgeAlerts } from "../services/bulkActions.js";
import { acknowledgeAlerts, BulkInputError, changeAssets, changeServerTag, MAX_BULK, restoreAcks, restoreAssets, restoreServerParents, restoreSnoozes, setServerParents, setServerTags, silenceTargets, snoozeAlerts, targetsOfAlerts, undoSilence, unacknowledgeAlerts, unsnoozeAlerts } from "../services/bulkActions.js";
import { InvalidTagError } from "../services/serverTags.js";
import { asyncHandler } from "../utils/asyncHandler.js";
@@ -102,7 +103,13 @@ const previousAck = z
})
.nullable();
const snoozeRecord = z.object({ conditionKey: z.string().max(700), message: z.string().max(500), until: z.string().max(40), snoozedBy: z.string().max(200), snoozedAt: z.string().max(40) });
const alertsBody = z.discriminatedUnion("action", [
/** When they should come back, as a date and time with its zone. */
z.object({ action: z.literal("snooze"), alertIds: stringIds, until: z.string().min(1).max(40) }),
z.object({ action: z.literal("unsnooze"), alertIds: stringIds }),
z.object({ action: z.literal("restore_snoozes"), items: z.array(z.object({ alertId: z.string().min(1).max(600), previous: snoozeRecord.nullable() })).min(1).max(MAX_BULK) }),
z.object({ action: z.literal("acknowledge"), alertIds: stringIds, note: z.string().max(MAX_NOTE).default(""), assigneeId: z.number().int().positive().nullable().default(null), hours: z.number().int().min(1).max(MAX_HOURS).nullable().default(null) }),
z.object({ action: z.literal("unacknowledge"), alertIds: stringIds }),
z.object({ action: z.literal("restore_acks"), items: z.array(z.object({ alertId: z.string().min(1).max(600), previous: previousAck })).min(1).max(MAX_BULK) }),
@@ -132,6 +139,23 @@ alertsBulkRouter.post("/", asyncHandler(async (req, res) => {
await recordAudit({ actor, category: "alert", action: "bulk_undo", detail: { changed } });
return res.json({ changed, skipped: [], undo: null });
}
case "snooze": {
const until = new Date(body.until);
if (Number.isNaN(until.getTime())) return bad(res, "Say when they should come back.");
const r = await snoozeAlerts(body.alertIds, until, nameOf(req));
await recordAudit({ actor, category: "alert", action: "bulk_snooze", detail: { changed: r.changed, until: until.toISOString(), alerts: detailIds(body.alertIds) } });
return res.json(r);
}
case "unsnooze": {
const r = await unsnoozeAlerts(body.alertIds);
await recordAudit({ actor, category: "alert", action: "bulk_unsnooze", detail: { changed: r.changed, alerts: detailIds(body.alertIds) } });
return res.json(r);
}
case "restore_snoozes": {
const changed = await restoreSnoozes(body.items);
await recordAudit({ actor, category: "alert", action: "bulk_undo", detail: { changed, what: "snooze" } });
return res.json({ changed, skipped: [], undo: null });
}
case "silence": {
const { targets, skipped } = await targetsOfAlerts(body.alertIds);
const r = await silenceTargets(targets, body.minutes, body.reason || "Maintenance, from several alerts", nameOf(req), false);
@@ -145,7 +169,7 @@ alertsBulkRouter.post("/", asyncHandler(async (req, res) => {
}
}
} catch (err) {
if (err instanceof AckError) return res.status(err.status).json({ error: err.code, message: err.message });
if (err instanceof AckError || err instanceof SnoozeError) return res.status(err.status).json({ error: err.code, message: err.message });
throw err;
}
}));
+83
View File
@@ -0,0 +1,83 @@
import express, { Router, type NextFunction, type Request, type Response } from "express";
import { z } from "zod";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { addPhoto, getPhoto, listPhotos, MAX_PHOTO_BYTES, PhotoError, removePhoto, setCaption } from "../services/notePhotos.js";
import { TARGET_TYPES, targetName } from "../services/todos.js";
import { asyncHandler } from "../utils/asyncHandler.js";
/** Photos on the notes of servers and devices: everyone signed in sees them; operators add, caption and remove. */
export const notePhotosRouter = Router();
notePhotosRouter.use(requireAuth);
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
const label = (u: { name: string | null; email: string | null; oidcSub: string }) => u.name ?? u.email ?? u.oidcSub;
notePhotosRouter.get("/", asyncHandler(async (req, res) => {
const type = z.enum(TARGET_TYPES).safeParse(req.query.targetType);
const id = idOf(String(req.query.targetId ?? ""));
if (!type.success || !Number.isInteger(id)) return res.status(400).json({ error: "invalid_target" });
res.json({ photos: await listPhotos(type.data, id) });
}));
// The picture itself. What it is is what its bytes say (it was checked when it was added), it cannot run anything — the sandbox policy and nosniff see to that even if it were opened on its own —
// and it never changes, so the browser may keep it.
notePhotosRouter.get("/:id/image", asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(404).json({ error: "not_found" });
const photo = await getPhoto(id);
if (!photo) return res.status(404).json({ error: "not_found" });
res.setHeader("Content-Type", photo.info.contentType);
res.setHeader("Content-Length", String(photo.data.length));
res.setHeader("X-Content-Type-Options", "nosniff");
res.setHeader("Content-Security-Policy", "default-src 'none'; sandbox");
res.setHeader("Cache-Control", "private, max-age=86400, immutable");
res.send(photo.data);
}));
// The bytes of the picture are the body, whatever it says its type is (the first bytes decide); the target and the caption are in the address.
notePhotosRouter.post(
"/",
requireRole("operator"),
express.raw({ type: () => true, limit: MAX_PHOTO_BYTES + 1024 }),
asyncHandler(async (req, res) => {
const type = z.enum(TARGET_TYPES).safeParse(req.query.targetType);
const id = idOf(String(req.query.targetId ?? ""));
if (!type.success || !Number.isInteger(id)) return res.status(400).json({ error: "invalid_target" });
const caption = typeof req.query.caption === "string" ? req.query.caption : null;
const body = Buffer.isBuffer(req.body) ? req.body : Buffer.alloc(0);
try {
const actor = req.currentUser!;
const photo = await addPhoto(type.data, id, body, caption, label(actor));
await recordAudit({ actor, category: "note", action: "photo_add", targetType: type.data, targetId: id, detail: { name: await targetName(type.data, id), photoId: photo.id, size: photo.size, caption: photo.caption } });
res.status(201).json({ photo });
} catch (err) {
if (err instanceof PhotoError) return res.status(err.status).json({ error: err.code, message: err.message });
throw err;
}
}),
);
notePhotosRouter.put("/:id", requireRole("operator"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
const parsed = z.object({ caption: z.string().max(500).nullable() }).safeParse(req.body);
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body" });
const photo = await setCaption(id, parsed.data.caption);
if (!photo) return res.status(404).json({ error: "not_found" });
res.json({ photo });
}));
notePhotosRouter.delete("/:id", requireRole("operator"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const removed = await removePhoto(id);
if (!removed) return res.status(404).json({ error: "not_found" });
await recordAudit({ actor: req.currentUser!, category: "note", action: "photo_delete", targetType: removed.targetType, targetId: removed.targetId, detail: { name: await targetName(removed.targetType, removed.targetId), photoId: id } });
res.status(204).end();
}));
// An upload over the limit is told plainly, not with the framework's page.
notePhotosRouter.use((err: Error & { type?: string }, _req: Request, res: Response, next: NextFunction) => {
if (err.type === "entity.too.large") return res.status(413).json({ error: "too_big", message: `A photo can be at most ${MAX_PHOTO_BYTES / 1024 / 1024} MB.` });
next(err);
});
+7 -5
View File
@@ -7,7 +7,7 @@
* past its limit on every sample for that long. A disk's temperature is only known as of its last report, so it is judged right now. A server that is not reporting is not
* judged — its own offline alert says that.
*/
import { eq, gte } from "drizzle-orm";
import { and, eq, gte } from "drizzle-orm";
import { db } from "../db/client.js";
import { alertRules, serverMetrics, servers, type AlertRuleMetric } from "../db/schema.js";
import { parseHostHealth } from "./hostHealth.js";
@@ -128,11 +128,12 @@ export async function listRules(): Promise<RuleRow[]> {
return (await db.select().from(alertRules)).sort((a, b) => a.name.localeCompare(b.name) || a.id - b.id);
}
/** What the given rules find right now, from the stored history. Reads nothing when no rule is on. */
export async function computeRuleFindings(rules: RuleRow[], now: number = Date.now()): Promise<RuleFinding[]> {
/** What the given rules find right now, from the stored history — on one server only when `onlyServerId` is given, which reads only that server's history. Reads nothing when no rule is on. */
export async function computeRuleFindings(rules: RuleRow[], now: number = Date.now(), onlyServerId?: number): Promise<RuleFinding[]> {
const on = rules.filter((r) => r.enabled);
if (on.length === 0) return [];
const rows = await db.select({ id: servers.id, name: servers.name, tags: servers.tags, hostHealth: servers.hostHealth }).from(servers);
const everything = db.select({ id: servers.id, name: servers.name, tags: servers.tags, hostHealth: servers.hostHealth }).from(servers);
const rows = await (onlyServerId === undefined ? everything : everything.where(eq(servers.id, onlyServerId)));
const serverList: RuleServer[] = rows.map((r) => ({
id: r.id,
name: r.name,
@@ -143,7 +144,8 @@ export async function computeRuleFindings(rules: RuleRow[], now: number = Date.n
if (on.some((r) => r.metric !== "disk_temp")) {
const longest = Math.max(...on.filter((r) => r.metric !== "disk_temp").map((r) => r.forMinutes));
const since = new Date(now - (longest * 60_000 + 3 * 3600_000)).toISOString();
for (const r of await db.select().from(serverMetrics).where(gte(serverMetrics.recordedAt, since))) {
const recent = gte(serverMetrics.recordedAt, since);
for (const r of await db.select().from(serverMetrics).where(onlyServerId === undefined ? recent : and(recent, eq(serverMetrics.serverId, onlyServerId)))) {
if (r.metric !== "cpu" && r.metric !== "mem" && !r.metric.startsWith("disk:")) continue;
const k = sampleKey(r.serverId, r.metric);
const list = samples.get(k) ?? [];
+25
View File
@@ -60,6 +60,31 @@ export async function setSnooze(input: { alertId: string; conditionKey: string;
return { until: values.until, snoozedBy: values.snoozedBy, snoozedAt: values.snoozedAt };
}
/** A snooze as it is stored, whole — what an undo puts back. */
export interface SnoozeRecord {
conditionKey: string;
message: string;
until: string;
snoozedBy: string;
snoozedAt: string;
}
/** The snooze on an alert as it stands, expired or not; null when it has none. */
export async function readSnooze(alertId: string): Promise<SnoozeRecord | null> {
const [row] = await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, alertId)).limit(1);
return row ? { conditionKey: row.conditionKey, message: row.message, until: row.until, snoozedBy: row.snoozedBy, snoozedAt: row.snoozedAt } : null;
}
/** Puts a snooze back as it was (an undo). One whose time has already passed is not worth restoring: it would only announce the alert again at once, so it is dropped. */
export async function restoreSnooze(alertId: string, record: SnoozeRecord, now: Date = new Date()): Promise<void> {
if (Date.parse(record.until) <= now.getTime()) {
await clearSnooze(alertId);
return;
}
const values = { ...record, lastSeenAt: now.toISOString() };
await db.insert(alertSnoozes).values({ alertId, ...values }).onConflictDoUpdate({ target: alertSnoozes.alertId, set: values });
}
export async function clearSnooze(alertId: string): Promise<{ message: string } | null> {
const [row] = await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, alertId)).limit(1);
if (!row) return null;
+53
View File
@@ -8,6 +8,8 @@ import { db } from "../db/client.js";
import { assets, maintenanceWindows, servers } from "../db/schema.js";
import { invalidateAlertsCache, getAlerts } from "./alerts.js";
import { clearAck, restoreAck, setAck, type AckInfo, type SetAckInput } from "./alertAcks.js";
import { clearSnooze, readSnooze, restoreSnooze, setSnooze, type SnoozeRecord } from "./alertSnoozes.js";
import { keyOfAlertId } from "./flapGuard.js";
import { impactOf } from "./dependencies.js";
import { isRealDate } from "./inventory.js";
import { listActiveWindows, startOrExtendWindow } from "./maintenance.js";
@@ -232,6 +234,57 @@ export async function restoreAcks(items: AcksUndo["items"]): Promise<number> {
return items.length;
}
export interface SnoozesUndo {
action: "restore_snoozes";
items: { alertId: string; previous: SnoozeRecord | null }[];
}
/** Snoozes each alert until the same time (an earlier snooze is replaced). Alerts that are no longer there are skipped. A time that is not allowed refuses the whole thing, before anything is changed. */
export async function snoozeAlerts(alertIds: string[], until: Date, actor: string, now: Date = new Date()): Promise<BulkResult<SnoozesUndo>> {
const present = new Map((await getAlerts(false)).alerts.map((a) => [a.id, a]));
const skipped: Skipped[] = [];
const items: SnoozesUndo["items"] = [];
for (const alertId of unique(alertIds)) {
const alert = present.get(alertId);
if (!alert) {
skipped.push({ id: alertId, name: alertId, reason: "it isn't there any more" });
continue;
}
const previous = await readSnooze(alertId);
await setSnooze({ alertId, conditionKey: keyOfAlertId(alert.id, alert.category), message: alert.message, until, actor }, now);
items.push({ alertId, previous });
}
if (items.length > 0) invalidateAlertsCache();
return { changed: items.length, skipped, undo: items.length > 0 ? { action: "restore_snoozes", items } : null };
}
/** Wakes each alert that is snoozed. */
export async function unsnoozeAlerts(alertIds: string[]): Promise<BulkResult<SnoozesUndo>> {
const skipped: Skipped[] = [];
const items: SnoozesUndo["items"] = [];
for (const alertId of unique(alertIds)) {
const previous = await readSnooze(alertId);
if (!previous) {
skipped.push({ id: alertId, name: alertId, reason: "it wasn't snoozed" });
continue;
}
await clearSnooze(alertId);
items.push({ alertId, previous });
}
if (items.length > 0) invalidateAlertsCache();
return { changed: items.length, skipped, undo: items.length > 0 ? { action: "restore_snoozes", items } : null };
}
/** Puts each alert's snooze back as it was (or takes it off, if it had none). */
export async function restoreSnoozes(items: SnoozesUndo["items"], now: Date = new Date()): Promise<number> {
for (const i of items) {
if (i.previous === null) await clearSnooze(i.alertId);
else await restoreSnooze(i.alertId, i.previous, now);
}
if (items.length > 0) invalidateAlertsCache();
return items.length;
}
/** The maintenance targets of the alerts given (those with none can't be silenced). */
export async function targetsOfAlerts(alertIds: string[]): Promise<{ targets: Target[]; skipped: Skipped[] }> {
const present = new Map((await getAlerts(false)).alerts.map((a) => [a.id, a]));
+9 -1
View File
@@ -178,8 +178,16 @@ export function describeChange(category: string, action: string, detail: Record<
return { title: `${typeof detail.changed === "number" ? detail.changed : "Several"} alert${detail.changed === 1 ? "" : "s"} acknowledged at once`, tone: "neutral", detail: typeof detail.note === "string" && detail.note ? detail.note : null };
case "alert/bulk_unack":
return { title: `Acknowledgement taken off ${typeof detail.changed === "number" ? detail.changed : "several"} alert${detail.changed === 1 ? "" : "s"}`, tone: "neutral", detail: null };
case "note/photo_add":
return { title: `A photo was added to a note${name ? ` on ${name}` : ""}`, tone: "neutral", detail: str(detail.caption) };
case "note/photo_delete":
return { title: `A photo was removed from a note${name ? ` on ${name}` : ""}`, tone: "neutral", detail: null };
case "alert/bulk_undo":
return { title: "Acknowledging several alerts at once was undone", tone: "neutral", detail: null };
return { title: detail.what === "snooze" ? "Snoozing several alerts at once was undone" : "Acknowledging several alerts at once was undone", tone: "neutral", detail: null };
case "alert/bulk_snooze":
return { title: `${typeof detail.changed === "number" ? detail.changed : "Several"} alert${detail.changed === 1 ? "" : "s"} snoozed at once`, tone: "neutral", detail: str(detail.until) ? `until ${str(detail.until)!.slice(0, 16).replace("T", " ")} UTC` : null };
case "alert/bulk_unsnooze":
return { title: `${typeof detail.changed === "number" ? detail.changed : "Several"} snoozed alert${detail.changed === 1 ? "" : "s"} woken at once`, tone: "neutral", detail: null };
case "inventory/bulk_change":
return { title: detail.change === "undo" ? "A change to several pieces of hardware was undone" : `${typeof detail.changed === "number" ? detail.changed : "Several"} piece${detail.changed === 1 ? "" : "s"} of hardware changed at once`, tone: "neutral", detail: genericSummary(detail, ["ids", "changed", "change"]) };
case "webhook/retry":
+43 -1
View File
@@ -689,7 +689,49 @@ export function alsoList(names: string[]): string {
return rest > 0 ? `${shown.join(", ")} and ${rest} more` : head;
}
export async function runHealthCheck(now: number = Date.now()): Promise<{ added: number; resolved: number }> {
// One look at a time: the pass and the check that follows an agent's report both read the state of what is known, change it and write it back.
let running: Promise<unknown> = Promise.resolve();
function serialized<T>(work: () => Promise<T>): Promise<T> {
const result = running.then(work, work);
running = result.catch(() => undefined);
return result;
}
export function runHealthCheck(now: number = Date.now()): Promise<{ added: number; resolved: number }> {
return serialized(() => runHealthCheckNow(now));
}
/** A report from the same server is not looked at again within this long. */
export const RULE_CHECK_MIN_GAP_MS = 60_000;
const lastRuleCheck = new Map<number, number>();
/**
* The server has just reported, so its figures are as new as they will be: judge the administrator's alert rules for it now, instead of at the next pass (up to fifteen minutes
* later). Only this server's rule findings are looked at and only they change; everything else — the built-in checks, the other servers, the integrations — waits for the pass, so
* what a pass would say is unchanged. Announced and recovered like the pass does. Not done during the start-up grace, nor for a server looked at in the last minute (`minGapMs`).
*/
export function checkRulesFor(serverId: number, now: number = Date.now(), minGapMs: number = RULE_CHECK_MIN_GAP_MS): Promise<{ added: number; resolved: number } | null> {
if (now - PROCESS_START < STARTUP_GRACE_MS) return Promise.resolve(null);
const last = lastRuleCheck.get(serverId);
if (last !== undefined && now - last < minGapMs) return Promise.resolve(null);
lastRuleCheck.set(serverId, now);
return serialized(async () => {
const findings = await computeRuleFindings(await listRules(), now, serverId);
const mine = new RegExp(`^rule:\\d+:server:${serverId}(:|$)`);
const previous = await loadState();
const before: ActiveState = Object.fromEntries(Object.entries(previous).filter(([k]) => mine.test(k)));
if (findings.length === 0 && Object.keys(before).length === 0) return { added: 0, resolved: 0 };
const current: HealthCondition[] = findings.map((f) => ({ key: findingKey(f), source: "server:rules", message: f.message, severity: f.severity }));
const { added, resolved, next, newlyDown, ended } = diffConditions(before, current, new Set(), await activeSubjects(new Date(now)));
await setInternalFlag(STATE_FLAG, JSON.stringify({ ...Object.fromEntries(Object.entries(previous).filter(([k]) => !mine.test(k))), ...next }));
await recordServerTransitions(newlyDown, ended);
if (added.length > 0) await notifyHealthIssues(added);
if (resolved.length > 0) await notifyHealthRecovered(resolved);
return { added: added.length, resolved: resolved.length };
});
}
async function runHealthCheckNow(now: number = Date.now()): Promise<{ added: number; resolved: number }> {
const { healthChecks } = await getSettings();
const { snapshot, held } = await collectSnapshot();
// What the pools read this time go into their history (one reading an hour at most), so they can be forecast.
+92
View File
@@ -0,0 +1,92 @@
/**
* Photos on the notes of servers and network devices. The browser makes each one small before it is sent (a phone's photo is several megabytes; a screen's worth is a few hundred
* kilobytes), and this is the second line: a photo over 4 MB is refused, so is anything that is not really a JPEG, PNG, GIF or WebP — told by the first bytes, never by the name or
* the type the upload claims (an SVG, which can carry script, is not accepted) — and a thing has at most 12.
*/
import { count, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { notePhotos } from "../db/schema.js";
import { targetName, type TargetType } from "./todos.js";
export const MAX_PHOTO_BYTES = 4 * 1024 * 1024;
export const MAX_PHOTOS_PER_TARGET = 12;
export const MAX_CAPTION = 120;
export class PhotoError extends Error {
constructor(
readonly status: number,
readonly code: string,
message: string,
) {
super(message);
}
}
export interface PhotoInfo {
id: number;
caption: string | null;
contentType: string;
size: number;
createdBy: string | null;
createdAt: string;
}
/** Pure: what the bytes are, from how they begin; null when they are none of the four. */
export function sniffImage(b: Buffer): "image/jpeg" | "image/png" | "image/gif" | "image/webp" | null {
if (b.length >= 3 && b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff) return "image/jpeg";
if (b.length >= 8 && b.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) return "image/png";
if (b.length >= 6 && (b.subarray(0, 6).toString("latin1") === "GIF87a" || b.subarray(0, 6).toString("latin1") === "GIF89a")) return "image/gif";
if (b.length >= 12 && b.subarray(0, 4).toString("latin1") === "RIFF" && b.subarray(8, 12).toString("latin1") === "WEBP") return "image/webp";
return null;
}
const column = (type: TargetType) => (type === "server" ? notePhotos.serverId : notePhotos.deviceId);
const cleanCaption = (c: string | null | undefined) => {
const t = (c ?? "").replace(/\s+/g, " ").trim();
return t === "" ? null : t.slice(0, MAX_CAPTION);
};
const info = (r: Pick<typeof notePhotos.$inferSelect, "id" | "caption" | "contentType" | "size" | "createdBy" | "createdAt">): PhotoInfo => ({ id: r.id, caption: r.caption, contentType: r.contentType, size: r.size, createdBy: r.createdBy, createdAt: r.createdAt });
/** The photos of a server or a device, oldest first (the order they were taken in). */
export async function listPhotos(type: TargetType, id: number): Promise<PhotoInfo[]> {
const rows = await db
.select({ id: notePhotos.id, caption: notePhotos.caption, contentType: notePhotos.contentType, size: notePhotos.size, createdBy: notePhotos.createdBy, createdAt: notePhotos.createdAt })
.from(notePhotos)
.where(eq(column(type), id))
.orderBy(notePhotos.id);
return rows.map(info);
}
export async function addPhoto(type: TargetType, id: number, bytes: Buffer, caption: string | null, actor: string, now: Date = new Date()): Promise<PhotoInfo> {
const targetLabel = await targetName(type, id);
if (!targetLabel) throw new PhotoError(404, "unknown_target", "That server or network device is not there.");
if (bytes.length === 0) throw new PhotoError(400, "empty", "There is nothing in that file.");
if (bytes.length > MAX_PHOTO_BYTES) throw new PhotoError(413, "too_big", `A photo can be at most ${MAX_PHOTO_BYTES / 1024 / 1024} MB.`);
const contentType = sniffImage(bytes);
if (!contentType) throw new PhotoError(400, "not_an_image", "That is not a JPEG, PNG, GIF or WebP picture.");
const [{ n }] = await db.select({ n: count() }).from(notePhotos).where(eq(column(type), id));
if (n >= MAX_PHOTOS_PER_TARGET) throw new PhotoError(400, "too_many", `There can be at most ${MAX_PHOTOS_PER_TARGET} photos on one note. Remove one first.`);
const [row] = await db
.insert(notePhotos)
.values({ serverId: type === "server" ? id : null, deviceId: type === "device" ? id : null, contentType, data: bytes, size: bytes.length, caption: cleanCaption(caption), createdBy: actor, createdAt: now.toISOString() })
.returning();
return info(row);
}
/** One photo with its bytes, and whose it is; null when there is none. */
export async function getPhoto(id: number): Promise<{ info: PhotoInfo; data: Buffer; targetType: TargetType; targetId: number } | null> {
const [row] = await db.select().from(notePhotos).where(eq(notePhotos.id, id)).limit(1);
if (!row) return null;
return { info: info(row), data: Buffer.from(row.data), targetType: row.serverId !== null ? "server" : "device", targetId: (row.serverId ?? row.deviceId)! };
}
export async function setCaption(id: number, caption: string | null): Promise<PhotoInfo | null> {
const [row] = await db.update(notePhotos).set({ caption: cleanCaption(caption) }).where(eq(notePhotos.id, id)).returning();
return row ? info(row) : null;
}
export async function removePhoto(id: number): Promise<{ targetType: TargetType; targetId: number } | null> {
const [row] = await db.delete(notePhotos).where(eq(notePhotos.id, id)).returning({ serverId: notePhotos.serverId, deviceId: notePhotos.deviceId });
return row ? { targetType: row.serverId !== null ? "server" : "device", targetId: (row.serverId ?? row.deviceId)! } : null;
}
+70 -2
View File
@@ -20,7 +20,7 @@ process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, heartbeats, assets, alertAcks, maintenanceWindows, auditLog } = await import("../src/db/schema.js");
const { users, servers, heartbeats, assets, alertAcks, alertSnoozes, maintenanceWindows, auditLog } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { serversBulkRouter, alertsBulkRouter, inventoryBulkRouter } = await import("../src/routes/bulk.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
@@ -269,8 +269,74 @@ describe("alerts", () => {
assert.equal((await listActiveWindows()).length, 0);
});
const hoursAhead = (h: number) => new Date(Date.now() + h * 3_600_000).toISOString();
test("several are snoozed until the same time; one that has gone is skipped; they are hidden from the live list; undo wakes them", async () => {
await db.delete(alertSnoozes);
const until = hoursAhead(5);
const r = await send("/alerts/bulk", { action: "snooze", alertIds: [...alertIds, "automation:heartbeat:99999"], until });
assert.equal(r.status, 200);
assert.equal(r.body.changed, 3);
assert.deepEqual(r.body.skipped.map((x: { reason: string }) => x.reason), ["it isn't there any more"]);
const rows = await db.select().from(alertSnoozes);
assert.equal(rows.length, 3);
assert.ok(rows.every((x) => x.until === new Date(until).toISOString() && x.snoozedBy === "Olga"));
invalidateAlertsCache();
const report = await getAlerts(true);
assert.ok(alertIds.every((id) => report.alerts.find((a) => a.id === id)?.snoozed), "each shows as snoozed");
assert.ok(r.body.undo.items.every((i: { previous: unknown }) => i.previous === null));
await send("/alerts/bulk", r.body.undo);
assert.equal((await db.select().from(alertSnoozes)).length, 0);
});
test("an earlier snooze is replaced, and undo puts it back exactly — when it ends and who made it", async () => {
await db.delete(alertSnoozes);
await send("/alerts/bulk", { action: "snooze", alertIds: [alertIds[0]], until: hoursAhead(2) });
const [first] = await db.select().from(alertSnoozes);
const over = await send("/alerts/bulk", { action: "snooze", alertIds: [alertIds[0], alertIds[1]], until: hoursAhead(30) });
assert.equal(over.body.changed, 2);
assert.notEqual((await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, alertIds[0])))[0].until, first.until);
await send("/alerts/bulk", over.body.undo);
const rows = await db.select().from(alertSnoozes);
assert.equal(rows.length, 1, "the second alert had none before");
assert.deepEqual([rows[0].alertId, rows[0].until, rows[0].snoozedAt, rows[0].snoozedBy], [first.alertId, first.until, first.snoozedAt, first.snoozedBy]);
});
test("snoozes are woken, those that were not snoozed are skipped, and undo puts them back", async () => {
await db.delete(alertSnoozes);
await send("/alerts/bulk", { action: "snooze", alertIds: [alertIds[0], alertIds[1]], until: hoursAhead(4) });
const r = await send("/alerts/bulk", { action: "unsnooze", alertIds });
assert.equal(r.body.changed, 2);
assert.deepEqual(r.body.skipped.map((x: { reason: string }) => x.reason), ["it wasn't snoozed"]);
assert.equal((await db.select().from(alertSnoozes)).length, 0);
await send("/alerts/bulk", r.body.undo);
assert.equal((await db.select().from(alertSnoozes)).length, 2);
await db.delete(alertSnoozes);
});
test("a time that has passed, is too far ahead or is not a time is refused, and nothing changes", async () => {
await db.delete(alertSnoozes);
for (const until of [hoursAhead(-1), hoursAhead(24 * 91), "tomorrow-ish"]) {
const r = await send("/alerts/bulk", { action: "snooze", alertIds, until });
assert.equal(r.status, 400, until);
assert.ok(typeof r.body.message === "string" && r.body.message.length > 0);
}
assert.equal((await db.select().from(alertSnoozes)).length, 0);
});
test("undoing to a snooze whose time has already passed drops it rather than announcing the alert at once", async () => {
await db.delete(alertSnoozes);
const previous = { conditionKey: "heartbeat-late:1", message: "late", until: hoursAhead(-1), snoozedBy: "Olga", snoozedAt: hoursAhead(-5) };
await send("/alerts/bulk", { action: "snooze", alertIds: [alertIds[0]], until: hoursAhead(3) });
const r = await send("/alerts/bulk", { action: "restore_snoozes", items: [{ alertId: alertIds[0], previous }] });
assert.equal(r.status, 200);
assert.equal((await db.select().from(alertSnoozes)).length, 0);
});
test("an operator is needed; a viewer is not enough", async () => {
who = "viewer";
assert.equal((await send("/alerts/bulk", { action: "snooze", alertIds, until: hoursAhead(3) })).status, 403);
assert.equal((await send("/alerts/bulk", { action: "acknowledge", alertIds })).status, 403);
who = "operator";
});
@@ -330,7 +396,7 @@ describe("what is written down", () => {
test("each bulk action is one audit entry saying what and how many, and the change log reads them as sentences", async () => {
const entries = await db.select().from(auditLog);
const actions = new Set(entries.map((e) => `${e.category}/${e.action}`));
for (const a of ["server/bulk_tag", "maintenance/bulk_start", "maintenance/bulk_undo", "server/bulk_runs_on", "alert/bulk_ack", "alert/bulk_unack", "alert/bulk_undo", "inventory/bulk_change"]) assert.ok(actions.has(a), a);
for (const a of ["server/bulk_tag", "maintenance/bulk_start", "maintenance/bulk_undo", "server/bulk_runs_on", "alert/bulk_ack", "alert/bulk_unack", "alert/bulk_undo", "alert/bulk_snooze", "alert/bulk_unsnooze", "inventory/bulk_change"]) assert.ok(actions.has(a), a);
const tag = entries.find((e) => e.action === "bulk_tag" && JSON.parse(e.detail!).change === "add_tag")!;
assert.equal(JSON.parse(tag.detail!).tag, "Nightly");
assert.equal(JSON.parse(tag.detail!).changed, 3);
@@ -340,6 +406,8 @@ describe("what is written down", () => {
assert.ok(titles.some((t) => /^Tag added to 3 servers: Nightly$/.test(t)), "tag");
assert.ok(titles.some((t) => /^Maintenance started on several at once/.test(t)), "maintenance");
assert.ok(titles.some((t) => /alerts? acknowledged at once$/.test(t)), "ack");
assert.ok(titles.some((t) => /alerts? snoozed at once$/.test(t)), "snooze");
assert.ok(titles.some((t) => /^Snoozing several alerts at once was undone$/.test(t)), "snooze undo");
assert.ok(titles.some((t) => /pieces? of hardware changed at once$/.test(t)), "hardware");
assert.ok(titles.some((t) => /was undone$/.test(t)), "undo");
});
+178
View File
@@ -0,0 +1,178 @@
/**
* Photos on notes: what is accepted (told by the bytes, never by the name or the type claimed), the limits, who may add and who may only look, how a picture is served, and that
* removing the server or the device removes its photos.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-note-photos-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, integrations, networkDevices, notePhotos, auditLog } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { sniffImage, MAX_PHOTO_BYTES, MAX_PHOTOS_PER_TARGET, MAX_CAPTION } = await import("../src/services/notePhotos.js");
const { notePhotosRouter } = await import("../src/routes/notePhotos.js");
const { getChangeLog } = await import("../src/services/changeLog.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const png = (extra = 0) => Buffer.concat([Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), Buffer.alloc(32 + extra, 7)]);
const jpeg = () => Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), Buffer.alloc(32, 3)]);
const gif = () => Buffer.concat([Buffer.from("GIF89a", "latin1"), Buffer.alloc(32, 1)]);
const webp = () => Buffer.concat([Buffer.from("RIFF", "latin1"), Buffer.from([0x20, 0, 0, 0]), Buffer.from("WEBP", "latin1"), Buffer.alloc(24, 2)]);
describe("what counts as a picture", () => {
test("a JPEG, a PNG, a GIF and a WebP are told by how they begin", () => {
assert.equal(sniffImage(jpeg()), "image/jpeg");
assert.equal(sniffImage(png()), "image/png");
assert.equal(sniffImage(gif()), "image/gif");
assert.equal(sniffImage(webp()), "image/webp");
});
test("an SVG, a page, a script, an empty or a truncated file are none of them", () => {
for (const bad of ['<svg xmlns="http://www.w3.org/2000/svg"><script>alert(1)</script></svg>', "<!doctype html><title>x</title>", "#!/bin/sh\nrm -rf /", ""]) assert.equal(sniffImage(Buffer.from(bad)), null, bad.slice(0, 20));
assert.equal(sniffImage(Buffer.from([0x89, 0x50, 0x4e])), null);
assert.equal(sniffImage(Buffer.concat([Buffer.from("RIFF", "latin1"), Buffer.alloc(4), Buffer.from("WAVE", "latin1")])), null, "RIFF, but not WebP");
});
});
describe("photos through the routes", () => {
let app: Server;
let base = "";
let who = "operator";
const ids: Record<string, number> = {};
const send = async (path: string, init: { method?: string; body?: Buffer | string; type?: string } = {}) => {
const res = await fetch(`${base}${path}`, { method: init.method ?? "GET", headers: init.type ? { "content-type": init.type } : undefined, body: init.body });
const text = res.headers.get("content-type")?.includes("json") ? await res.json() : null;
return { status: res.status, res, body: text as any };
};
const upload = (target: string, type: "server" | "device", bytes: Buffer, caption = "", claimed = "image/png") =>
send(`/api/note-photos?targetType=${type}&targetId=${ids[target]}${caption ? `&caption=${encodeURIComponent(caption)}` : ""}`, { method: "POST", body: bytes, type: claimed });
before(async () => {
await db.insert(users).values([
{ oidcSub: "operator", name: "Olga", role: "operator" },
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
]);
ids.rack = (await db.insert(servers).values({ name: "rack-pc", apiTokenHash: "r", apiTokenPrefix: "r" }).returning({ id: servers.id }))[0].id;
ids.other = (await db.insert(servers).values({ name: "other", apiTokenHash: "o", apiTokenPrefix: "o" }).returning({ id: servers.id }))[0].id;
const [unifi] = await db.insert(integrations).values({ type: "unifi", name: "UniFi", baseUrl: "http://10.0.0.1", config: "{}" }).returning({ id: integrations.id });
ids.switch = (await db.insert(networkDevices).values({ integrationId: unifi.id, externalId: "aa:bb:cc:dd:ee:ff", name: "core switch", kind: "switch", firstSeenAt: new Date().toISOString(), updatedAt: new Date().toISOString() }).returning({ id: networkDevices.id }))[0].id;
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/note-photos", notePhotosRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("an operator adds a photo with a caption; it is listed, and served with its own type and a policy that lets it run nothing", async () => {
const r = await upload("rack", "server", png(), " The back of the rack ");
assert.equal(r.status, 201);
assert.deepEqual([r.body.photo.caption, r.body.photo.contentType, r.body.photo.createdBy, r.body.photo.size], ["The back of the rack", "image/png", "Olga", png().length]);
const list = await send(`/api/note-photos?targetType=server&targetId=${ids.rack}`);
assert.deepEqual(list.body.photos.map((p: { id: number }) => p.id), [r.body.photo.id]);
const img = await send(`/api/note-photos/${r.body.photo.id}/image`);
assert.equal(img.status, 200);
assert.equal(img.res.headers.get("content-type"), "image/png");
assert.equal(img.res.headers.get("x-content-type-options"), "nosniff");
assert.match(img.res.headers.get("content-security-policy") ?? "", /sandbox/);
assert.match(img.res.headers.get("cache-control") ?? "", /immutable/);
assert.deepEqual(Buffer.from(await img.res.arrayBuffer()), png(), "exactly the bytes that were sent");
});
test("what the picture is comes from its bytes: a PNG sent as something else is a PNG; a script sent as image/png is refused", async () => {
const ok = await upload("rack", "server", jpeg(), "", "application/octet-stream");
assert.equal(ok.body.photo.contentType, "image/jpeg");
const bad = await upload("rack", "server", Buffer.from('<svg xmlns="http://www.w3.org/2000/svg"><script>alert(1)</script></svg>'), "", "image/png");
assert.deepEqual([bad.status, bad.body.error], [400, "not_an_image"]);
assert.equal((await upload("rack", "server", Buffer.alloc(0))).status, 400);
});
test("a device's photos are its own, and a photo can be of a device as well as of a server", async () => {
const r = await upload("switch", "device", gif(), "Port labels");
assert.equal(r.status, 201);
assert.equal((await send(`/api/note-photos?targetType=device&targetId=${ids.switch}`)).body.photos.length, 1);
assert.equal((await send(`/api/note-photos?targetType=server&targetId=${ids.other}`)).body.photos.length, 0);
});
test("a photo over 4 MB is refused with a plain message; one that is not for anything is not found", async () => {
const big = await upload("other", "server", png(MAX_PHOTO_BYTES));
assert.equal(big.status, 413);
assert.equal(big.body.error, "too_big");
const nowhere = await send(`/api/note-photos?targetType=server&targetId=999999`, { method: "POST", body: png(), type: "image/png" });
assert.equal(nowhere.status, 404);
assert.equal((await send(`/api/note-photos?targetType=thing&targetId=1`, { method: "POST", body: png(), type: "image/png" })).status, 400);
});
test("a thing has at most twelve photos", async () => {
await db.delete(notePhotos).where(eq(notePhotos.serverId, ids.other));
for (let i = 0; i < MAX_PHOTOS_PER_TARGET; i++) assert.equal((await upload("other", "server", webp())).status, 201);
const r = await upload("other", "server", webp());
assert.deepEqual([r.status, r.body.error], [400, "too_many"]);
});
test("a caption is kept short and can be changed or cleared; a photo can be removed, and the change log says so", async () => {
const [photo] = await db.select().from(notePhotos).where(eq(notePhotos.serverId, ids.rack));
const long = await send(`/api/note-photos/${photo.id}`, { method: "PUT", body: JSON.stringify({ caption: "x".repeat(MAX_CAPTION + 50) }), type: "application/json" });
assert.equal(long.body.photo.caption.length, MAX_CAPTION);
const cleared = await send(`/api/note-photos/${photo.id}`, { method: "PUT", body: JSON.stringify({ caption: " " }), type: "application/json" });
assert.equal(cleared.body.photo.caption, null);
assert.equal((await send(`/api/note-photos/999999`, { method: "PUT", body: JSON.stringify({ caption: "x" }), type: "application/json" })).status, 404);
assert.equal((await send(`/api/note-photos/${photo.id}`, { method: "DELETE" })).status, 204);
assert.equal((await send(`/api/note-photos/${photo.id}/image`)).status, 404);
assert.equal((await send(`/api/note-photos/${photo.id}`, { method: "DELETE" })).status, 404);
const actions = (await db.select().from(auditLog)).map((e) => `${e.category}/${e.action}`);
assert.ok(actions.includes("note/photo_add") && actions.includes("note/photo_delete"));
const titles = (await getChangeLog({ days: 1, limit: 200 })).entries.map((e) => e.title);
assert.ok(titles.some((t) => /^A photo was added to a note on rack-pc$/.test(t)), "added");
assert.ok(titles.some((t) => /^A photo was removed from a note on rack-pc$/.test(t)), "removed");
});
test("everyone signed in can look; only an operator can add, caption or remove", async () => {
who = "viewer";
const [photo] = await db.select().from(notePhotos).where(eq(notePhotos.serverId, ids.other));
assert.equal((await send(`/api/note-photos?targetType=server&targetId=${ids.other}`)).status, 200);
assert.equal((await send(`/api/note-photos/${photo.id}/image`)).status, 200);
assert.equal((await upload("other", "server", png())).status, 403);
assert.equal((await send(`/api/note-photos/${photo.id}`, { method: "PUT", body: JSON.stringify({ caption: "x" }), type: "application/json" })).status, 403);
assert.equal((await send(`/api/note-photos/${photo.id}`, { method: "DELETE" })).status, 403);
who = "operator";
});
test("removing the server or the device removes its photos", async () => {
assert.ok((await db.select().from(notePhotos).where(eq(notePhotos.serverId, ids.other))).length > 0);
await db.delete(servers).where(eq(servers.id, ids.other));
assert.equal((await db.select().from(notePhotos).where(eq(notePhotos.serverId, ids.other))).length, 0);
assert.equal((await db.select().from(notePhotos).where(eq(notePhotos.deviceId, ids.switch))).length, 1);
await db.delete(networkDevices).where(eq(networkDevices.id, ids.switch));
assert.equal((await db.select().from(notePhotos).where(eq(notePhotos.deviceId, ids.switch))).length, 0);
});
});
+173
View File
@@ -0,0 +1,173 @@
/**
* The alert rules are judged right after a server reports, not only at the next pass: announced once, recovered when the figure comes back, only for the server that reported,
* leaving everything else in the health state alone, and not run over the top of a pass.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { createServer as createHttpServer, type Server } from "node:http";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-rules-report-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { servers, serverMetrics, alertRules } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { checkRulesFor, runHealthCheck, RULE_CHECK_MIN_GAP_MS } = await import("../src/services/healthMonitor.js");
const { findingKey, computeRuleFindings, listRules } = await import("../src/services/alertRules.js");
const { HEALTH_STATE_FLAG } = await import("../src/services/dependencies.js");
const { getInternalFlag, setInternalFlag, updateSettings } = await import("../src/services/settingsStore.js");
const { agentReportRouter } = await import("../src/routes/agentReport.js");
const { hashToken } = await import("../src/services/tokens.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const MIN = 60_000;
const messages: string[] = [];
let hook: Server;
const ids: Record<string, number> = {};
let ruleId = 0;
const state = async () => JSON.parse((await getInternalFlag(HEALTH_STATE_FLAG)) ?? "{}") as Record<string, { message: string; source: string }>;
const cpu = async (server: string, value: number, agoMin = 1) => db.insert(serverMetrics).values({ serverId: ids[server], metric: "cpu", value, recordedAt: new Date(Date.now() - agoMin * MIN).toISOString() });
const keyFor = (server: string) => findingKey({ ruleId, serverId: ids[server], detail: null });
const settle = () => new Promise((r) => setTimeout(r, 150));
before(async () => {
hook = createHttpServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (c) => chunks.push(c));
req.on("end", () => {
messages.push(String(JSON.parse(Buffer.concat(chunks).toString()).content));
res.end("{}");
});
});
await new Promise<void>((r) => hook.listen(0, "127.0.0.1", r));
await updateSettings({ webhook: { enabled: true, url: `http://127.0.0.1:${(hook.address() as AddressInfo).port}/` } });
for (const name of ["web", "pi"]) {
ids[name] = (await db.insert(servers).values({ name, apiTokenHash: name, apiTokenPrefix: name, lastSeenAt: new Date().toISOString() }).returning({ id: servers.id }))[0].id;
}
ruleId = (await db.insert(alertRules).values({ name: "CPU hot", enabled: true, metric: "cpu", mount: null, operator: "above", threshold: 90, forMinutes: 0, severity: "warning", serverId: null, tag: null, createdBy: null, createdAt: new Date().toISOString() }).returning({ id: alertRules.id }))[0].id;
});
after(() => {
hook.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
describe("rules judged when a server reports", () => {
test("a server that reports a figure past the limit is announced at once, and only that server; what else is in the state is left alone", async () => {
await setInternalFlag(HEALTH_STATE_FLAG, JSON.stringify({ "disk:server:999": { message: "elsewhere: disk is 95% full", source: "server" } }));
await cpu("web", 97);
await cpu("pi", 97); // pi is just as hot, but it has not reported just now
const r = await checkRulesFor(ids.web, Date.now(), 0);
assert.deepEqual(r, { added: 1, resolved: 0 });
const s = await state();
assert.ok(keyFor("web") in s);
assert.equal(keyFor("pi") in s, false, "the other server waits for its own report or the pass");
assert.equal(s["disk:server:999"].message, "elsewhere: disk is 95% full", "what else was known is kept exactly");
await settle();
assert.equal(messages.filter((m) => /web: CPU/i.test(m) && /CPU hot/.test(m)).length, 1, "one message, naming the rule");
});
test("the same figure again is not announced again", async () => {
await cpu("web", 98, 0);
const before = messages.length;
assert.deepEqual(await checkRulesFor(ids.web, Date.now(), 0), { added: 0, resolved: 0 });
await settle();
assert.equal(messages.length, before);
});
test("when the next report brings the figure back under the limit it is recovered", async () => {
await db.delete(serverMetrics).where(eq(serverMetrics.serverId, ids.web));
await cpu("web", 40, 0);
const before = messages.length;
assert.deepEqual(await checkRulesFor(ids.web, Date.now(), 0), { added: 0, resolved: 1 });
assert.equal(keyFor("web") in (await state()), false);
assert.equal((await state())["disk:server:999"].source, "server", "still untouched");
await settle();
assert.ok(messages.length > before, "the recovery is said");
});
test("a server looked at in the last minute is not looked at again; a restart's grace and a server no rule covers cost nothing", async () => {
assert.equal(RULE_CHECK_MIN_GAP_MS, 60_000);
await cpu("pi", 99, 0);
const first = await checkRulesFor(ids.pi, Date.now(), 60_000);
assert.equal(first?.added, 1);
assert.equal(await checkRulesFor(ids.pi, Date.now() + 10_000, 60_000), null, "ten seconds later");
assert.notEqual(await checkRulesFor(ids.pi, Date.now() + 90_000, 60_000), null, "a minute and a half later");
await db.update(alertRules).set({ enabled: false }).where(eq(alertRules.id, ruleId));
const off = await checkRulesFor(ids.pi, Date.now() + 200_000, 0);
assert.equal(off?.resolved, 1, "a rule that was switched off no longer finds it: recovered");
await db.update(alertRules).set({ enabled: true }).where(eq(alertRules.id, ruleId));
});
test("only the one server's history is read: the findings are the same as for all, narrowed to it", async () => {
await db.delete(serverMetrics);
await cpu("web", 95);
await cpu("pi", 95);
const rules = await listRules();
const all = await computeRuleFindings(rules, Date.now());
const one = await computeRuleFindings(rules, Date.now(), ids.web);
assert.deepEqual(all.map((f) => f.serverId).sort(), [ids.web, ids.pi].sort());
assert.deepEqual(one.map((f) => f.serverId), [ids.web]);
});
test("a pass and a check after a report at the same moment do not lose each other's changes", async () => {
await db.delete(serverMetrics);
await setInternalFlag(HEALTH_STATE_FLAG, "{}");
await cpu("web", 97);
await cpu("pi", 97);
await db.update(servers).set({ lastSeenAt: new Date().toISOString() });
await Promise.all([runHealthCheck(Date.now()), checkRulesFor(ids.web, Date.now(), 0)]);
const s = await state();
assert.ok(keyFor("web") in s && keyFor("pi") in s, "both found, whichever went first");
});
});
describe("through the agent's report", () => {
let app: Server;
let base = "";
before(async () => {
await db.insert(servers).values({ name: "api", apiTokenHash: hashToken("tok-api"), apiTokenPrefix: "tok-api", lastSeenAt: new Date().toISOString() });
const a = express();
a.use("/api/agent/report", agentReportRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/agent/report`;
});
after(() => app.close());
test("a report that brings the CPU past a rule's limit raises it before the pass would, and the agent gets its answer first", async () => {
await setInternalFlag(HEALTH_STATE_FLAG, "{}");
const [{ id }] = await db.select({ id: servers.id }).from(servers).where(eq(servers.name, "api"));
const res = await fetch(base, { method: "POST", headers: { "content-type": "application/json", authorization: "Bearer tok-api" }, body: JSON.stringify({ system: { cpu: { load_percent: 99 } }, tasks: [] }) });
assert.equal(res.status, 202);
await new Promise((r) => setTimeout(r, 600));
const s = await state();
assert.ok(findingKey({ ruleId, serverId: id, detail: null }) in s, "judged on the report");
assert.match(s[findingKey({ ruleId, serverId: id, detail: null })].message, /api: .*99%.*[CPU hot]/);
});
test("a report that is refused does not set anything off", async () => {
await setInternalFlag(HEALTH_STATE_FLAG, "{}");
const res = await fetch(base, { method: "POST", headers: { "content-type": "application/json", authorization: "Bearer wrong" }, body: JSON.stringify({ system: { cpu: { load_percent: 99 } }, tasks: [] }) });
assert.equal(res.status, 401);
await new Promise((r) => setTimeout(r, 200));
assert.deepEqual(await state(), {});
});
});