Bulk snooze, alert rules judged on report, photos on notes
Bulk snooze: Snooze... and Wake on the ticked alerts, with undo that puts back each alert's earlier snooze exactly. Alert rules are judged for a server right after its agent reports, instead of at the next 15-minute pass; only that server's rule findings change, and the check is serialized with the pass so neither overwrites the other's state. Photos on server and device notes: up to 12 each, shrunk in the browser, accepted by their own bytes (JPEG/PNG/GIF/WebP, 4 MB), kept in the database and removed with the server or device. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
251a4a9d73
commit
a16996d975
27 files changed
+8442
-54
No files matched your search
@@ -7,7 +7,7 @@
|
||||
* past its limit on every sample for that long. A disk's temperature is only known as of its last report, so it is judged right now. A server that is not reporting is not
|
||||
* judged — its own offline alert says that.
|
||||
*/
|
||||
import { eq, gte } from "drizzle-orm";
|
||||
import { and, eq, gte } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { alertRules, serverMetrics, servers, type AlertRuleMetric } from "../db/schema.js";
|
||||
import { parseHostHealth } from "./hostHealth.js";
|
||||
@@ -128,11 +128,12 @@ export async function listRules(): Promise<RuleRow[]> {
|
||||
return (await db.select().from(alertRules)).sort((a, b) => a.name.localeCompare(b.name) || a.id - b.id);
|
||||
}
|
||||
|
||||
/** What the given rules find right now, from the stored history. Reads nothing when no rule is on. */
|
||||
export async function computeRuleFindings(rules: RuleRow[], now: number = Date.now()): Promise<RuleFinding[]> {
|
||||
/** What the given rules find right now, from the stored history — on one server only when `onlyServerId` is given, which reads only that server's history. Reads nothing when no rule is on. */
|
||||
export async function computeRuleFindings(rules: RuleRow[], now: number = Date.now(), onlyServerId?: number): Promise<RuleFinding[]> {
|
||||
const on = rules.filter((r) => r.enabled);
|
||||
if (on.length === 0) return [];
|
||||
const rows = await db.select({ id: servers.id, name: servers.name, tags: servers.tags, hostHealth: servers.hostHealth }).from(servers);
|
||||
const everything = db.select({ id: servers.id, name: servers.name, tags: servers.tags, hostHealth: servers.hostHealth }).from(servers);
|
||||
const rows = await (onlyServerId === undefined ? everything : everything.where(eq(servers.id, onlyServerId)));
|
||||
const serverList: RuleServer[] = rows.map((r) => ({
|
||||
id: r.id,
|
||||
name: r.name,
|
||||
@@ -143,7 +144,8 @@ export async function computeRuleFindings(rules: RuleRow[], now: number = Date.n
|
||||
if (on.some((r) => r.metric !== "disk_temp")) {
|
||||
const longest = Math.max(...on.filter((r) => r.metric !== "disk_temp").map((r) => r.forMinutes));
|
||||
const since = new Date(now - (longest * 60_000 + 3 * 3600_000)).toISOString();
|
||||
for (const r of await db.select().from(serverMetrics).where(gte(serverMetrics.recordedAt, since))) {
|
||||
const recent = gte(serverMetrics.recordedAt, since);
|
||||
for (const r of await db.select().from(serverMetrics).where(onlyServerId === undefined ? recent : and(recent, eq(serverMetrics.serverId, onlyServerId)))) {
|
||||
if (r.metric !== "cpu" && r.metric !== "mem" && !r.metric.startsWith("disk:")) continue;
|
||||
const k = sampleKey(r.serverId, r.metric);
|
||||
const list = samples.get(k) ?? [];
|
||||
|
||||
@@ -60,6 +60,31 @@ export async function setSnooze(input: { alertId: string; conditionKey: string;
|
||||
return { until: values.until, snoozedBy: values.snoozedBy, snoozedAt: values.snoozedAt };
|
||||
}
|
||||
|
||||
/** A snooze as it is stored, whole — what an undo puts back. */
|
||||
export interface SnoozeRecord {
|
||||
conditionKey: string;
|
||||
message: string;
|
||||
until: string;
|
||||
snoozedBy: string;
|
||||
snoozedAt: string;
|
||||
}
|
||||
|
||||
/** The snooze on an alert as it stands, expired or not; null when it has none. */
|
||||
export async function readSnooze(alertId: string): Promise<SnoozeRecord | null> {
|
||||
const [row] = await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, alertId)).limit(1);
|
||||
return row ? { conditionKey: row.conditionKey, message: row.message, until: row.until, snoozedBy: row.snoozedBy, snoozedAt: row.snoozedAt } : null;
|
||||
}
|
||||
|
||||
/** Puts a snooze back as it was (an undo). One whose time has already passed is not worth restoring: it would only announce the alert again at once, so it is dropped. */
|
||||
export async function restoreSnooze(alertId: string, record: SnoozeRecord, now: Date = new Date()): Promise<void> {
|
||||
if (Date.parse(record.until) <= now.getTime()) {
|
||||
await clearSnooze(alertId);
|
||||
return;
|
||||
}
|
||||
const values = { ...record, lastSeenAt: now.toISOString() };
|
||||
await db.insert(alertSnoozes).values({ alertId, ...values }).onConflictDoUpdate({ target: alertSnoozes.alertId, set: values });
|
||||
}
|
||||
|
||||
export async function clearSnooze(alertId: string): Promise<{ message: string } | null> {
|
||||
const [row] = await db.select().from(alertSnoozes).where(eq(alertSnoozes.alertId, alertId)).limit(1);
|
||||
if (!row) return null;
|
||||
|
||||
@@ -8,6 +8,8 @@ import { db } from "../db/client.js";
|
||||
import { assets, maintenanceWindows, servers } from "../db/schema.js";
|
||||
import { invalidateAlertsCache, getAlerts } from "./alerts.js";
|
||||
import { clearAck, restoreAck, setAck, type AckInfo, type SetAckInput } from "./alertAcks.js";
|
||||
import { clearSnooze, readSnooze, restoreSnooze, setSnooze, type SnoozeRecord } from "./alertSnoozes.js";
|
||||
import { keyOfAlertId } from "./flapGuard.js";
|
||||
import { impactOf } from "./dependencies.js";
|
||||
import { isRealDate } from "./inventory.js";
|
||||
import { listActiveWindows, startOrExtendWindow } from "./maintenance.js";
|
||||
@@ -232,6 +234,57 @@ export async function restoreAcks(items: AcksUndo["items"]): Promise<number> {
|
||||
return items.length;
|
||||
}
|
||||
|
||||
export interface SnoozesUndo {
|
||||
action: "restore_snoozes";
|
||||
items: { alertId: string; previous: SnoozeRecord | null }[];
|
||||
}
|
||||
|
||||
/** Snoozes each alert until the same time (an earlier snooze is replaced). Alerts that are no longer there are skipped. A time that is not allowed refuses the whole thing, before anything is changed. */
|
||||
export async function snoozeAlerts(alertIds: string[], until: Date, actor: string, now: Date = new Date()): Promise<BulkResult<SnoozesUndo>> {
|
||||
const present = new Map((await getAlerts(false)).alerts.map((a) => [a.id, a]));
|
||||
const skipped: Skipped[] = [];
|
||||
const items: SnoozesUndo["items"] = [];
|
||||
for (const alertId of unique(alertIds)) {
|
||||
const alert = present.get(alertId);
|
||||
if (!alert) {
|
||||
skipped.push({ id: alertId, name: alertId, reason: "it isn't there any more" });
|
||||
continue;
|
||||
}
|
||||
const previous = await readSnooze(alertId);
|
||||
await setSnooze({ alertId, conditionKey: keyOfAlertId(alert.id, alert.category), message: alert.message, until, actor }, now);
|
||||
items.push({ alertId, previous });
|
||||
}
|
||||
if (items.length > 0) invalidateAlertsCache();
|
||||
return { changed: items.length, skipped, undo: items.length > 0 ? { action: "restore_snoozes", items } : null };
|
||||
}
|
||||
|
||||
/** Wakes each alert that is snoozed. */
|
||||
export async function unsnoozeAlerts(alertIds: string[]): Promise<BulkResult<SnoozesUndo>> {
|
||||
const skipped: Skipped[] = [];
|
||||
const items: SnoozesUndo["items"] = [];
|
||||
for (const alertId of unique(alertIds)) {
|
||||
const previous = await readSnooze(alertId);
|
||||
if (!previous) {
|
||||
skipped.push({ id: alertId, name: alertId, reason: "it wasn't snoozed" });
|
||||
continue;
|
||||
}
|
||||
await clearSnooze(alertId);
|
||||
items.push({ alertId, previous });
|
||||
}
|
||||
if (items.length > 0) invalidateAlertsCache();
|
||||
return { changed: items.length, skipped, undo: items.length > 0 ? { action: "restore_snoozes", items } : null };
|
||||
}
|
||||
|
||||
/** Puts each alert's snooze back as it was (or takes it off, if it had none). */
|
||||
export async function restoreSnoozes(items: SnoozesUndo["items"], now: Date = new Date()): Promise<number> {
|
||||
for (const i of items) {
|
||||
if (i.previous === null) await clearSnooze(i.alertId);
|
||||
else await restoreSnooze(i.alertId, i.previous, now);
|
||||
}
|
||||
if (items.length > 0) invalidateAlertsCache();
|
||||
return items.length;
|
||||
}
|
||||
|
||||
/** The maintenance targets of the alerts given (those with none can't be silenced). */
|
||||
export async function targetsOfAlerts(alertIds: string[]): Promise<{ targets: Target[]; skipped: Skipped[] }> {
|
||||
const present = new Map((await getAlerts(false)).alerts.map((a) => [a.id, a]));
|
||||
|
||||
@@ -178,8 +178,16 @@ export function describeChange(category: string, action: string, detail: Record<
|
||||
return { title: `${typeof detail.changed === "number" ? detail.changed : "Several"} alert${detail.changed === 1 ? "" : "s"} acknowledged at once`, tone: "neutral", detail: typeof detail.note === "string" && detail.note ? detail.note : null };
|
||||
case "alert/bulk_unack":
|
||||
return { title: `Acknowledgement taken off ${typeof detail.changed === "number" ? detail.changed : "several"} alert${detail.changed === 1 ? "" : "s"}`, tone: "neutral", detail: null };
|
||||
case "note/photo_add":
|
||||
return { title: `A photo was added to a note${name ? ` on ${name}` : ""}`, tone: "neutral", detail: str(detail.caption) };
|
||||
case "note/photo_delete":
|
||||
return { title: `A photo was removed from a note${name ? ` on ${name}` : ""}`, tone: "neutral", detail: null };
|
||||
case "alert/bulk_undo":
|
||||
return { title: "Acknowledging several alerts at once was undone", tone: "neutral", detail: null };
|
||||
return { title: detail.what === "snooze" ? "Snoozing several alerts at once was undone" : "Acknowledging several alerts at once was undone", tone: "neutral", detail: null };
|
||||
case "alert/bulk_snooze":
|
||||
return { title: `${typeof detail.changed === "number" ? detail.changed : "Several"} alert${detail.changed === 1 ? "" : "s"} snoozed at once`, tone: "neutral", detail: str(detail.until) ? `until ${str(detail.until)!.slice(0, 16).replace("T", " ")} UTC` : null };
|
||||
case "alert/bulk_unsnooze":
|
||||
return { title: `${typeof detail.changed === "number" ? detail.changed : "Several"} snoozed alert${detail.changed === 1 ? "" : "s"} woken at once`, tone: "neutral", detail: null };
|
||||
case "inventory/bulk_change":
|
||||
return { title: detail.change === "undo" ? "A change to several pieces of hardware was undone" : `${typeof detail.changed === "number" ? detail.changed : "Several"} piece${detail.changed === 1 ? "" : "s"} of hardware changed at once`, tone: "neutral", detail: genericSummary(detail, ["ids", "changed", "change"]) };
|
||||
case "webhook/retry":
|
||||
|
||||
@@ -689,7 +689,49 @@ export function alsoList(names: string[]): string {
|
||||
return rest > 0 ? `${shown.join(", ")} and ${rest} more` : head;
|
||||
}
|
||||
|
||||
export async function runHealthCheck(now: number = Date.now()): Promise<{ added: number; resolved: number }> {
|
||||
// One look at a time: the pass and the check that follows an agent's report both read the state of what is known, change it and write it back.
|
||||
let running: Promise<unknown> = Promise.resolve();
|
||||
function serialized<T>(work: () => Promise<T>): Promise<T> {
|
||||
const result = running.then(work, work);
|
||||
running = result.catch(() => undefined);
|
||||
return result;
|
||||
}
|
||||
|
||||
export function runHealthCheck(now: number = Date.now()): Promise<{ added: number; resolved: number }> {
|
||||
return serialized(() => runHealthCheckNow(now));
|
||||
}
|
||||
|
||||
/** A report from the same server is not looked at again within this long. */
|
||||
export const RULE_CHECK_MIN_GAP_MS = 60_000;
|
||||
const lastRuleCheck = new Map<number, number>();
|
||||
|
||||
/**
|
||||
* The server has just reported, so its figures are as new as they will be: judge the administrator's alert rules for it now, instead of at the next pass (up to fifteen minutes
|
||||
* later). Only this server's rule findings are looked at and only they change; everything else — the built-in checks, the other servers, the integrations — waits for the pass, so
|
||||
* what a pass would say is unchanged. Announced and recovered like the pass does. Not done during the start-up grace, nor for a server looked at in the last minute (`minGapMs`).
|
||||
*/
|
||||
export function checkRulesFor(serverId: number, now: number = Date.now(), minGapMs: number = RULE_CHECK_MIN_GAP_MS): Promise<{ added: number; resolved: number } | null> {
|
||||
if (now - PROCESS_START < STARTUP_GRACE_MS) return Promise.resolve(null);
|
||||
const last = lastRuleCheck.get(serverId);
|
||||
if (last !== undefined && now - last < minGapMs) return Promise.resolve(null);
|
||||
lastRuleCheck.set(serverId, now);
|
||||
return serialized(async () => {
|
||||
const findings = await computeRuleFindings(await listRules(), now, serverId);
|
||||
const mine = new RegExp(`^rule:\\d+:server:${serverId}(:|$)`);
|
||||
const previous = await loadState();
|
||||
const before: ActiveState = Object.fromEntries(Object.entries(previous).filter(([k]) => mine.test(k)));
|
||||
if (findings.length === 0 && Object.keys(before).length === 0) return { added: 0, resolved: 0 };
|
||||
const current: HealthCondition[] = findings.map((f) => ({ key: findingKey(f), source: "server:rules", message: f.message, severity: f.severity }));
|
||||
const { added, resolved, next, newlyDown, ended } = diffConditions(before, current, new Set(), await activeSubjects(new Date(now)));
|
||||
await setInternalFlag(STATE_FLAG, JSON.stringify({ ...Object.fromEntries(Object.entries(previous).filter(([k]) => !mine.test(k))), ...next }));
|
||||
await recordServerTransitions(newlyDown, ended);
|
||||
if (added.length > 0) await notifyHealthIssues(added);
|
||||
if (resolved.length > 0) await notifyHealthRecovered(resolved);
|
||||
return { added: added.length, resolved: resolved.length };
|
||||
});
|
||||
}
|
||||
|
||||
async function runHealthCheckNow(now: number = Date.now()): Promise<{ added: number; resolved: number }> {
|
||||
const { healthChecks } = await getSettings();
|
||||
const { snapshot, held } = await collectSnapshot();
|
||||
// What the pools read this time go into their history (one reading an hour at most), so they can be forecast.
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
/**
|
||||
* Photos on the notes of servers and network devices. The browser makes each one small before it is sent (a phone's photo is several megabytes; a screen's worth is a few hundred
|
||||
* kilobytes), and this is the second line: a photo over 4 MB is refused, so is anything that is not really a JPEG, PNG, GIF or WebP — told by the first bytes, never by the name or
|
||||
* the type the upload claims (an SVG, which can carry script, is not accepted) — and a thing has at most 12.
|
||||
*/
|
||||
import { count, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { notePhotos } from "../db/schema.js";
|
||||
import { targetName, type TargetType } from "./todos.js";
|
||||
|
||||
export const MAX_PHOTO_BYTES = 4 * 1024 * 1024;
|
||||
export const MAX_PHOTOS_PER_TARGET = 12;
|
||||
export const MAX_CAPTION = 120;
|
||||
|
||||
export class PhotoError extends Error {
|
||||
constructor(
|
||||
readonly status: number,
|
||||
readonly code: string,
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
export interface PhotoInfo {
|
||||
id: number;
|
||||
caption: string | null;
|
||||
contentType: string;
|
||||
size: number;
|
||||
createdBy: string | null;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
/** Pure: what the bytes are, from how they begin; null when they are none of the four. */
|
||||
export function sniffImage(b: Buffer): "image/jpeg" | "image/png" | "image/gif" | "image/webp" | null {
|
||||
if (b.length >= 3 && b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff) return "image/jpeg";
|
||||
if (b.length >= 8 && b.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) return "image/png";
|
||||
if (b.length >= 6 && (b.subarray(0, 6).toString("latin1") === "GIF87a" || b.subarray(0, 6).toString("latin1") === "GIF89a")) return "image/gif";
|
||||
if (b.length >= 12 && b.subarray(0, 4).toString("latin1") === "RIFF" && b.subarray(8, 12).toString("latin1") === "WEBP") return "image/webp";
|
||||
return null;
|
||||
}
|
||||
|
||||
const column = (type: TargetType) => (type === "server" ? notePhotos.serverId : notePhotos.deviceId);
|
||||
const cleanCaption = (c: string | null | undefined) => {
|
||||
const t = (c ?? "").replace(/\s+/g, " ").trim();
|
||||
return t === "" ? null : t.slice(0, MAX_CAPTION);
|
||||
};
|
||||
const info = (r: Pick<typeof notePhotos.$inferSelect, "id" | "caption" | "contentType" | "size" | "createdBy" | "createdAt">): PhotoInfo => ({ id: r.id, caption: r.caption, contentType: r.contentType, size: r.size, createdBy: r.createdBy, createdAt: r.createdAt });
|
||||
|
||||
/** The photos of a server or a device, oldest first (the order they were taken in). */
|
||||
export async function listPhotos(type: TargetType, id: number): Promise<PhotoInfo[]> {
|
||||
const rows = await db
|
||||
.select({ id: notePhotos.id, caption: notePhotos.caption, contentType: notePhotos.contentType, size: notePhotos.size, createdBy: notePhotos.createdBy, createdAt: notePhotos.createdAt })
|
||||
.from(notePhotos)
|
||||
.where(eq(column(type), id))
|
||||
.orderBy(notePhotos.id);
|
||||
return rows.map(info);
|
||||
}
|
||||
|
||||
export async function addPhoto(type: TargetType, id: number, bytes: Buffer, caption: string | null, actor: string, now: Date = new Date()): Promise<PhotoInfo> {
|
||||
const targetLabel = await targetName(type, id);
|
||||
if (!targetLabel) throw new PhotoError(404, "unknown_target", "That server or network device is not there.");
|
||||
if (bytes.length === 0) throw new PhotoError(400, "empty", "There is nothing in that file.");
|
||||
if (bytes.length > MAX_PHOTO_BYTES) throw new PhotoError(413, "too_big", `A photo can be at most ${MAX_PHOTO_BYTES / 1024 / 1024} MB.`);
|
||||
const contentType = sniffImage(bytes);
|
||||
if (!contentType) throw new PhotoError(400, "not_an_image", "That is not a JPEG, PNG, GIF or WebP picture.");
|
||||
const [{ n }] = await db.select({ n: count() }).from(notePhotos).where(eq(column(type), id));
|
||||
if (n >= MAX_PHOTOS_PER_TARGET) throw new PhotoError(400, "too_many", `There can be at most ${MAX_PHOTOS_PER_TARGET} photos on one note. Remove one first.`);
|
||||
const [row] = await db
|
||||
.insert(notePhotos)
|
||||
.values({ serverId: type === "server" ? id : null, deviceId: type === "device" ? id : null, contentType, data: bytes, size: bytes.length, caption: cleanCaption(caption), createdBy: actor, createdAt: now.toISOString() })
|
||||
.returning();
|
||||
return info(row);
|
||||
}
|
||||
|
||||
/** One photo with its bytes, and whose it is; null when there is none. */
|
||||
export async function getPhoto(id: number): Promise<{ info: PhotoInfo; data: Buffer; targetType: TargetType; targetId: number } | null> {
|
||||
const [row] = await db.select().from(notePhotos).where(eq(notePhotos.id, id)).limit(1);
|
||||
if (!row) return null;
|
||||
return { info: info(row), data: Buffer.from(row.data), targetType: row.serverId !== null ? "server" : "device", targetId: (row.serverId ?? row.deviceId)! };
|
||||
}
|
||||
|
||||
export async function setCaption(id: number, caption: string | null): Promise<PhotoInfo | null> {
|
||||
const [row] = await db.update(notePhotos).set({ caption: cleanCaption(caption) }).where(eq(notePhotos.id, id)).returning();
|
||||
return row ? info(row) : null;
|
||||
}
|
||||
|
||||
export async function removePhoto(id: number): Promise<{ targetType: TargetType; targetId: number } | null> {
|
||||
const [row] = await db.delete(notePhotos).where(eq(notePhotos.id, id)).returning({ serverId: notePhotos.serverId, deviceId: notePhotos.deviceId });
|
||||
return row ? { targetType: row.serverId !== null ? "server" : "device", targetId: (row.serverId ?? row.deviceId)! } : null;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user