Alert settings keeps Alert rules, Service levels and Alert tickets; administrators get a link to Settings -> Notifications. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
118 lines
6.3 KiB
TypeScript
118 lines
6.3 KiB
TypeScript
/**
|
|
* Event webhooks: tell another system when something happens. Each webhook is a URL with the events it wants; when one happens it is
|
|
* sent a JSON message, signed with the webhook's secret so the receiver can tell it came from here. This is separate from the chat-style
|
|
* webhook under Settings → Notifications, which carries the same sentence a person would read.
|
|
*
|
|
* alert.opened / alert.resolved a problem began or ended — on the same terms as a notification (a problem under maintenance,
|
|
* or folded under another, isn't announced until it stands alone)
|
|
* maintenance.started / .ended a maintenance window began or finished, however it was made
|
|
*
|
|
* Delivery is best-effort and never holds anything up: one try, one more if that fails, then the result is noted on the webhook (shown in
|
|
* Settings) and forgotten.
|
|
*/
|
|
import { createHmac, randomUUID } from "node:crypto";
|
|
import { and, desc, eq, lt, notInArray } from "drizzle-orm";
|
|
import { db } from "../db/client.js";
|
|
import { eventWebhooks, webhookDeliveries, webhookEvents, type WebhookEvent } from "../db/schema.js";
|
|
import { openSecret, sealSecret } from "./sealed.js";
|
|
|
|
export { sealSecret };
|
|
|
|
export type EventWebhookRow = typeof eventWebhooks.$inferSelect;
|
|
|
|
const TIMEOUT_MS = 5000;
|
|
const RETRY_AFTER_MS = 2000;
|
|
|
|
export function parseEvents(stored: string): WebhookEvent[] {
|
|
try {
|
|
const v = JSON.parse(stored);
|
|
return Array.isArray(v) ? v.filter((e): e is WebhookEvent => (webhookEvents as readonly string[]).includes(e)) : [];
|
|
} catch {
|
|
return [];
|
|
}
|
|
}
|
|
|
|
/** The signature a receiver should compute over the exact body it got: HMAC-SHA256 with the secret, as hex, prefixed "sha256=". */
|
|
export function signBody(secret: string, body: string): string {
|
|
return `sha256=${createHmac("sha256", secret).update(body).digest("hex")}`;
|
|
}
|
|
|
|
export interface Delivery {
|
|
ok: boolean;
|
|
/** "200", or what went wrong. */
|
|
result: string;
|
|
}
|
|
|
|
async function post(row: EventWebhookRow, event: string, body: string, deliveryId: string): Promise<Delivery> {
|
|
const secret = openSecret(row.secret);
|
|
const headers: Record<string, string> = {
|
|
"Content-Type": "application/json",
|
|
"User-Agent": "HomelabManager-Webhook",
|
|
"X-HLM-Event": event,
|
|
"X-HLM-Delivery": deliveryId,
|
|
};
|
|
if (secret) headers["X-HLM-Signature"] = signBody(secret, body);
|
|
try {
|
|
const res = await fetch(row.url, { method: "POST", headers, body, signal: AbortSignal.timeout(TIMEOUT_MS), redirect: "manual" });
|
|
return { ok: res.status >= 200 && res.status < 300, result: String(res.status) };
|
|
} catch (err) {
|
|
const name = err instanceof Error ? err.name : "";
|
|
return { ok: false, result: name === "TimeoutError" ? "timed out" : err instanceof Error ? err.message.slice(0, 120) : "failed" };
|
|
}
|
|
}
|
|
|
|
/** How many deliveries are kept for each webhook, and for how long. */
|
|
export const KEEP_DELIVERIES = 100;
|
|
export const KEEP_DELIVERY_DAYS = 14;
|
|
/** A body longer than this is not kept, so it cannot be sent again by hand. */
|
|
export const MAX_STORED_BODY = 16_000;
|
|
|
|
/** One message to one webhook, retried once if it didn't go through. The outcome is written on the webhook and in its delivery log. */
|
|
export async function deliver(row: EventWebhookRow, event: string, data: unknown, options: { retryAfterMs?: number } = {}): Promise<Delivery> {
|
|
const deliveryId = randomUUID();
|
|
const body = JSON.stringify({ event, id: deliveryId, createdAt: new Date().toISOString(), source: "homelab-manager", data });
|
|
return sendAndNote(row, event, body, deliveryId, options, null);
|
|
}
|
|
|
|
/** Sends a message that was sent before, exactly as it was (the same id, so a receiver that already handled it can ignore it). Says how it went, and writes that down as a delivery of its own. */
|
|
export async function redeliver(row: EventWebhookRow, earlier: typeof webhookDeliveries.$inferSelect, options: { retryAfterMs?: number } = {}): Promise<Delivery> {
|
|
return sendAndNote(row, earlier.event, earlier.body, earlier.deliveryId, options, earlier.id);
|
|
}
|
|
|
|
async function sendAndNote(row: EventWebhookRow, event: string, body: string, deliveryId: string, options: { retryAfterMs?: number }, retryOf: number | null): Promise<Delivery> {
|
|
let attempts = 1;
|
|
let outcome = await post(row, event, body, deliveryId);
|
|
if (!outcome.ok) {
|
|
await new Promise((r) => setTimeout(r, options.retryAfterMs ?? RETRY_AFTER_MS));
|
|
outcome = await post(row, event, body, deliveryId);
|
|
attempts = 2;
|
|
}
|
|
const now = new Date();
|
|
try {
|
|
await db.update(eventWebhooks).set({ lastAttemptAt: now.toISOString(), lastResult: outcome.result }).where(eq(eventWebhooks.id, row.id));
|
|
await db.insert(webhookDeliveries).values({ webhookId: row.id, event, deliveryId, attempts, ok: outcome.ok, result: outcome.result, body: body.slice(0, MAX_STORED_BODY + 1), retryOf, createdAt: now.toISOString() });
|
|
await pruneDeliveries(row.id, now);
|
|
} catch (err) {
|
|
console.error("[webhooks] couldn't note the result:", err);
|
|
}
|
|
return outcome;
|
|
}
|
|
|
|
/** Keeps the last hundred deliveries of a webhook, and none older than two weeks. */
|
|
export async function pruneDeliveries(webhookId: number, now: Date = new Date()): Promise<void> {
|
|
const keep = await db.select({ id: webhookDeliveries.id }).from(webhookDeliveries).where(eq(webhookDeliveries.webhookId, webhookId)).orderBy(desc(webhookDeliveries.createdAt), desc(webhookDeliveries.id)).limit(KEEP_DELIVERIES);
|
|
if (keep.length === KEEP_DELIVERIES) await db.delete(webhookDeliveries).where(and(eq(webhookDeliveries.webhookId, webhookId), notInArray(webhookDeliveries.id, keep.map((k) => k.id))));
|
|
await db.delete(webhookDeliveries).where(and(eq(webhookDeliveries.webhookId, webhookId), lt(webhookDeliveries.createdAt, new Date(now.getTime() - KEEP_DELIVERY_DAYS * 86_400_000).toISOString())));
|
|
}
|
|
|
|
/**
|
|
* Sends an event to every enabled webhook that wants it. Returns at once — delivery carries on in the background and can never delay or
|
|
* fail what caused the event.
|
|
*/
|
|
export function emitEvent(event: WebhookEvent, data: unknown): void {
|
|
void (async () => {
|
|
const rows = await db.select().from(eventWebhooks).where(eq(eventWebhooks.enabled, true));
|
|
await Promise.all(rows.filter((r) => parseEvents(r.events).includes(event)).map((r) => deliver(r, event, data)));
|
|
})().catch((err) => console.error("[webhooks] couldn't send an event:", err));
|
|
}
|