How big the app's data is, and a webhook delivery log with Send again
Settings -> Data -> Storage measures the database, backups and sessions and every table (size, rows, growth, how long it is kept, how far back it reaches), and flags histories that are not being trimmed or have no limit. Event webhooks now keep a delivery log (last 100 per webhook, 14 days): what was sent, how it went, the message itself, and Send again, which resends it exactly as it was (same id and signature). Resends are audited and in the Change log. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
aebf3c6ce1
commit
e114567d64
22 files changed
+6833
-19
No files matched your search
@@ -0,0 +1,15 @@
|
||||
CREATE TABLE `webhook_deliveries` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`webhook_id` integer NOT NULL,
|
||||
`event` text NOT NULL,
|
||||
`delivery_id` text NOT NULL,
|
||||
`attempts` integer DEFAULT 1 NOT NULL,
|
||||
`ok` integer NOT NULL,
|
||||
`result` text NOT NULL,
|
||||
`body` text NOT NULL,
|
||||
`retry_of` integer,
|
||||
`created_at` text NOT NULL,
|
||||
FOREIGN KEY (`webhook_id`) REFERENCES `event_webhooks`(`id`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX `webhook_deliveries_webhook_idx` ON `webhook_deliveries` (`webhook_id`,`created_at`);
|
||||
File diff suppressed because it is too large.
Load diff
@@ -365,6 +365,13 @@
|
||||
"when": 1792531006435,
|
||||
"tag": "0051_next_nitro",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 52,
|
||||
"version": "6",
|
||||
"when": 1792608282223,
|
||||
"tag": "0052_brainy_steve_rogers",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -52,6 +52,7 @@ import { notesRouter, todosRouter } from "./routes/todos.js";
|
||||
import { npmRouter } from "./routes/npm.js";
|
||||
import { alertRulesRouter } from "./routes/alertRules.js";
|
||||
import { storageRouter } from "./routes/storage.js";
|
||||
import { dataUsageRouter } from "./routes/dataUsage.js";
|
||||
import { alertActionsRouter } from "./routes/alertActions.js";
|
||||
import { homeAssistantRouter } from "./routes/homeAssistant.js";
|
||||
import { publicApiRouter } from "./routes/publicApi.js";
|
||||
@@ -181,6 +182,7 @@ export function createApp() {
|
||||
app.use("/api/npm", npmRouter);
|
||||
app.use("/api/alert-rules", alertRulesRouter);
|
||||
app.use("/api/storage", storageRouter);
|
||||
app.use("/api/data-usage", dataUsageRouter);
|
||||
app.use("/api/home-assistant", homeAssistantRouter);
|
||||
app.use("/api/todos", todosRouter);
|
||||
app.use("/api/notes", notesRouter);
|
||||
|
||||
@@ -1294,3 +1294,25 @@ export const alertSnoozes = sqliteTable(
|
||||
},
|
||||
(t) => [uniqueIndex("alert_snoozes_alert_idx").on(t.alertId)],
|
||||
);
|
||||
|
||||
// ─── What each event webhook was sent, and how it went ──────────────────────
|
||||
|
||||
// One row per message to one webhook (the try and its one retry together), with the exact body so it can be sent again by hand. The last 100 per webhook and no more than two weeks.
|
||||
export const webhookDeliveries = sqliteTable(
|
||||
"webhook_deliveries",
|
||||
{
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
webhookId: integer("webhook_id")
|
||||
.notNull()
|
||||
.references(() => eventWebhooks.id, { onDelete: "cascade" }),
|
||||
event: text("event").notNull(),
|
||||
deliveryId: text("delivery_id").notNull(), // the message's id, which a receiver can use to ignore a repeat
|
||||
attempts: integer("attempts").notNull().default(1),
|
||||
ok: integer("ok", { mode: "boolean" }).notNull(),
|
||||
result: text("result").notNull(), // "200", or what went wrong
|
||||
body: text("body").notNull(), // the exact JSON that was sent (what the signature covers)
|
||||
retryOf: integer("retry_of"), // the delivery this one repeats, when somebody pressed Retry
|
||||
createdAt: text("created_at").notNull(), // ISO
|
||||
},
|
||||
(t) => [index("webhook_deliveries_webhook_idx").on(t.webhookId, t.createdAt)],
|
||||
);
|
||||
@@ -0,0 +1,12 @@
|
||||
import { Router } from "express";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { getDataUsage } from "../services/dataUsage.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
/** How big the app's own data is and whether it is being trimmed: for administrators (it names every table and how much each holds). */
|
||||
export const dataUsageRouter = Router();
|
||||
dataUsageRouter.use(requireAuth, requireRole("admin"));
|
||||
|
||||
dataUsageRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
res.json(await getDataUsage());
|
||||
}));
|
||||
@@ -1,11 +1,11 @@
|
||||
import { Router } from "express";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { and, desc, eq } from "drizzle-orm";
|
||||
import { z } from "zod";
|
||||
import { db } from "../db/client.js";
|
||||
import { eventWebhooks, webhookEvents } from "../db/schema.js";
|
||||
import { eventWebhooks, webhookDeliveries, webhookEvents } from "../db/schema.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { deliver, parseEvents, sealSecret, type EventWebhookRow } from "../services/eventWebhooks.js";
|
||||
import { deliver, MAX_STORED_BODY, parseEvents, redeliver, sealSecret, type EventWebhookRow } from "../services/eventWebhooks.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const eventWebhooksRouter = Router();
|
||||
@@ -86,6 +86,34 @@ eventWebhooksRouter.delete("/:id", asyncHandler(async (req, res) => {
|
||||
res.status(204).end();
|
||||
}));
|
||||
|
||||
// What was sent to a webhook lately and how each went, with the message itself, newest first.
|
||||
eventWebhooksRouter.get("/:id/deliveries", asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [row] = await db.select({ id: eventWebhooks.id }).from(eventWebhooks).where(eq(eventWebhooks.id, id)).limit(1);
|
||||
if (!row) return res.status(404).json({ error: "not_found" });
|
||||
const asked = Number(req.query.limit);
|
||||
const limit = Number.isFinite(asked) ? Math.min(100, Math.max(1, Math.round(asked))) : 50;
|
||||
const rows = await db.select().from(webhookDeliveries).where(eq(webhookDeliveries.webhookId, id)).orderBy(desc(webhookDeliveries.createdAt), desc(webhookDeliveries.id)).limit(limit);
|
||||
res.json({
|
||||
deliveries: rows.map((r) => ({ id: r.id, event: r.event, deliveryId: r.deliveryId, attempts: r.attempts, ok: r.ok, result: r.result, body: r.body.length > MAX_STORED_BODY ? null : r.body, retryable: r.body.length <= MAX_STORED_BODY, retryOf: r.retryOf, createdAt: r.createdAt })),
|
||||
});
|
||||
}));
|
||||
|
||||
// Sends a delivery again exactly as it was, with the same id — for when the receiver was down, or the fault was at its end.
|
||||
eventWebhooksRouter.post("/:id/deliveries/:deliveryId/retry", asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
const deliveryRowId = idOf(req.params.deliveryId);
|
||||
if (!Number.isInteger(id) || !Number.isInteger(deliveryRowId)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [hook] = await db.select().from(eventWebhooks).where(eq(eventWebhooks.id, id)).limit(1);
|
||||
const [earlier] = await db.select().from(webhookDeliveries).where(and(eq(webhookDeliveries.id, deliveryRowId), eq(webhookDeliveries.webhookId, id))).limit(1);
|
||||
if (!hook || !earlier) return res.status(404).json({ error: "not_found" });
|
||||
if (earlier.body.length > MAX_STORED_BODY) return res.status(409).json({ error: "too_large", message: "That message was too large to keep, so it cannot be sent again." });
|
||||
const outcome = await redeliver(hook, earlier);
|
||||
await recordAudit({ actor: req.currentUser!, category: "webhook", action: "retry", targetType: "event_webhook", targetId: id, detail: { label: hook.label, event: earlier.event, ok: outcome.ok } });
|
||||
res.json(outcome);
|
||||
}));
|
||||
|
||||
// Sends one test message, signed like the real ones, and says how it went — whether or not the webhook is switched on or wants the event.
|
||||
eventWebhooksRouter.post("/:id/test", asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
|
||||
@@ -179,6 +179,8 @@ export function describeChange(category: string, action: string, detail: Record<
|
||||
return { title: "Acknowledging several alerts at once was undone", tone: "neutral", detail: null };
|
||||
case "inventory/bulk_change":
|
||||
return { title: detail.change === "undo" ? "A change to several pieces of hardware was undone" : `${typeof detail.changed === "number" ? detail.changed : "Several"} piece${detail.changed === 1 ? "" : "s"} of hardware changed at once`, tone: "neutral", detail: genericSummary(detail, ["ids", "changed", "change"]) };
|
||||
case "webhook/retry":
|
||||
return { title: `A webhook message was sent again${name ? `: ${name}` : str(detail.label) ? `: ${str(detail.label)}` : ""}`, tone: detail.ok === false ? "bad" : "neutral", detail: str(detail.event) };
|
||||
case "alert_rule/create":
|
||||
return { title: `Alert rule added${name ? `: ${name}` : ""}`, tone: "neutral", detail: genericSummary(detail) };
|
||||
case "alert_rule/update":
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
/**
|
||||
* How big the app's own data is, and whether what is meant to be trimmed is being trimmed: the database file and what it holds table by table (rows and bytes), the
|
||||
* backups and the sessions beside it, and — for each table that keeps a history — how far back it reaches, how fast it is growing and whether that matches how long
|
||||
* it is meant to be kept. A history that holds far more than its limit means its trimming is not running; one that has no limit at all (the logs, until purging is
|
||||
* switched on) is said to grow without end.
|
||||
*
|
||||
* Nothing here changes anything. It only reads, and the table names it puts in a query come from a fixed list, never from a request.
|
||||
*/
|
||||
import { existsSync, readdirSync, statSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { client, dbPath } from "../db/client.js";
|
||||
import { env } from "../env.js";
|
||||
import { resolveDataPath } from "../paths.js";
|
||||
import { backupDir } from "./selfBackup.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
|
||||
const DAY_MS = 86_400_000;
|
||||
|
||||
/** How a table with a history is trimmed. `log` follows the log-retention setting. */
|
||||
interface HistoryMeta {
|
||||
/** The column that says when a row is from. */
|
||||
time: string;
|
||||
keep: number | "log" | null;
|
||||
/** Narrows to the rows the limit is about (an incident is only trimmed once resolved; the daily rows of the summary table). */
|
||||
where?: string;
|
||||
/** Said under the table's name. */
|
||||
note?: string;
|
||||
}
|
||||
|
||||
export const HISTORY: Record<string, HistoryMeta> = {
|
||||
server_metrics: { time: "recorded_at", keep: 35, note: "a sample of each server's load, memory and disks every 10–15 minutes" },
|
||||
server_metrics_rollup: { time: "at", keep: 800, where: "grain = 'd'", note: "hourly figures are kept 120 days, daily ones about 800" },
|
||||
endpoint_results: { time: "at", keep: 14, note: "every answer of every endpoint check" },
|
||||
endpoint_hourly: { time: "hour", keep: 120 },
|
||||
endpoint_daily: { time: "day", keep: 800 },
|
||||
heartbeat_pings: { time: "at", keep: 14 },
|
||||
storage_samples: { time: "at", keep: 400, note: "one reading an hour of each Proxmox storage and Synology volume" },
|
||||
ha_readings: { time: "at", keep: 35, note: "the watched Home Assistant numbers" },
|
||||
alert_incidents: { time: "resolved_at", keep: 400, where: "resolved_at IS NOT NULL", note: "trimmed 400 days after an incident resolved" },
|
||||
webhook_deliveries: { time: "created_at", keep: 14, note: "the last 100 per webhook, and none older than two weeks" },
|
||||
audit_log: { time: "created_at", keep: "log", note: "who did what" },
|
||||
diag_log: { time: "created_at", keep: "log", note: "every call to an integration or DNS provider" },
|
||||
notification_queue: { time: "created_at", keep: null, note: "held for quiet hours; emptied when the digest is sent" },
|
||||
renewals: { time: "at", keep: null, note: "small; never trimmed" },
|
||||
};
|
||||
|
||||
export type UsageState = "limit" | "filling" | "not_trimmed" | "unlimited" | "plain";
|
||||
|
||||
export interface TableUsage {
|
||||
name: string;
|
||||
rows: number;
|
||||
/** The table and its indexes, in bytes; null when the database cannot say. */
|
||||
bytes: number | null;
|
||||
/** How long it keeps what it holds, in days; null for no limit. */
|
||||
keepDays: number | null;
|
||||
note: string | null;
|
||||
/** ISO — the oldest and newest row, for the tables that keep a history. */
|
||||
oldest: string | null;
|
||||
newest: string | null;
|
||||
/** Rows added a day lately (over the last week, or for as long as it has been going if less). */
|
||||
perDay: number | null;
|
||||
/** How many days its history reaches back. */
|
||||
spanDays: number | null;
|
||||
state: UsageState;
|
||||
stateText: string;
|
||||
}
|
||||
|
||||
export interface DataUsage {
|
||||
generatedAt: string;
|
||||
database: { file: string; bytes: number; walBytes: number; pages: number; pageSize: number; freePages: number; freeBytes: number; tables: number };
|
||||
backups: { files: number; bytes: number };
|
||||
sessions: { files: number; bytes: number };
|
||||
/** Everything in the data folder that is measured here. */
|
||||
totalBytes: number;
|
||||
tables: TableUsage[];
|
||||
/** What needs a look: the tables not trimmed as they should be, and those with no limit. */
|
||||
attention: { table: string; text: string }[];
|
||||
}
|
||||
|
||||
function folderSize(dir: string): { files: number; bytes: number } {
|
||||
let files = 0;
|
||||
let bytes = 0;
|
||||
try {
|
||||
if (!existsSync(dir)) return { files, bytes };
|
||||
for (const name of readdirSync(dir)) {
|
||||
try {
|
||||
const s = statSync(join(dir, name));
|
||||
if (s.isFile()) {
|
||||
files++;
|
||||
bytes += s.size;
|
||||
}
|
||||
} catch {
|
||||
// gone while looking
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// unreadable: counts as nothing
|
||||
}
|
||||
return { files, bytes };
|
||||
}
|
||||
|
||||
const sizeOf = (path: string) => {
|
||||
try {
|
||||
return statSync(path).size;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
|
||||
/** Database time columns are "2026-10-03 14:05:09" (UTC), ISO, or just a date; all of them become an ISO moment. */
|
||||
export function toIso(value: string): string | null {
|
||||
const t = /^\d{4}-\d{2}-\d{2}$/.test(value) ? Date.parse(`${value}T00:00:00Z`) : /T|Z$/.test(value) ? Date.parse(value) : Date.parse(`${value.replace(" ", "T")}Z`);
|
||||
return Number.isFinite(t) ? new Date(t).toISOString() : null;
|
||||
}
|
||||
|
||||
/** Pure: whether a history is being trimmed as it should be, in words. `keepDays` null is no limit. */
|
||||
export function judge(input: { keepDays: number | null; spanDays: number | null; rows: number; logsOff?: boolean }): { state: UsageState; text: string } {
|
||||
const { keepDays, spanDays, rows } = input;
|
||||
if (keepDays === null) return input.logsOff ? { state: "unlimited", text: "Grows without limit — automatic purging is switched off (Settings → Data → Logs)." } : { state: "plain", text: "" };
|
||||
if (rows === 0 || spanDays === null) return { state: "limit", text: "Empty." };
|
||||
if (spanDays > keepDays + 2) return { state: "not_trimmed", text: `Reaches back ${Math.round(spanDays)} days, but should keep only ${keepDays} — the trimming is not running, or has not caught up.` };
|
||||
if (spanDays < keepDays - 1) return { state: "filling", text: `Holds ${Math.max(1, Math.round(spanDays))} of its ${keepDays} days so far.` };
|
||||
return { state: "limit", text: `At its limit: the oldest is about ${Math.round(spanDays)} days old, and it keeps ${keepDays}.` };
|
||||
}
|
||||
|
||||
export async function getDataUsage(now: Date = new Date()): Promise<DataUsage> {
|
||||
const settings = await getSettings();
|
||||
const names = (await client.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name")).rows.map((r) => String(r.name));
|
||||
|
||||
// Bytes per table, with its indexes counted in: the database reports its pages by the name of what owns them.
|
||||
const bytes = new Map<string, number>();
|
||||
let dbstat = true;
|
||||
try {
|
||||
const owners = new Map((await client.execute("SELECT name, tbl_name FROM sqlite_master")).rows.map((r) => [String(r.name), String(r.tbl_name)]));
|
||||
for (const r of (await client.execute("SELECT name, sum(pgsize) AS bytes FROM dbstat GROUP BY name")).rows) {
|
||||
const owner = owners.get(String(r.name)) ?? String(r.name);
|
||||
bytes.set(owner, (bytes.get(owner) ?? 0) + Number(r.bytes));
|
||||
}
|
||||
} catch {
|
||||
dbstat = false;
|
||||
}
|
||||
|
||||
const tables: TableUsage[] = [];
|
||||
const weekAgo = new Date(now.getTime() - 7 * DAY_MS).toISOString();
|
||||
for (const name of names) {
|
||||
const rows = Number((await client.execute(`SELECT count(*) AS n FROM "${name}"`)).rows[0].n);
|
||||
const meta = HISTORY[name];
|
||||
let keepDays: number | null = null;
|
||||
let oldest: string | null = null;
|
||||
let newest: string | null = null;
|
||||
let perDay: number | null = null;
|
||||
let spanDays: number | null = null;
|
||||
if (meta) {
|
||||
keepDays = meta.keep === "log" ? (settings.logRetention.enabled ? settings.logRetention.retentionDays : null) : meta.keep;
|
||||
const where = meta.where ? `WHERE ${meta.where}` : "";
|
||||
const r = (await client.execute(`SELECT min("${meta.time}") AS oldest, max("${meta.time}") AS newest FROM "${name}" ${where}`)).rows[0];
|
||||
oldest = r.oldest === null ? null : toIso(String(r.oldest));
|
||||
newest = r.newest === null ? null : toIso(String(r.newest));
|
||||
if (oldest) {
|
||||
spanDays = Math.max(0, (now.getTime() - Date.parse(oldest)) / DAY_MS);
|
||||
// The rows of the last week, by the way this table writes its times: a date alone, SQLite's "2026-10-03 14:05:09", or ISO.
|
||||
const sample = String((await client.execute(`SELECT "${meta.time}" AS t FROM "${name}" WHERE "${meta.time}" IS NOT NULL LIMIT 1`)).rows[0]?.t ?? "");
|
||||
const cutoff = /^\d{4}-\d{2}-\d{2}$/.test(sample) ? weekAgo.slice(0, 10) : /^\d{4}-\d{2}-\d{2} /.test(sample) ? weekAgo.slice(0, 19).replace("T", " ") : weekAgo;
|
||||
const recent = Number((await client.execute({ sql: `SELECT count(*) AS n FROM "${name}" ${where ? `${where} AND` : "WHERE"} "${meta.time}" >= ?`, args: [cutoff] })).rows[0].n);
|
||||
perDay = Math.round((recent / Math.max(1, Math.min(7, spanDays))) * 10) / 10;
|
||||
}
|
||||
}
|
||||
const logsOff = meta?.keep === "log" && keepDays === null;
|
||||
const verdict = meta ? judge({ keepDays, spanDays, rows, logsOff }) : { state: "plain" as const, text: "" };
|
||||
tables.push({ name, rows, bytes: dbstat ? (bytes.get(name) ?? 0) : null, keepDays, note: meta?.note ?? null, oldest, newest, perDay, spanDays: spanDays === null ? null : Math.round(spanDays * 10) / 10, state: verdict.state, stateText: verdict.text });
|
||||
}
|
||||
tables.sort((a, b) => (b.bytes ?? b.rows) - (a.bytes ?? a.rows) || a.name.localeCompare(b.name));
|
||||
|
||||
const pages = Number((await client.execute("PRAGMA page_count")).rows[0].page_count);
|
||||
const pageSize = Number((await client.execute("PRAGMA page_size")).rows[0].page_size);
|
||||
const freePages = Number((await client.execute("PRAGMA freelist_count")).rows[0].freelist_count);
|
||||
const fileBytes = sizeOf(dbPath);
|
||||
const walBytes = sizeOf(`${dbPath}-wal`);
|
||||
const backups = folderSize(backupDir());
|
||||
const sessions = folderSize(resolveDataPath(env.sessionDir));
|
||||
|
||||
return {
|
||||
generatedAt: now.toISOString(),
|
||||
database: { file: dbPath.split(/[\\/]/).pop() ?? "", bytes: fileBytes, walBytes, pages, pageSize, freePages, freeBytes: freePages * pageSize, tables: names.length },
|
||||
backups,
|
||||
sessions,
|
||||
totalBytes: fileBytes + walBytes + backups.bytes + sessions.bytes,
|
||||
tables,
|
||||
attention: tables.filter((t) => t.state === "not_trimmed" || t.state === "unlimited").map((t) => ({ table: t.name, text: t.stateText })),
|
||||
};
|
||||
}
|
||||
@@ -11,9 +11,9 @@
|
||||
* Settings) and forgotten.
|
||||
*/
|
||||
import { createHmac, randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { and, desc, eq, lt, notInArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { eventWebhooks, webhookEvents, type WebhookEvent } from "../db/schema.js";
|
||||
import { eventWebhooks, webhookDeliveries, webhookEvents, type WebhookEvent } from "../db/schema.js";
|
||||
import { decryptSecret, encryptSecret } from "../crypto.js";
|
||||
import { env } from "../env.js";
|
||||
|
||||
@@ -76,23 +76,50 @@ async function post(row: EventWebhookRow, event: string, body: string, deliveryI
|
||||
}
|
||||
}
|
||||
|
||||
/** One message to one webhook, retried once if it didn't go through. The outcome is written on the webhook. */
|
||||
/** How many deliveries are kept for each webhook, and for how long. */
|
||||
export const KEEP_DELIVERIES = 100;
|
||||
export const KEEP_DELIVERY_DAYS = 14;
|
||||
/** A body longer than this is not kept, so it cannot be sent again by hand. */
|
||||
export const MAX_STORED_BODY = 16_000;
|
||||
|
||||
/** One message to one webhook, retried once if it didn't go through. The outcome is written on the webhook and in its delivery log. */
|
||||
export async function deliver(row: EventWebhookRow, event: string, data: unknown, options: { retryAfterMs?: number } = {}): Promise<Delivery> {
|
||||
const deliveryId = randomUUID();
|
||||
const body = JSON.stringify({ event, id: deliveryId, createdAt: new Date().toISOString(), source: "homelab-manager", data });
|
||||
return sendAndNote(row, event, body, deliveryId, options, null);
|
||||
}
|
||||
|
||||
/** Sends a message that was sent before, exactly as it was (the same id, so a receiver that already handled it can ignore it). Says how it went, and writes that down as a delivery of its own. */
|
||||
export async function redeliver(row: EventWebhookRow, earlier: typeof webhookDeliveries.$inferSelect, options: { retryAfterMs?: number } = {}): Promise<Delivery> {
|
||||
return sendAndNote(row, earlier.event, earlier.body, earlier.deliveryId, options, earlier.id);
|
||||
}
|
||||
|
||||
async function sendAndNote(row: EventWebhookRow, event: string, body: string, deliveryId: string, options: { retryAfterMs?: number }, retryOf: number | null): Promise<Delivery> {
|
||||
let attempts = 1;
|
||||
let outcome = await post(row, event, body, deliveryId);
|
||||
if (!outcome.ok) {
|
||||
await new Promise((r) => setTimeout(r, options.retryAfterMs ?? RETRY_AFTER_MS));
|
||||
outcome = await post(row, event, body, deliveryId);
|
||||
attempts = 2;
|
||||
}
|
||||
const now = new Date();
|
||||
try {
|
||||
await db.update(eventWebhooks).set({ lastAttemptAt: new Date().toISOString(), lastResult: outcome.result }).where(eq(eventWebhooks.id, row.id));
|
||||
await db.update(eventWebhooks).set({ lastAttemptAt: now.toISOString(), lastResult: outcome.result }).where(eq(eventWebhooks.id, row.id));
|
||||
await db.insert(webhookDeliveries).values({ webhookId: row.id, event, deliveryId, attempts, ok: outcome.ok, result: outcome.result, body: body.slice(0, MAX_STORED_BODY + 1), retryOf, createdAt: now.toISOString() });
|
||||
await pruneDeliveries(row.id, now);
|
||||
} catch (err) {
|
||||
console.error("[webhooks] couldn't note the result:", err);
|
||||
}
|
||||
return outcome;
|
||||
}
|
||||
|
||||
/** Keeps the last hundred deliveries of a webhook, and none older than two weeks. */
|
||||
export async function pruneDeliveries(webhookId: number, now: Date = new Date()): Promise<void> {
|
||||
const keep = await db.select({ id: webhookDeliveries.id }).from(webhookDeliveries).where(eq(webhookDeliveries.webhookId, webhookId)).orderBy(desc(webhookDeliveries.createdAt), desc(webhookDeliveries.id)).limit(KEEP_DELIVERIES);
|
||||
if (keep.length === KEEP_DELIVERIES) await db.delete(webhookDeliveries).where(and(eq(webhookDeliveries.webhookId, webhookId), notInArray(webhookDeliveries.id, keep.map((k) => k.id))));
|
||||
await db.delete(webhookDeliveries).where(and(eq(webhookDeliveries.webhookId, webhookId), lt(webhookDeliveries.createdAt, new Date(now.getTime() - KEEP_DELIVERY_DAYS * 86_400_000).toISOString())));
|
||||
}
|
||||
|
||||
/**
|
||||
* Sends an event to every enabled webhook that wants it. Returns at once — delivery carries on in the background and can never delay or
|
||||
* fail what caused the event.
|
||||
|
||||
@@ -0,0 +1,269 @@
|
||||
/**
|
||||
* How big the app's own data is and whether it is being trimmed, and the delivery log of the event webhooks with its retry.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { createServer, type IncomingMessage, type Server } from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-usage-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, serverMetrics, endpointChecks, endpointResults, auditLog, eventWebhooks, webhookDeliveries } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const usage = await import("../src/services/dataUsage.js");
|
||||
const hooks = await import("../src/services/eventWebhooks.js");
|
||||
const { updateSettings } = await import("../src/services/settingsStore.js");
|
||||
const { dataUsageRouter } = await import("../src/routes/dataUsage.js");
|
||||
const { eventWebhooksRouter } = await import("../src/routes/eventWebhooks.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
mkdirSync(join(dir, "sessions"), { recursive: true });
|
||||
mkdirSync(join(dir, "backups"), { recursive: true });
|
||||
|
||||
after(() => {
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows may still hold the database file
|
||||
}
|
||||
});
|
||||
|
||||
const DAY = 86_400_000;
|
||||
const NOW = new Date("2026-10-20T12:00:00Z");
|
||||
const ago = (days: number) => new Date(NOW.getTime() - days * DAY).toISOString();
|
||||
|
||||
describe("is a history being trimmed?", () => {
|
||||
test("it reaches back as far as it should, or it is still filling, or it is holding far too much, or it has no limit at all", () => {
|
||||
assert.deepEqual(usage.judge({ keepDays: 35, spanDays: 34.5, rows: 900 }).state, "limit");
|
||||
assert.deepEqual(usage.judge({ keepDays: 35, spanDays: 36, rows: 900 }).state, "limit", "a day or two over is the trimming not having run yet today");
|
||||
assert.equal(usage.judge({ keepDays: 35, spanDays: 12, rows: 300 }).state, "filling");
|
||||
assert.match(usage.judge({ keepDays: 35, spanDays: 12, rows: 300 }).text, /Holds 12 of its 35 days so far/);
|
||||
const bad = usage.judge({ keepDays: 35, spanDays: 90, rows: 5000 });
|
||||
assert.equal(bad.state, "not_trimmed");
|
||||
assert.match(bad.text, /Reaches back 90 days, but should keep only 35/);
|
||||
assert.equal(usage.judge({ keepDays: null, spanDays: 400, rows: 1000, logsOff: true }).state, "unlimited");
|
||||
assert.equal(usage.judge({ keepDays: null, spanDays: 10, rows: 10 }).state, "plain", "no limit and no promise of one");
|
||||
assert.equal(usage.judge({ keepDays: 14, spanDays: null, rows: 0 }).text, "Empty.");
|
||||
});
|
||||
|
||||
test("the times the database writes, in all three forms, become moments", () => {
|
||||
assert.equal(usage.toIso("2026-10-03 14:05:09"), "2026-10-03T14:05:09.000Z");
|
||||
assert.equal(usage.toIso("2026-10-03T14:05:09.123Z"), "2026-10-03T14:05:09.123Z");
|
||||
assert.equal(usage.toIso("2026-10-03"), "2026-10-03T00:00:00.000Z");
|
||||
assert.equal(usage.toIso("nonsense"), null);
|
||||
});
|
||||
});
|
||||
|
||||
describe("what the data folder holds", () => {
|
||||
before(async () => {
|
||||
const [s] = await db.insert(servers).values({ name: "web", apiTokenHash: "h", apiTokenPrefix: "p", ipAddresses: "[]" }).returning();
|
||||
// Server samples going back 60 days, though only 35 are meant to be kept: the trimming has not run.
|
||||
for (let d = 60; d >= 0; d -= 2) await db.insert(serverMetrics).values({ serverId: s.id, metric: "cpu", value: 10, recordedAt: ago(d) });
|
||||
// An endpoint check's answers for only five days of its fourteen.
|
||||
const [c] = await db.insert(endpointChecks).values({ name: "Wiki", kind: "http", target: "https://x/" } as never).returning();
|
||||
for (let h = 0; h < 5 * 24; h += 6) await db.insert(endpointResults).values({ checkId: c.id, at: new Date(NOW.getTime() - h * 3600_000).toISOString(), ok: true, ms: 40, message: "ok" } as never);
|
||||
// The audit log, in SQLite's own time format, going back 200 days.
|
||||
for (let d = 200; d >= 0; d -= 20) await db.insert(auditLog).values({ category: "settings", action: "update", createdAt: ago(d).slice(0, 19).replace("T", " ") });
|
||||
writeFileSync(join(dir, "backups", "homelab-manager-2026-10-19-03-30-00.sqlite"), Buffer.alloc(1500));
|
||||
writeFileSync(join(dir, "backups", "homelab-manager-2026-10-18-03-30-00.sqlite"), Buffer.alloc(500));
|
||||
writeFileSync(join(dir, "sessions", "abc.json"), "{}");
|
||||
});
|
||||
|
||||
test("the file, the backups and the sessions beside it, and the tables by what they hold", async () => {
|
||||
const u = await usage.getDataUsage(NOW);
|
||||
assert.ok(u.database.bytes > 100_000 && u.database.pages * u.database.pageSize >= u.database.bytes - 8192, "the file is as big as its pages");
|
||||
assert.deepEqual([u.backups.files, u.backups.bytes, u.sessions.files], [2, 2000, 1]);
|
||||
assert.equal(u.totalBytes, u.database.bytes + u.database.walBytes + 2000 + u.sessions.bytes);
|
||||
assert.ok(u.database.tables >= 60 && u.tables.length === u.database.tables);
|
||||
const t = Object.fromEntries(u.tables.map((x) => [x.name, x]));
|
||||
assert.equal(t.server_metrics.rows, 31);
|
||||
assert.ok(t.server_metrics.bytes! > 0 && t.users.bytes! > 0, "sizes come from the database's own page count, indexes included");
|
||||
assert.ok(u.tables.every((x, i) => i === 0 || (u.tables[i - 1].bytes ?? 0) >= (x.bytes ?? 0)), "the biggest first");
|
||||
});
|
||||
|
||||
test("a history that holds more than it should is said to be untrimmed; one still filling says how far it has got", async () => {
|
||||
const t = Object.fromEntries((await usage.getDataUsage(NOW)).tables.map((x) => [x.name, x]));
|
||||
assert.deepEqual([t.server_metrics.state, t.server_metrics.keepDays, Math.round(t.server_metrics.spanDays!)], ["not_trimmed", 35, 60]);
|
||||
assert.match(t.server_metrics.stateText, /should keep only 35/);
|
||||
assert.equal(t.server_metrics.oldest, ago(60));
|
||||
assert.deepEqual([t.endpoint_results.state, t.endpoint_results.keepDays], ["filling", 14]);
|
||||
assert.match(t.endpoint_results.stateText, /Holds 5 of its 14 days/);
|
||||
assert.ok(t.endpoint_results.perDay! >= 4, "four readings a day, over the five days");
|
||||
assert.equal(t.server_metrics.perDay, 0.6, "four samples (every other day) in the last week, over seven days");
|
||||
});
|
||||
|
||||
test("the logs have no limit until purging is switched on, and then are judged by it", async () => {
|
||||
await updateSettings({ logRetention: { enabled: false, retentionDays: 90, intervalHours: 24 } });
|
||||
let u = await usage.getDataUsage(NOW);
|
||||
let audit = u.tables.find((x) => x.name === "audit_log")!;
|
||||
assert.deepEqual([audit.state, audit.keepDays], ["unlimited", null]);
|
||||
assert.ok(u.attention.some((a) => a.table === "audit_log" && /switched off/.test(a.text)));
|
||||
assert.ok(u.attention.some((a) => a.table === "server_metrics"), "and what is not trimmed is on the list too");
|
||||
assert.ok(!u.attention.some((a) => a.table === "endpoint_results"), "a table that is merely filling needs no look");
|
||||
|
||||
await updateSettings({ logRetention: { enabled: true, retentionDays: 90, intervalHours: 24 } });
|
||||
u = await usage.getDataUsage(NOW);
|
||||
audit = u.tables.find((x) => x.name === "audit_log")!;
|
||||
assert.deepEqual([audit.state, audit.keepDays], ["not_trimmed", 90]);
|
||||
assert.match(audit.stateText, /Reaches back 200 days, but should keep only 90/);
|
||||
assert.equal(audit.perDay, 0.1, "the week is counted in the form this table writes its times in");
|
||||
});
|
||||
|
||||
test("only an administrator can ask", async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
||||
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
||||
]);
|
||||
let who = "viewer";
|
||||
const a = express();
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/data-usage", dataUsageRouter);
|
||||
const server = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
const base = `http://127.0.0.1:${(server.address() as AddressInfo).port}/api/data-usage`;
|
||||
assert.equal((await fetch(base, { headers: { connection: "close" } })).status, 403);
|
||||
who = "admin";
|
||||
const res = await fetch(base, { headers: { connection: "close" } });
|
||||
assert.equal(res.status, 200);
|
||||
const body = (await res.json()) as any;
|
||||
assert.ok(body.tables.length > 0 && body.database.bytes > 0 && !JSON.stringify(body).includes(dir.replace(/\\/g, "\\\\")), "no folder path leaves the server");
|
||||
server.close();
|
||||
});
|
||||
});
|
||||
|
||||
describe("the webhook delivery log", () => {
|
||||
let receiver: Server;
|
||||
let receiverUrl = "";
|
||||
const got: { headers: IncomingMessage["headers"]; body: string }[] = [];
|
||||
let mode: "ok" | "fail" = "ok";
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "admin";
|
||||
let hookId = 0;
|
||||
|
||||
before(async () => {
|
||||
receiver = await new Promise<Server>((resolve) => {
|
||||
const s = createServer((req, res) => {
|
||||
let b = "";
|
||||
req.on("data", (c) => (b += c));
|
||||
req.on("end", () => {
|
||||
got.push({ headers: req.headers, body: b });
|
||||
res.statusCode = mode === "ok" ? 200 : 500;
|
||||
res.end("x");
|
||||
});
|
||||
}).listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
receiverUrl = `http://127.0.0.1:${(receiver.address() as AddressInfo).port}/hook`;
|
||||
const [hook] = await db.insert(eventWebhooks).values({ label: "Receiver", url: receiverUrl, secret: hooks.sealSecret("s3cret"), events: JSON.stringify(["alert.opened"]) }).returning();
|
||||
hookId = hook.id;
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/event-webhooks", eventWebhooksRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/event-webhooks`;
|
||||
});
|
||||
after(() => {
|
||||
receiver.close();
|
||||
app.close();
|
||||
});
|
||||
|
||||
const call = (method: string, path: string) => fetch(`${base}${path}`, { method, headers: { connection: "close" } });
|
||||
const rowOf = async () => (await db.select().from(eventWebhooks).where(eq(eventWebhooks.id, hookId)))[0];
|
||||
|
||||
test("every message is logged with how it went: one try that worked, and a failure that was tried twice", async () => {
|
||||
const hook = await rowOf();
|
||||
mode = "ok";
|
||||
assert.equal((await hooks.deliver(hook, "alert.opened", { alert: { key: "offline:server:1", message: "web is down" } }, { retryAfterMs: 1 })).ok, true);
|
||||
mode = "fail";
|
||||
assert.equal((await hooks.deliver(hook, "alert.opened", { alert: { key: "offline:server:2", message: "pi is down" } }, { retryAfterMs: 1 })).ok, false);
|
||||
const rows = await db.select().from(webhookDeliveries).orderBy(webhookDeliveries.id);
|
||||
assert.deepEqual(rows.map((r) => [r.event, r.attempts, r.ok, r.result]), [["alert.opened", 1, true, "200"], ["alert.opened", 2, false, "500"]]);
|
||||
assert.equal(got.length, 3, "one, then two");
|
||||
assert.equal(JSON.parse(rows[1].body).data.alert.message, "pi is down");
|
||||
assert.equal(rows[1].deliveryId, JSON.parse(rows[1].body).id);
|
||||
assert.equal(rows[1].retryOf, null);
|
||||
});
|
||||
|
||||
test("the log is for administrators; it lists newest first, with the message and whether it can be sent again", async () => {
|
||||
who = "viewer";
|
||||
assert.equal((await call("GET", `/${hookId}/deliveries`)).status, 403, "operators and viewers cannot read what was sent");
|
||||
who = "admin";
|
||||
const res = await call("GET", `/${hookId}/deliveries`);
|
||||
assert.equal(res.status, 200);
|
||||
const list = ((await res.json()) as any).deliveries;
|
||||
assert.deepEqual(list.map((d: any) => [d.ok, d.attempts, d.retryable]), [[false, 2, true], [true, 1, true]]);
|
||||
assert.equal(JSON.parse(list[0].body).data.alert.key, "offline:server:2");
|
||||
assert.equal((await call("GET", "/99999/deliveries")).status, 404);
|
||||
assert.equal((await call("GET", "/abc/deliveries")).status, 400);
|
||||
});
|
||||
|
||||
test("retrying sends the very same message — the same body, the same id, the same signature — and logs it as a delivery of its own", async () => {
|
||||
const [failed] = await db.select().from(webhookDeliveries).where(eq(webhookDeliveries.ok, false));
|
||||
mode = "ok";
|
||||
got.length = 0;
|
||||
const res = await call("POST", `/${hookId}/deliveries/${failed.id}/retry`);
|
||||
assert.deepEqual([res.status, ((await res.json()) as any).ok], [200, true]);
|
||||
assert.equal(got.length, 1);
|
||||
assert.equal(got[0].body, failed.body, "the exact bytes");
|
||||
assert.equal(got[0].headers["x-hlm-delivery"], failed.deliveryId);
|
||||
assert.equal(got[0].headers["x-hlm-signature"], hooks.signBody("s3cret", failed.body), "so a receiver checking the signature is satisfied");
|
||||
const rows = await db.select().from(webhookDeliveries).orderBy(webhookDeliveries.id);
|
||||
assert.deepEqual([rows.length, rows[2].retryOf, rows[2].ok, rows[2].deliveryId], [3, failed.id, true, failed.deliveryId]);
|
||||
assert.equal((await rowOf()).lastResult, "200", "and the webhook's own last result follows");
|
||||
const [audit] = await db.select().from(auditLog).where(eq(auditLog.action, "retry"));
|
||||
assert.deepEqual([audit.category, audit.targetId, JSON.parse(audit.detail!).event], ["webhook", String(hookId), "alert.opened"]);
|
||||
assert.equal((await call("POST", `/${hookId}/deliveries/99999/retry`)).status, 404);
|
||||
assert.equal((await call("POST", `/${hookId + 1}/deliveries/${failed.id}/retry`)).status, 404, "a delivery of another webhook is not this one's");
|
||||
who = "viewer";
|
||||
assert.equal((await call("POST", `/${hookId}/deliveries/${failed.id}/retry`)).status, 403);
|
||||
who = "admin";
|
||||
});
|
||||
|
||||
test("a message too big to have been kept cannot be sent again, and says so", async () => {
|
||||
const [big] = await db.insert(webhookDeliveries).values({ webhookId: hookId, event: "alert.opened", deliveryId: "big", attempts: 1, ok: false, result: "500", body: "x".repeat(hooks.MAX_STORED_BODY + 1), createdAt: new Date().toISOString() }).returning();
|
||||
const list = ((await (await call("GET", `/${hookId}/deliveries`)).json()) as any).deliveries;
|
||||
const row = list.find((d: any) => d.id === big.id);
|
||||
assert.deepEqual([row.retryable, row.body], [false, null]);
|
||||
const res = await call("POST", `/${hookId}/deliveries/${big.id}/retry`);
|
||||
assert.equal(res.status, 409);
|
||||
await db.delete(webhookDeliveries).where(eq(webhookDeliveries.id, big.id));
|
||||
});
|
||||
|
||||
test("only the last hundred are kept for a webhook, and none older than two weeks; removing the webhook removes its log", async () => {
|
||||
const hook = await rowOf();
|
||||
const base0 = Date.now();
|
||||
for (let i = 0; i < 110; i++) await db.insert(webhookDeliveries).values({ webhookId: hookId, event: "alert.opened", deliveryId: `d${i}`, attempts: 1, ok: true, result: "200", body: "{}", createdAt: new Date(base0 - i * 1000).toISOString() });
|
||||
await db.insert(webhookDeliveries).values({ webhookId: hookId, event: "alert.opened", deliveryId: "old", attempts: 1, ok: true, result: "200", body: "{}", createdAt: new Date(base0 - 20 * DAY).toISOString() });
|
||||
mode = "ok";
|
||||
await hooks.deliver(hook, "alert.opened", { alert: { key: "k", message: "m" } }, { retryAfterMs: 1 });
|
||||
const rows = await db.select().from(webhookDeliveries).where(eq(webhookDeliveries.webhookId, hookId));
|
||||
assert.equal(rows.length, hooks.KEEP_DELIVERIES);
|
||||
assert.ok(!rows.some((r) => r.deliveryId === "old"), "two weeks and more is gone");
|
||||
assert.ok(rows.some((r) => r.deliveryId === "d0") && !rows.some((r) => r.deliveryId === "d109"), "the newest kept, the oldest dropped");
|
||||
const removed = await call("DELETE", `/${hookId}`);
|
||||
assert.equal(removed.status, 204);
|
||||
assert.equal((await db.select().from(webhookDeliveries).where(eq(webhookDeliveries.webhookId, hookId))).length, 0);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user