How big the app's data is, and a webhook delivery log with Send again
Settings -> Data -> Storage measures the database, backups and sessions and every table (size, rows, growth, how long it is kept, how far back it reaches), and flags histories that are not being trimmed or have no limit. Event webhooks now keep a delivery log (last 100 per webhook, 14 days): what was sent, how it went, the message itself, and Send again, which resends it exactly as it was (same id and signature). Resends are audited and in the Change log. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
aebf3c6ce1
commit
e114567d64
22 files changed
+6833
-19
No files matched your search
@@ -179,6 +179,8 @@ export function describeChange(category: string, action: string, detail: Record<
|
||||
return { title: "Acknowledging several alerts at once was undone", tone: "neutral", detail: null };
|
||||
case "inventory/bulk_change":
|
||||
return { title: detail.change === "undo" ? "A change to several pieces of hardware was undone" : `${typeof detail.changed === "number" ? detail.changed : "Several"} piece${detail.changed === 1 ? "" : "s"} of hardware changed at once`, tone: "neutral", detail: genericSummary(detail, ["ids", "changed", "change"]) };
|
||||
case "webhook/retry":
|
||||
return { title: `A webhook message was sent again${name ? `: ${name}` : str(detail.label) ? `: ${str(detail.label)}` : ""}`, tone: detail.ok === false ? "bad" : "neutral", detail: str(detail.event) };
|
||||
case "alert_rule/create":
|
||||
return { title: `Alert rule added${name ? `: ${name}` : ""}`, tone: "neutral", detail: genericSummary(detail) };
|
||||
case "alert_rule/update":
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
/**
|
||||
* How big the app's own data is, and whether what is meant to be trimmed is being trimmed: the database file and what it holds table by table (rows and bytes), the
|
||||
* backups and the sessions beside it, and — for each table that keeps a history — how far back it reaches, how fast it is growing and whether that matches how long
|
||||
* it is meant to be kept. A history that holds far more than its limit means its trimming is not running; one that has no limit at all (the logs, until purging is
|
||||
* switched on) is said to grow without end.
|
||||
*
|
||||
* Nothing here changes anything. It only reads, and the table names it puts in a query come from a fixed list, never from a request.
|
||||
*/
|
||||
import { existsSync, readdirSync, statSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { client, dbPath } from "../db/client.js";
|
||||
import { env } from "../env.js";
|
||||
import { resolveDataPath } from "../paths.js";
|
||||
import { backupDir } from "./selfBackup.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
|
||||
const DAY_MS = 86_400_000;
|
||||
|
||||
/** How a table with a history is trimmed. `log` follows the log-retention setting. */
|
||||
interface HistoryMeta {
|
||||
/** The column that says when a row is from. */
|
||||
time: string;
|
||||
keep: number | "log" | null;
|
||||
/** Narrows to the rows the limit is about (an incident is only trimmed once resolved; the daily rows of the summary table). */
|
||||
where?: string;
|
||||
/** Said under the table's name. */
|
||||
note?: string;
|
||||
}
|
||||
|
||||
export const HISTORY: Record<string, HistoryMeta> = {
|
||||
server_metrics: { time: "recorded_at", keep: 35, note: "a sample of each server's load, memory and disks every 10–15 minutes" },
|
||||
server_metrics_rollup: { time: "at", keep: 800, where: "grain = 'd'", note: "hourly figures are kept 120 days, daily ones about 800" },
|
||||
endpoint_results: { time: "at", keep: 14, note: "every answer of every endpoint check" },
|
||||
endpoint_hourly: { time: "hour", keep: 120 },
|
||||
endpoint_daily: { time: "day", keep: 800 },
|
||||
heartbeat_pings: { time: "at", keep: 14 },
|
||||
storage_samples: { time: "at", keep: 400, note: "one reading an hour of each Proxmox storage and Synology volume" },
|
||||
ha_readings: { time: "at", keep: 35, note: "the watched Home Assistant numbers" },
|
||||
alert_incidents: { time: "resolved_at", keep: 400, where: "resolved_at IS NOT NULL", note: "trimmed 400 days after an incident resolved" },
|
||||
webhook_deliveries: { time: "created_at", keep: 14, note: "the last 100 per webhook, and none older than two weeks" },
|
||||
audit_log: { time: "created_at", keep: "log", note: "who did what" },
|
||||
diag_log: { time: "created_at", keep: "log", note: "every call to an integration or DNS provider" },
|
||||
notification_queue: { time: "created_at", keep: null, note: "held for quiet hours; emptied when the digest is sent" },
|
||||
renewals: { time: "at", keep: null, note: "small; never trimmed" },
|
||||
};
|
||||
|
||||
export type UsageState = "limit" | "filling" | "not_trimmed" | "unlimited" | "plain";
|
||||
|
||||
export interface TableUsage {
|
||||
name: string;
|
||||
rows: number;
|
||||
/** The table and its indexes, in bytes; null when the database cannot say. */
|
||||
bytes: number | null;
|
||||
/** How long it keeps what it holds, in days; null for no limit. */
|
||||
keepDays: number | null;
|
||||
note: string | null;
|
||||
/** ISO — the oldest and newest row, for the tables that keep a history. */
|
||||
oldest: string | null;
|
||||
newest: string | null;
|
||||
/** Rows added a day lately (over the last week, or for as long as it has been going if less). */
|
||||
perDay: number | null;
|
||||
/** How many days its history reaches back. */
|
||||
spanDays: number | null;
|
||||
state: UsageState;
|
||||
stateText: string;
|
||||
}
|
||||
|
||||
export interface DataUsage {
|
||||
generatedAt: string;
|
||||
database: { file: string; bytes: number; walBytes: number; pages: number; pageSize: number; freePages: number; freeBytes: number; tables: number };
|
||||
backups: { files: number; bytes: number };
|
||||
sessions: { files: number; bytes: number };
|
||||
/** Everything in the data folder that is measured here. */
|
||||
totalBytes: number;
|
||||
tables: TableUsage[];
|
||||
/** What needs a look: the tables not trimmed as they should be, and those with no limit. */
|
||||
attention: { table: string; text: string }[];
|
||||
}
|
||||
|
||||
function folderSize(dir: string): { files: number; bytes: number } {
|
||||
let files = 0;
|
||||
let bytes = 0;
|
||||
try {
|
||||
if (!existsSync(dir)) return { files, bytes };
|
||||
for (const name of readdirSync(dir)) {
|
||||
try {
|
||||
const s = statSync(join(dir, name));
|
||||
if (s.isFile()) {
|
||||
files++;
|
||||
bytes += s.size;
|
||||
}
|
||||
} catch {
|
||||
// gone while looking
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// unreadable: counts as nothing
|
||||
}
|
||||
return { files, bytes };
|
||||
}
|
||||
|
||||
const sizeOf = (path: string) => {
|
||||
try {
|
||||
return statSync(path).size;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
|
||||
/** Database time columns are "2026-10-03 14:05:09" (UTC), ISO, or just a date; all of them become an ISO moment. */
|
||||
export function toIso(value: string): string | null {
|
||||
const t = /^\d{4}-\d{2}-\d{2}$/.test(value) ? Date.parse(`${value}T00:00:00Z`) : /T|Z$/.test(value) ? Date.parse(value) : Date.parse(`${value.replace(" ", "T")}Z`);
|
||||
return Number.isFinite(t) ? new Date(t).toISOString() : null;
|
||||
}
|
||||
|
||||
/** Pure: whether a history is being trimmed as it should be, in words. `keepDays` null is no limit. */
|
||||
export function judge(input: { keepDays: number | null; spanDays: number | null; rows: number; logsOff?: boolean }): { state: UsageState; text: string } {
|
||||
const { keepDays, spanDays, rows } = input;
|
||||
if (keepDays === null) return input.logsOff ? { state: "unlimited", text: "Grows without limit — automatic purging is switched off (Settings → Data → Logs)." } : { state: "plain", text: "" };
|
||||
if (rows === 0 || spanDays === null) return { state: "limit", text: "Empty." };
|
||||
if (spanDays > keepDays + 2) return { state: "not_trimmed", text: `Reaches back ${Math.round(spanDays)} days, but should keep only ${keepDays} — the trimming is not running, or has not caught up.` };
|
||||
if (spanDays < keepDays - 1) return { state: "filling", text: `Holds ${Math.max(1, Math.round(spanDays))} of its ${keepDays} days so far.` };
|
||||
return { state: "limit", text: `At its limit: the oldest is about ${Math.round(spanDays)} days old, and it keeps ${keepDays}.` };
|
||||
}
|
||||
|
||||
export async function getDataUsage(now: Date = new Date()): Promise<DataUsage> {
|
||||
const settings = await getSettings();
|
||||
const names = (await client.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name")).rows.map((r) => String(r.name));
|
||||
|
||||
// Bytes per table, with its indexes counted in: the database reports its pages by the name of what owns them.
|
||||
const bytes = new Map<string, number>();
|
||||
let dbstat = true;
|
||||
try {
|
||||
const owners = new Map((await client.execute("SELECT name, tbl_name FROM sqlite_master")).rows.map((r) => [String(r.name), String(r.tbl_name)]));
|
||||
for (const r of (await client.execute("SELECT name, sum(pgsize) AS bytes FROM dbstat GROUP BY name")).rows) {
|
||||
const owner = owners.get(String(r.name)) ?? String(r.name);
|
||||
bytes.set(owner, (bytes.get(owner) ?? 0) + Number(r.bytes));
|
||||
}
|
||||
} catch {
|
||||
dbstat = false;
|
||||
}
|
||||
|
||||
const tables: TableUsage[] = [];
|
||||
const weekAgo = new Date(now.getTime() - 7 * DAY_MS).toISOString();
|
||||
for (const name of names) {
|
||||
const rows = Number((await client.execute(`SELECT count(*) AS n FROM "${name}"`)).rows[0].n);
|
||||
const meta = HISTORY[name];
|
||||
let keepDays: number | null = null;
|
||||
let oldest: string | null = null;
|
||||
let newest: string | null = null;
|
||||
let perDay: number | null = null;
|
||||
let spanDays: number | null = null;
|
||||
if (meta) {
|
||||
keepDays = meta.keep === "log" ? (settings.logRetention.enabled ? settings.logRetention.retentionDays : null) : meta.keep;
|
||||
const where = meta.where ? `WHERE ${meta.where}` : "";
|
||||
const r = (await client.execute(`SELECT min("${meta.time}") AS oldest, max("${meta.time}") AS newest FROM "${name}" ${where}`)).rows[0];
|
||||
oldest = r.oldest === null ? null : toIso(String(r.oldest));
|
||||
newest = r.newest === null ? null : toIso(String(r.newest));
|
||||
if (oldest) {
|
||||
spanDays = Math.max(0, (now.getTime() - Date.parse(oldest)) / DAY_MS);
|
||||
// The rows of the last week, by the way this table writes its times: a date alone, SQLite's "2026-10-03 14:05:09", or ISO.
|
||||
const sample = String((await client.execute(`SELECT "${meta.time}" AS t FROM "${name}" WHERE "${meta.time}" IS NOT NULL LIMIT 1`)).rows[0]?.t ?? "");
|
||||
const cutoff = /^\d{4}-\d{2}-\d{2}$/.test(sample) ? weekAgo.slice(0, 10) : /^\d{4}-\d{2}-\d{2} /.test(sample) ? weekAgo.slice(0, 19).replace("T", " ") : weekAgo;
|
||||
const recent = Number((await client.execute({ sql: `SELECT count(*) AS n FROM "${name}" ${where ? `${where} AND` : "WHERE"} "${meta.time}" >= ?`, args: [cutoff] })).rows[0].n);
|
||||
perDay = Math.round((recent / Math.max(1, Math.min(7, spanDays))) * 10) / 10;
|
||||
}
|
||||
}
|
||||
const logsOff = meta?.keep === "log" && keepDays === null;
|
||||
const verdict = meta ? judge({ keepDays, spanDays, rows, logsOff }) : { state: "plain" as const, text: "" };
|
||||
tables.push({ name, rows, bytes: dbstat ? (bytes.get(name) ?? 0) : null, keepDays, note: meta?.note ?? null, oldest, newest, perDay, spanDays: spanDays === null ? null : Math.round(spanDays * 10) / 10, state: verdict.state, stateText: verdict.text });
|
||||
}
|
||||
tables.sort((a, b) => (b.bytes ?? b.rows) - (a.bytes ?? a.rows) || a.name.localeCompare(b.name));
|
||||
|
||||
const pages = Number((await client.execute("PRAGMA page_count")).rows[0].page_count);
|
||||
const pageSize = Number((await client.execute("PRAGMA page_size")).rows[0].page_size);
|
||||
const freePages = Number((await client.execute("PRAGMA freelist_count")).rows[0].freelist_count);
|
||||
const fileBytes = sizeOf(dbPath);
|
||||
const walBytes = sizeOf(`${dbPath}-wal`);
|
||||
const backups = folderSize(backupDir());
|
||||
const sessions = folderSize(resolveDataPath(env.sessionDir));
|
||||
|
||||
return {
|
||||
generatedAt: now.toISOString(),
|
||||
database: { file: dbPath.split(/[\\/]/).pop() ?? "", bytes: fileBytes, walBytes, pages, pageSize, freePages, freeBytes: freePages * pageSize, tables: names.length },
|
||||
backups,
|
||||
sessions,
|
||||
totalBytes: fileBytes + walBytes + backups.bytes + sessions.bytes,
|
||||
tables,
|
||||
attention: tables.filter((t) => t.state === "not_trimmed" || t.state === "unlimited").map((t) => ({ table: t.name, text: t.stateText })),
|
||||
};
|
||||
}
|
||||
@@ -11,9 +11,9 @@
|
||||
* Settings) and forgotten.
|
||||
*/
|
||||
import { createHmac, randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { and, desc, eq, lt, notInArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { eventWebhooks, webhookEvents, type WebhookEvent } from "../db/schema.js";
|
||||
import { eventWebhooks, webhookDeliveries, webhookEvents, type WebhookEvent } from "../db/schema.js";
|
||||
import { decryptSecret, encryptSecret } from "../crypto.js";
|
||||
import { env } from "../env.js";
|
||||
|
||||
@@ -76,23 +76,50 @@ async function post(row: EventWebhookRow, event: string, body: string, deliveryI
|
||||
}
|
||||
}
|
||||
|
||||
/** One message to one webhook, retried once if it didn't go through. The outcome is written on the webhook. */
|
||||
/** How many deliveries are kept for each webhook, and for how long. */
|
||||
export const KEEP_DELIVERIES = 100;
|
||||
export const KEEP_DELIVERY_DAYS = 14;
|
||||
/** A body longer than this is not kept, so it cannot be sent again by hand. */
|
||||
export const MAX_STORED_BODY = 16_000;
|
||||
|
||||
/** One message to one webhook, retried once if it didn't go through. The outcome is written on the webhook and in its delivery log. */
|
||||
export async function deliver(row: EventWebhookRow, event: string, data: unknown, options: { retryAfterMs?: number } = {}): Promise<Delivery> {
|
||||
const deliveryId = randomUUID();
|
||||
const body = JSON.stringify({ event, id: deliveryId, createdAt: new Date().toISOString(), source: "homelab-manager", data });
|
||||
return sendAndNote(row, event, body, deliveryId, options, null);
|
||||
}
|
||||
|
||||
/** Sends a message that was sent before, exactly as it was (the same id, so a receiver that already handled it can ignore it). Says how it went, and writes that down as a delivery of its own. */
|
||||
export async function redeliver(row: EventWebhookRow, earlier: typeof webhookDeliveries.$inferSelect, options: { retryAfterMs?: number } = {}): Promise<Delivery> {
|
||||
return sendAndNote(row, earlier.event, earlier.body, earlier.deliveryId, options, earlier.id);
|
||||
}
|
||||
|
||||
async function sendAndNote(row: EventWebhookRow, event: string, body: string, deliveryId: string, options: { retryAfterMs?: number }, retryOf: number | null): Promise<Delivery> {
|
||||
let attempts = 1;
|
||||
let outcome = await post(row, event, body, deliveryId);
|
||||
if (!outcome.ok) {
|
||||
await new Promise((r) => setTimeout(r, options.retryAfterMs ?? RETRY_AFTER_MS));
|
||||
outcome = await post(row, event, body, deliveryId);
|
||||
attempts = 2;
|
||||
}
|
||||
const now = new Date();
|
||||
try {
|
||||
await db.update(eventWebhooks).set({ lastAttemptAt: new Date().toISOString(), lastResult: outcome.result }).where(eq(eventWebhooks.id, row.id));
|
||||
await db.update(eventWebhooks).set({ lastAttemptAt: now.toISOString(), lastResult: outcome.result }).where(eq(eventWebhooks.id, row.id));
|
||||
await db.insert(webhookDeliveries).values({ webhookId: row.id, event, deliveryId, attempts, ok: outcome.ok, result: outcome.result, body: body.slice(0, MAX_STORED_BODY + 1), retryOf, createdAt: now.toISOString() });
|
||||
await pruneDeliveries(row.id, now);
|
||||
} catch (err) {
|
||||
console.error("[webhooks] couldn't note the result:", err);
|
||||
}
|
||||
return outcome;
|
||||
}
|
||||
|
||||
/** Keeps the last hundred deliveries of a webhook, and none older than two weeks. */
|
||||
export async function pruneDeliveries(webhookId: number, now: Date = new Date()): Promise<void> {
|
||||
const keep = await db.select({ id: webhookDeliveries.id }).from(webhookDeliveries).where(eq(webhookDeliveries.webhookId, webhookId)).orderBy(desc(webhookDeliveries.createdAt), desc(webhookDeliveries.id)).limit(KEEP_DELIVERIES);
|
||||
if (keep.length === KEEP_DELIVERIES) await db.delete(webhookDeliveries).where(and(eq(webhookDeliveries.webhookId, webhookId), notInArray(webhookDeliveries.id, keep.map((k) => k.id))));
|
||||
await db.delete(webhookDeliveries).where(and(eq(webhookDeliveries.webhookId, webhookId), lt(webhookDeliveries.createdAt, new Date(now.getTime() - KEEP_DELIVERY_DAYS * 86_400_000).toISOString())));
|
||||
}
|
||||
|
||||
/**
|
||||
* Sends an event to every enabled webhook that wants it. Returns at once — delivery carries on in the background and can never delay or
|
||||
* fail what caused the event.
|
||||
|
||||
Reference in new issue
Block a user