Disk fill forecasts, skeleton loaders, and a Silence button on alerts
A disk that is on course to be full within 14 days (setting: 0 turns it off) is flagged on the Alerts page and dashboard, from the server's own history: needs three days of steady growth, skips erratic disks and ones the ordinary disk alert covers. Skeleton placeholders replace "Fetching status…" and "Loading…". Alerts that a maintenance window can silence get a Silence button (1, 4 or 24 hours) that starts one, and the alert list is refreshed at once when a window starts or ends. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
82974b98d0
commit
f7f5e1cd41
24 files changed
+566
-25
No files matched your search
@@ -0,0 +1,136 @@
|
||||
/**
|
||||
* "This disk will be full in N days" on the Alerts page, from a server's own history.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-forecast-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
|
||||
const { eq } = await import("drizzle-orm");
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { servers, serverMetrics, maintenanceWindows } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { collectAlerts } = await import("../src/services/alerts.js");
|
||||
const { updateSettings } = await import("../src/services/settingsStore.js");
|
||||
const { trendOf, disksFillingSoon } = await import("../src/services/serverMetrics.js");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const DAY = 24 * 3600_000;
|
||||
|
||||
describe("trendOf: how well a straight line fits", () => {
|
||||
const steady = Array.from({ length: 30 }, (_, i) => ({ t: i * 0.25 * DAY, v: 40 + i * 0.5 + (i % 2 ? 0.2 : -0.2) }));
|
||||
const jump = Array.from({ length: 30 }, (_, i) => ({ t: i * 0.25 * DAY, v: i < 15 ? 40 : 70 }));
|
||||
|
||||
test("a steady climb fits well; a single jump does not", () => {
|
||||
assert.ok(trendOf(steady)!.fit > 0.95);
|
||||
assert.ok(trendOf(jump)!.fit < 0.8);
|
||||
});
|
||||
|
||||
test("it says how long a time it covers", () => {
|
||||
assert.equal(trendOf(steady)!.spanDays, 7.3);
|
||||
});
|
||||
});
|
||||
|
||||
describe("disksFillingSoon", () => {
|
||||
const series = (over: Partial<{ latest: number; perDay: number; fit: number; spanDays: number; daysToFull: number | null }> = {}) => {
|
||||
const t = { perDay: 2, fit: 0.95, spanDays: 7, daysToFull: 10, ...over };
|
||||
return [{ metric: "disk:/data", kind: "disk" as const, label: "Disk /data", points: [], latest: over.latest ?? 80, trend: t }];
|
||||
};
|
||||
|
||||
test("picks a disk that fills within the window", () => {
|
||||
assert.deepEqual(disksFillingSoon(series(), 14, 90).map((d) => d.mount), ["/data"]);
|
||||
});
|
||||
|
||||
test("leaves out one that is later, already past the usual threshold, short of history, or erratic", () => {
|
||||
assert.equal(disksFillingSoon(series({ daysToFull: 30 }), 14, 90).length, 0, "later than the window");
|
||||
assert.equal(disksFillingSoon(series({ latest: 92 }), 14, 90).length, 0, "the ordinary disk alert covers it");
|
||||
assert.equal(disksFillingSoon(series({ spanDays: 1.5 }), 14, 90).length, 0, "too little history");
|
||||
assert.equal(disksFillingSoon(series({ fit: 0.3 }), 14, 90).length, 0, "growth is erratic");
|
||||
assert.equal(disksFillingSoon(series({ daysToFull: null }), 14, 90).length, 0, "not rising");
|
||||
});
|
||||
});
|
||||
|
||||
describe("the alert", () => {
|
||||
let steadyId = 0;
|
||||
let jumpyId = 0;
|
||||
let fullId = 0;
|
||||
|
||||
/** A disk sampled every 6 hours for 8 days ending now. */
|
||||
async function history(serverId: number, mount: string, valueAt: (i: number) => number) {
|
||||
const now = Date.now();
|
||||
for (let i = 0; i <= 32; i++) {
|
||||
await db.insert(serverMetrics).values({ serverId, metric: `disk:${mount}`, value: valueAt(i), recordedAt: new Date(now - (32 - i) * 0.25 * DAY).toISOString() });
|
||||
}
|
||||
}
|
||||
|
||||
before(async () => {
|
||||
steadyId = (await db.insert(servers).values({ name: "elsa", apiTokenHash: "a", apiTokenPrefix: "a" }).returning())[0].id;
|
||||
jumpyId = (await db.insert(servers).values({ name: "jumpy", apiTokenHash: "b", apiTokenPrefix: "b" }).returning())[0].id;
|
||||
fullId = (await db.insert(servers).values({ name: "nearlyfull", apiTokenHash: "c", apiTokenPrefix: "c" }).returning())[0].id;
|
||||
// 2 points a day: 50% a week ago... 66% now → 17 days to full at that rate.
|
||||
await history(steadyId, "/data", (i) => 50 + i * 0.5);
|
||||
await history(jumpyId, "/data", (i) => (i < 16 ? 50 : 72));
|
||||
// Already past the 90% disk alert, and still climbing.
|
||||
await history(fullId, "/data", (i) => 88 + i * 0.1);
|
||||
await updateSettings({ healthChecks: { diskForecastDays: 30 } });
|
||||
});
|
||||
|
||||
after(() => {
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const forecasts = async () => (await collectAlerts()).alerts.filter((a) => a.id.startsWith("disk:disk-forecast:"));
|
||||
|
||||
test("a steadily filling disk is flagged, with the rate, the days left and where to look", async () => {
|
||||
const found = await forecasts();
|
||||
assert.equal(found.length, 1, found.map((a) => a.message).join(" | "));
|
||||
assert.match(found[0].message, /Disk \/data on elsa/);
|
||||
assert.match(found[0].message, /about 2\.0 points a day/);
|
||||
assert.match(found[0].message, /full in about 17 days/);
|
||||
assert.equal(found[0].link, `/servers/${steadyId}`);
|
||||
assert.equal(found[0].severity, "warning");
|
||||
assert.deepEqual(found[0].target, { type: "server", id: steadyId, name: "elsa" });
|
||||
assert.equal(found[0].silenced, false);
|
||||
});
|
||||
|
||||
test("a disk that jumped once, and one the ordinary alert already covers, are not flagged", async () => {
|
||||
const names = (await forecasts()).map((a) => a.message).join(" ");
|
||||
assert.ok(!names.includes("jumpy"));
|
||||
assert.ok(!names.includes("nearlyfull"));
|
||||
});
|
||||
|
||||
test("a window on the server silences it", async () => {
|
||||
const now = new Date();
|
||||
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: steadyId, startedAt: new Date(now.getTime() - 3600_000).toISOString(), endsAt: new Date(now.getTime() + 3600_000).toISOString() });
|
||||
const [alert] = await forecasts();
|
||||
assert.equal(alert.silenced, true);
|
||||
await db.delete(maintenanceWindows);
|
||||
});
|
||||
|
||||
test("a disk that fills within three days is critical", async () => {
|
||||
await db.delete(serverMetrics).where(eq(serverMetrics.serverId, steadyId));
|
||||
await history(steadyId, "/data", (i) => 60 + i * 1.1); // 4.4 points a day from ~95% — full in a day or so
|
||||
await updateSettings({ healthChecks: { diskUsagePercent: 99 } });
|
||||
const [alert] = await forecasts();
|
||||
assert.equal(alert.severity, "critical");
|
||||
await updateSettings({ healthChecks: { diskUsagePercent: 90 } });
|
||||
});
|
||||
|
||||
test("with the setting at 0 nothing is flagged", async () => {
|
||||
await updateSettings({ healthChecks: { diskForecastDays: 0 } });
|
||||
assert.equal((await forecasts()).length, 0);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user