Service-level targets with error budgets, and quarterly and annual reports
Monitoring -> Service levels: a target (99.9 %) on an endpoint check, a server or an integration, measured over calendar months. Shows the figure, downtime used, the error budget left, and how the last six months went. 80 % of the budget used is a warning and an overspent budget a critical alert. Endpoints are measured day by day, servers and integrations to the minute from the audit log. The report page gains Quarter and Year (same figures, with a Service levels section), and two settings send the quarterly and annual report after the month that ends them. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
31044b2b64
commit
3b31b16796
31 files changed
+7706
-61
No files matched your search
@@ -0,0 +1,10 @@
|
||||
CREATE TABLE `service_levels` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`name` text NOT NULL,
|
||||
`subject_type` text NOT NULL,
|
||||
`subject_id` integer NOT NULL,
|
||||
`target` real NOT NULL,
|
||||
`alert` integer DEFAULT true NOT NULL,
|
||||
`created_by` text,
|
||||
`created_at` text DEFAULT (current_timestamp) NOT NULL
|
||||
);
|
||||
File diff suppressed because it is too large.
Load diff
@@ -386,6 +386,13 @@
|
||||
"when": 1792610229172,
|
||||
"tag": "0054_green_beast",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 55,
|
||||
"version": "6",
|
||||
"when": 1792612881951,
|
||||
"tag": "0055_gorgeous_the_phantom",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -56,6 +56,7 @@ import { dataUsageRouter } from "./routes/dataUsage.js";
|
||||
import { alertActionsRouter } from "./routes/alertActions.js";
|
||||
import { homeAssistantRouter } from "./routes/homeAssistant.js";
|
||||
import { upsRouter } from "./routes/ups.js";
|
||||
import { serviceLevelsRouter } from "./routes/serviceLevels.js";
|
||||
import { publicApiRouter } from "./routes/publicApi.js";
|
||||
import { generatorRouter } from "./routes/generator.js";
|
||||
import { alertTicketsRouter } from "./routes/alertTickets.js";
|
||||
@@ -186,6 +187,7 @@ export function createApp() {
|
||||
app.use("/api/data-usage", dataUsageRouter);
|
||||
app.use("/api/home-assistant", homeAssistantRouter);
|
||||
app.use("/api/ups", upsRouter);
|
||||
app.use("/api/service-levels", serviceLevelsRouter);
|
||||
app.use("/api/todos", todosRouter);
|
||||
app.use("/api/notes", notesRouter);
|
||||
app.use("/api/generator", generatorRouter);
|
||||
|
||||
@@ -1159,6 +1159,23 @@ export const haReadings = sqliteTable(
|
||||
(t) => [index("ha_readings_lookup_idx").on(t.integrationId, t.entityId, t.at)],
|
||||
);
|
||||
|
||||
// ─── Service-level targets ──────────────────────────────────────────────────
|
||||
|
||||
// "99.9 % for Gitea": a target on an endpoint check, a server or an integration, measured over calendar months (services/serviceLevels.ts). The thing measured is a
|
||||
// plain reference (type and id), not a foreign key: one whose subject is removed is shown as having nothing to measure.
|
||||
export const serviceLevels = sqliteTable("service_levels", {
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
name: text("name").notNull(),
|
||||
subjectType: text("subject_type").notNull(), // endpoint | server | integration
|
||||
subjectId: integer("subject_id").notNull(),
|
||||
target: real("target").notNull(), // percent, 90–99.999
|
||||
alert: integer("alert", { mode: "boolean" }).notNull().default(true), // warn when most of the month's error budget is used, and when it is overspent
|
||||
createdBy: text("created_by"),
|
||||
createdAt: text("created_at")
|
||||
.notNull()
|
||||
.default(sql`(current_timestamp)`),
|
||||
});
|
||||
|
||||
// ─── UPS ────────────────────────────────────────────────────────────────────
|
||||
|
||||
// How a UPS (an integration of type "ups", read over NUT or SNMP by services/upsSync.ts) is right now: one row each, replaced at every read. `updatedAt` is the last
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { Router } from "express";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { buildMonthlyReport, isMonthId, lastFullMonth, monthlyReportText, recentMonths } from "../services/monthlyReport.js";
|
||||
import { buildMonthlyReport, isPeriodId, lastFullMonth, monthlyReportText, recentPeriods } from "../services/monthlyReport.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { notifyNow } from "../services/notify.js";
|
||||
import { getSettings } from "../services/settingsStore.js";
|
||||
@@ -10,29 +10,30 @@ export const reportsRouter = Router();
|
||||
|
||||
reportsRouter.use(requireAuth);
|
||||
|
||||
/** The month asked for, or the last whole one; null when what was asked is not a month. */
|
||||
async function monthFrom(query: unknown): Promise<{ id: string; months: { id: string; label: string }[] } | null> {
|
||||
/** The month, quarter or year asked for, or the last whole month; null when what was asked is not one that can be had. */
|
||||
async function monthFrom(query: unknown): Promise<({ id: string } & ReturnType<typeof recentPeriods>) | null> {
|
||||
const tz = (await getSettings()).notifications.timezone || "UTC";
|
||||
const months = recentMonths(new Date(), tz);
|
||||
if (query === undefined || query === "") return { id: lastFullMonth(new Date(), tz), months };
|
||||
if (!isMonthId(query) || !months.some((m) => m.id === query)) return null;
|
||||
return { id: query, months };
|
||||
const options = recentPeriods(new Date(), tz);
|
||||
if (query === undefined || query === "") return { id: lastFullMonth(new Date(), tz), ...options };
|
||||
const all = [...options.months, ...options.quarters, ...options.years];
|
||||
if (!isPeriodId(query) || !all.some((m) => m.id === query)) return null;
|
||||
return { id: query, ...options };
|
||||
}
|
||||
|
||||
// How a month went, for everyone signed in: it is only what the pages it comes from already show them. ?month=2026-09, the last whole month by default;
|
||||
// ?format=text is the plain-text version a notification carries.
|
||||
// How a month, a quarter or a year went, for everyone signed in: it is only what the pages it comes from already show them. ?month=2026-09 (or ?month=2026-Q3,
|
||||
// ?month=2026), the last whole month by default; ?format=text is the plain-text version a notification carries.
|
||||
reportsRouter.get("/monthly", asyncHandler(async (req, res) => {
|
||||
const picked = await monthFrom(req.query.month);
|
||||
if (!picked) return res.status(400).json({ error: "invalid_month", message: "Ask for one of the last twelve months, like 2026-09." });
|
||||
const picked = await monthFrom(req.query.month ?? req.query.period);
|
||||
if (!picked) return res.status(400).json({ error: "invalid_month", message: "Ask for one of the last twelve months (2026-09), the last eight quarters (2026-Q3) or the last three years (2026)." });
|
||||
const report = await buildMonthlyReport(picked.id);
|
||||
if (req.query.format === "text") return res.json(monthlyReportText(report));
|
||||
res.json({ report, months: picked.months });
|
||||
res.json({ report, months: picked.months, quarters: picked.quarters, years: picked.years });
|
||||
}));
|
||||
|
||||
// Send it now to the notification channels, as if it were the day. Administrators.
|
||||
reportsRouter.post("/monthly/send", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const picked = await monthFrom((req.body as { month?: unknown } | undefined)?.month);
|
||||
if (!picked) return res.status(400).json({ error: "invalid_month", message: "Ask for one of the last twelve months, like 2026-09." });
|
||||
if (!picked) return res.status(400).json({ error: "invalid_month", message: "Ask for one of the last twelve months (2026-09), the last eight quarters (2026-Q3) or the last three years (2026)." });
|
||||
const { title, message } = monthlyReportText(await buildMonthlyReport(picked.id));
|
||||
await notifyNow(title, message);
|
||||
await recordAudit({ actor: req.currentUser!, category: "report", action: "send_monthly", targetType: "report", detail: { month: picked.id } });
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
import { Router } from "express";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { z } from "zod";
|
||||
import { db } from "../db/client.js";
|
||||
import { endpointChecks, integrations, servers, serviceLevels } from "../db/schema.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { levelStatuses, listLevels, MAX_LEVELS, MAX_TARGET, MIN_TARGET, SUBJECT_TYPES, type SubjectType } from "../services/serviceLevels.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const serviceLevelsRouter = Router();
|
||||
serviceLevelsRouter.use(requireAuth);
|
||||
|
||||
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
|
||||
|
||||
const levelSchema = z.object({
|
||||
name: z.string().trim().min(1).max(100),
|
||||
subjectType: z.enum(SUBJECT_TYPES),
|
||||
subjectId: z.number().int().positive(),
|
||||
target: z.number().min(MIN_TARGET, `A target is at least ${MIN_TARGET}%.`).max(MAX_TARGET, `A target is at most ${MAX_TARGET}% — 100% leaves no room for any downtime.`),
|
||||
alert: z.boolean().optional(),
|
||||
});
|
||||
|
||||
async function subjectExists(type: SubjectType, id: number): Promise<boolean> {
|
||||
const table = type === "endpoint" ? endpointChecks : type === "server" ? servers : integrations;
|
||||
return (await db.select({ id: table.id }).from(table).where(eq(table.id, id)).limit(1)).length > 0;
|
||||
}
|
||||
|
||||
const round3 = (n: number) => Math.round(n * 1000) / 1000;
|
||||
|
||||
/** Everything that can have a target, for the form. */
|
||||
async function subjects() {
|
||||
const [e, s, i] = await Promise.all([
|
||||
db.select({ id: endpointChecks.id, name: endpointChecks.name }).from(endpointChecks).orderBy(endpointChecks.name),
|
||||
db.select({ id: servers.id, name: servers.name }).from(servers).orderBy(servers.name),
|
||||
db.select({ id: integrations.id, name: integrations.name }).from(integrations).orderBy(integrations.name),
|
||||
]);
|
||||
return { endpoint: e, server: s, integration: i };
|
||||
}
|
||||
|
||||
// Every target, where this month's error budget stands and how the last months went: for everyone signed in, like the pages the figures come from.
|
||||
serviceLevelsRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
res.json({ levels: await levelStatuses(), subjects: await subjects() });
|
||||
}));
|
||||
|
||||
// Setting targets is an administrator's.
|
||||
serviceLevelsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const parsed = levelSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.error.issues[0]?.message ?? "That isn't a target.", details: parsed.error.flatten() });
|
||||
if ((await listLevels()).length >= MAX_LEVELS) return res.status(409).json({ error: "too_many", message: `At most ${MAX_LEVELS} targets.` });
|
||||
if (!(await subjectExists(parsed.data.subjectType, parsed.data.subjectId))) return res.status(400).json({ error: "unknown_subject", message: "That doesn't exist." });
|
||||
const [row] = await db
|
||||
.insert(serviceLevels)
|
||||
.values({ name: parsed.data.name, subjectType: parsed.data.subjectType, subjectId: parsed.data.subjectId, target: round3(parsed.data.target), alert: parsed.data.alert ?? true, createdBy: req.currentUser!.name ?? req.currentUser!.email ?? null })
|
||||
.returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "service_level", action: "create", targetType: "service_level", targetId: row.id, detail: { name: row.name, subjectType: row.subjectType, target: row.target } });
|
||||
invalidateAlertsCache();
|
||||
res.status(201).json({ id: row.id });
|
||||
}));
|
||||
|
||||
serviceLevelsRouter.put("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
const parsed = levelSchema.safeParse(req.body);
|
||||
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.success ? "That isn't a target." : (parsed.error.issues[0]?.message ?? "That isn't a target.") });
|
||||
const [before] = await db.select().from(serviceLevels).where(eq(serviceLevels.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
if (!(await subjectExists(parsed.data.subjectType, parsed.data.subjectId))) return res.status(400).json({ error: "unknown_subject", message: "That doesn't exist." });
|
||||
await db.update(serviceLevels).set({ name: parsed.data.name, subjectType: parsed.data.subjectType, subjectId: parsed.data.subjectId, target: round3(parsed.data.target), alert: parsed.data.alert ?? before.alert }).where(eq(serviceLevels.id, id));
|
||||
await recordAudit({ actor: req.currentUser!, category: "service_level", action: "update", targetType: "service_level", targetId: id, detail: { name: parsed.data.name, target: round3(parsed.data.target), ...(before.target !== parsed.data.target ? { targetFrom: before.target } : {}) } });
|
||||
invalidateAlertsCache();
|
||||
res.json({ ok: true });
|
||||
}));
|
||||
|
||||
serviceLevelsRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [before] = await db.select().from(serviceLevels).where(eq(serviceLevels.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
await db.delete(serviceLevels).where(eq(serviceLevels.id, id));
|
||||
await recordAudit({ actor: req.currentUser!, category: "service_level", action: "delete", targetType: "service_level", targetId: id, detail: { name: before.name } });
|
||||
invalidateAlertsCache();
|
||||
res.status(204).end();
|
||||
}));
|
||||
@@ -168,7 +168,7 @@ const updateSchema = z.object({
|
||||
.partial()
|
||||
.optional(),
|
||||
monthlyReport: z
|
||||
.object({ enabled: z.boolean(), day: z.number().int().min(1).max(28), time: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/) })
|
||||
.object({ enabled: z.boolean(), quarterly: z.boolean(), annual: z.boolean(), day: z.number().int().min(1).max(28), time: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/) })
|
||||
.partial()
|
||||
.optional(),
|
||||
});
|
||||
|
||||
@@ -73,6 +73,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
|
||||
npm: { label: "Nginx Proxy Manager", link: "/proxy-hosts" },
|
||||
homeassistant: { label: "Home Assistant", link: "/home-assistant" },
|
||||
ups: { label: "UPS", link: "/ups" },
|
||||
slo: { label: "Service level", link: "/service-levels" },
|
||||
secrets: { label: "Secrets", link: "/secrets" },
|
||||
domains: { label: "Domains", link: "/dns?view=domains" },
|
||||
endpoint: { label: "Endpoint", link: "/endpoints" },
|
||||
@@ -105,6 +106,8 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
|
||||
if (m) return { type: m[1], id: Number(m[2]), category: "automation" };
|
||||
m = /^ups-(?:battery|low|replace|load):(\d+)$/.exec(key);
|
||||
if (m) return { type: "ups", id: Number(m[1]), category: "monitoring" };
|
||||
m = /^slo:(\d+)$/.exec(key);
|
||||
if (m) return { type: "slo", id: Number(m[1]), category: "monitoring" };
|
||||
return { type: "server", id: null, category: "disk" };
|
||||
}
|
||||
|
||||
@@ -227,12 +230,13 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
type: origin.type,
|
||||
message: c.message,
|
||||
key: c.key,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : undefined,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : undefined,
|
||||
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
|
||||
});
|
||||
}
|
||||
// Integrations the check couldn't read this time. (A whole-integration entry covers its nodes, so skip those.)
|
||||
if (held.has("server:rules")) cannotCheck("Alert rules", "couldn't be judged — see the server log", "Server and storage health");
|
||||
if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health");
|
||||
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
|
||||
for (const h of held) {
|
||||
const m = /^(proxmox|synology|unifi|ups):(\d+)(?::(.+))?$/.exec(h);
|
||||
|
||||
@@ -41,6 +41,7 @@ const KIND_OF_CATEGORY: Record<string, ChangeKind> = {
|
||||
metrics_token: "settings",
|
||||
api_token: "settings",
|
||||
alert_rule: "alerts",
|
||||
service_level: "endpoints",
|
||||
todo: "other",
|
||||
note: "servers",
|
||||
webhook: "settings",
|
||||
@@ -181,6 +182,12 @@ export function describeChange(category: string, action: string, detail: Record<
|
||||
return { title: detail.change === "undo" ? "A change to several pieces of hardware was undone" : `${typeof detail.changed === "number" ? detail.changed : "Several"} piece${detail.changed === 1 ? "" : "s"} of hardware changed at once`, tone: "neutral", detail: genericSummary(detail, ["ids", "changed", "change"]) };
|
||||
case "webhook/retry":
|
||||
return { title: `A webhook message was sent again${name ? `: ${name}` : str(detail.label) ? `: ${str(detail.label)}` : ""}`, tone: detail.ok === false ? "bad" : "neutral", detail: str(detail.event) };
|
||||
case "service_level/create":
|
||||
return { title: `Service-level target set${name ? `: ${name}` : ""}`, tone: "neutral", detail: typeof detail.target === "number" ? `${detail.target}%` : null };
|
||||
case "service_level/update":
|
||||
return { title: `Service-level target changed${name ? `: ${name}` : ""}`, tone: "neutral", detail: typeof detail.target === "number" ? `${typeof detail.targetFrom === "number" ? `${detail.targetFrom}% → ` : ""}${detail.target}%` : null };
|
||||
case "service_level/delete":
|
||||
return { title: `Service-level target removed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
|
||||
case "alert_rule/create":
|
||||
return { title: `Alert rule added${name ? `: ${name}` : ""}`, tone: "neutral", detail: genericSummary(detail) };
|
||||
case "alert_rule/update":
|
||||
|
||||
@@ -17,6 +17,7 @@ import { parseJson, type AgentPort } from "./agentPorts.js";
|
||||
import { diskLimit, parseThresholds, type ServerThresholds } from "./serverThresholds.js";
|
||||
import { computeRuleFindings, findingKey, listRules, type RuleFinding } from "./alertRules.js";
|
||||
import { baselineKey, computeBaselineFindings, type BaselineFinding } from "./baselines.js";
|
||||
import { levelFindings, levelKey, type LevelFinding } from "./serviceLevels.js";
|
||||
import { recordStorageSamples } from "./storageHistory.js";
|
||||
import { isUpsFresh, upsProblems, upsSyncStatuses, type UpsStateRow } from "./upsSync.js";
|
||||
|
||||
@@ -81,6 +82,8 @@ export interface HealthSnapshot {
|
||||
rules?: RuleFinding[];
|
||||
/** What is clearly higher than usual for this hour of the week, from each server's own history (services/baselines.ts). */
|
||||
baselines?: BaselineFinding[];
|
||||
/** Service-level targets whose error budget for the month is mostly or wholly used (services/serviceLevels.ts). */
|
||||
levels?: LevelFinding[];
|
||||
}
|
||||
export interface Thresholds {
|
||||
serverOfflineMinutes: number;
|
||||
@@ -183,6 +186,9 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
}
|
||||
}
|
||||
|
||||
// A service-level target that has used most of its month's error budget, or all of it.
|
||||
for (const f of snapshot.levels ?? []) out.push({ key: levelKey(f), source: "slo", message: f.message, severity: f.severity });
|
||||
|
||||
for (const u of snapshot.unifi ?? []) {
|
||||
const limitMs = (thresholds.deviceOfflineMinutes ?? 10) * 60_000;
|
||||
for (const d of u.devices) {
|
||||
@@ -402,6 +408,14 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
}
|
||||
}
|
||||
|
||||
// Service-level targets: how much of the month's error budget is used. A failure here costs only this check.
|
||||
try {
|
||||
snapshot.levels = await levelFindings();
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't judge the service-level targets:", err instanceof Error ? err.message : err);
|
||||
held.add("slo");
|
||||
}
|
||||
|
||||
const rows = await db
|
||||
.select({ id: integrations.id, name: integrations.name, type: integrations.type })
|
||||
.from(integrations)
|
||||
|
||||
@@ -7,6 +7,7 @@ import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState } from "../db/schema.js";
|
||||
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
|
||||
import { levelStatuses } from "./serviceLevels.js";
|
||||
import { latestTests } from "./restoreTests.js";
|
||||
import { heartbeatState, parseStamp } from "./heartbeats.js";
|
||||
import { env } from "../env.js";
|
||||
@@ -58,6 +59,17 @@ class Exposition {
|
||||
}
|
||||
}
|
||||
|
||||
/** Each service-level target, for /metrics: its target, how it is doing this month and how much error budget is left. */
|
||||
async function addServiceLevelMetrics(out: Exposition, now: Date) {
|
||||
for (const s of await levelStatuses(now, { history: false })) {
|
||||
const labels = { name: s.name, subject: s.subjectName ?? "", type: s.subjectType };
|
||||
out.add("homelab_service_level_target_percent", "gauge", "The availability a service-level target asks for.", s.target, labels);
|
||||
if (s.measure.percent === null) continue;
|
||||
out.add("homelab_service_level_availability_percent", "gauge", "How available the thing has been so far this calendar month.", s.measure.percent, labels);
|
||||
out.add("homelab_service_level_error_budget_remaining_minutes", "gauge", "Downtime still allowed this month before the target is missed (negative once it is).", Math.round(s.budget.remainingMinutes * 10) / 10, labels);
|
||||
}
|
||||
}
|
||||
|
||||
/** Each UPS, for /metrics: whether it is on battery, how full the battery is, how long it would last and how loaded it is. */
|
||||
async function addUpsMetrics(out: Exposition, now: Date) {
|
||||
const fresh = new Set<number>();
|
||||
@@ -239,6 +251,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
out.add("homelab_scrape_timestamp_seconds", "gauge", "When this answer was built, as a Unix time.", nowSec);
|
||||
await addEndpointMetrics(out, now);
|
||||
await addUpsMetrics(out, now);
|
||||
await addServiceLevelMetrics(out, now);
|
||||
await addHeartbeatMetrics(out, now);
|
||||
return out.toString();
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
/**
|
||||
* The monthly report: how a month went — availability of servers, integrations and endpoints, incidents and how long they took, updates applied (by hand and by
|
||||
* The monthly report (and, by the same figures over longer stretches, the quarterly and annual ones): how a month went — availability of servers, integrations and endpoints, incidents and how long they took, updates applied (by hand and by
|
||||
* patch plans), restore tests and the state of the backups, and what was renewed. Everything in it the app already knows; this gathers it for one calendar month
|
||||
* (on the clock of the notification time zone) into one structure that the page shows, prints and exports, and into a plain-text version that can be sent like the
|
||||
* weekly digest.
|
||||
@@ -18,6 +18,7 @@ import { renewalsBetween } from "./renewals.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { getUptime } from "./uptimeHistory.js";
|
||||
import { spanWords } from "./weeklyDigest.js";
|
||||
import { levelResultsFor, type PeriodResult } from "./serviceLevels.js";
|
||||
|
||||
const DAY_MS = 86_400_000;
|
||||
const MAX_LISTED = 12;
|
||||
@@ -38,10 +39,14 @@ export interface EndpointRow {
|
||||
checks: number;
|
||||
}
|
||||
|
||||
export type PeriodKind = "month" | "quarter" | "year";
|
||||
|
||||
export interface MonthlyReport {
|
||||
/** "2026-09". */
|
||||
/** What stretch it is about: a month (id "2026-09"), a quarter ("2026-Q3") or a year ("2026"). */
|
||||
period: PeriodKind;
|
||||
/** The id of that stretch. (Named for the month, which is what it was made for.) */
|
||||
month: string;
|
||||
/** "September 2026". */
|
||||
/** "September 2026", "Q3 2026", "2026". */
|
||||
label: string;
|
||||
/** ISO — the start of the month and its end (or now, for a month that is not over). */
|
||||
from: string;
|
||||
@@ -52,6 +57,8 @@ export interface MonthlyReport {
|
||||
servers: AvailabilityRow[];
|
||||
integrations: AvailabilityRow[];
|
||||
endpoints: EndpointRow[];
|
||||
/** How each service-level target did over the stretch (those with a figure for it). */
|
||||
serviceLevels: PeriodResult[];
|
||||
incidents: {
|
||||
began: number;
|
||||
resolved: number;
|
||||
@@ -94,6 +101,17 @@ function monthOf(now: Date, tz: string): { year: number; month: number } {
|
||||
}
|
||||
|
||||
export const monthId = (year: number, month: number) => `${year}-${pad(month)}`;
|
||||
|
||||
export function isQuarterId(v: unknown): boolean {
|
||||
return typeof v === "string" && /^\d{4}-Q[1-4]$/.test(v);
|
||||
}
|
||||
export function isYearId(v: unknown): boolean {
|
||||
return typeof v === "string" && /^\d{4}$/.test(v);
|
||||
}
|
||||
export const isPeriodId = (v: unknown): v is string => isMonthId(v) || isQuarterId(v) || isYearId(v);
|
||||
export function periodKind(id: string): PeriodKind {
|
||||
return isQuarterId(id) ? "quarter" : isYearId(id) ? "year" : "month";
|
||||
}
|
||||
const shift = (year: number, month: number, by: number) => {
|
||||
const index = year * 12 + (month - 1) + by;
|
||||
return { year: Math.floor(index / 12), month: (index % 12) + 1 };
|
||||
@@ -121,6 +139,61 @@ export function recentMonths(now: Date, tz: string, count = 12): { id: string; l
|
||||
});
|
||||
}
|
||||
|
||||
export function periodLabel(id: string): string {
|
||||
if (isQuarterId(id)) return `Q${id.slice(6)} ${id.slice(0, 4)}`;
|
||||
return isYearId(id) ? id : monthLabel(id);
|
||||
}
|
||||
|
||||
/** The year and quarter (1–4) `now` is in. */
|
||||
function quarterOf(now: Date, tz: string): { year: number; quarter: number } {
|
||||
const { year, month } = monthOf(now, tz);
|
||||
return { year, quarter: Math.ceil(month / 3) };
|
||||
}
|
||||
|
||||
/** The last whole quarter ("2026-Q2" in July), and the last whole year. */
|
||||
export function lastFullQuarter(now: Date, tz: string): string {
|
||||
const { year, quarter } = quarterOf(now, tz);
|
||||
return quarter === 1 ? `${year - 1}-Q4` : `${year}-Q${quarter - 1}`;
|
||||
}
|
||||
export const lastFullYear = (now: Date, tz: string) => String(monthOf(now, tz).year - 1);
|
||||
|
||||
/** What can be asked for: this month and eleven before it, this quarter and seven before it, this year and two before it — newest first. */
|
||||
export function recentPeriods(now: Date, tz: string): { months: { id: string; label: string }[]; quarters: { id: string; label: string }[]; years: { id: string; label: string }[] } {
|
||||
const { year, quarter } = quarterOf(now, tz);
|
||||
const quarters = Array.from({ length: 8 }, (_, i) => {
|
||||
const index = year * 4 + (quarter - 1) - i;
|
||||
const id = `${Math.floor(index / 4)}-Q${(index % 4) + 1}`;
|
||||
return { id, label: periodLabel(id) };
|
||||
});
|
||||
const years = Array.from({ length: 3 }, (_, i) => ({ id: String(year - i), label: String(year - i) }));
|
||||
return { months: recentMonths(now, tz), quarters, years };
|
||||
}
|
||||
|
||||
export interface PeriodBounds {
|
||||
start: Date;
|
||||
end: Date;
|
||||
/** The first UTC day of it and of what follows, as "YYYY-MM-DD", for the figures kept per UTC day. */
|
||||
firstDay: string;
|
||||
nextFirstDay: string;
|
||||
}
|
||||
|
||||
/** The instants a month, quarter or year begins and ends at, on the clock of `tz`. */
|
||||
export function periodBounds(id: string, tz: string): PeriodBounds {
|
||||
if (isYearId(id)) {
|
||||
const y = Number(id);
|
||||
return { start: zonedTime(y, 1, 1, 0, 0, tz), end: zonedTime(y + 1, 1, 1, 0, 0, tz), firstDay: `${y}-01-01`, nextFirstDay: `${y + 1}-01-01` };
|
||||
}
|
||||
if (isQuarterId(id)) {
|
||||
const y = Number(id.slice(0, 4));
|
||||
const first = (Number(id.slice(6)) - 1) * 3 + 1;
|
||||
const next = shift(y, first, 3);
|
||||
return { start: zonedTime(y, first, 1, 0, 0, tz), end: zonedTime(next.year, next.month, 1, 0, 0, tz), firstDay: `${y}-${pad(first)}-01`, nextFirstDay: `${monthId(next.year, next.month)}-01` };
|
||||
}
|
||||
const [y, m] = id.split("-").map(Number);
|
||||
const next = shift(y, m, 1);
|
||||
return { ...monthBounds(id, tz), firstDay: `${id}-01`, nextFirstDay: `${monthId(next.year, next.month)}-01` };
|
||||
}
|
||||
|
||||
/** The instants a month begins and ends at, on the clock of `tz`. */
|
||||
export function monthBounds(id: string, tz: string): { start: Date; end: Date } {
|
||||
const [y, m] = id.split("-").map(Number);
|
||||
@@ -135,7 +208,8 @@ const round1 = (n: number) => Math.round(n * 10) / 10;
|
||||
export async function buildMonthlyReport(id: string, now: Date = new Date()): Promise<MonthlyReport> {
|
||||
const settings = await getSettings();
|
||||
const tz = settings.notifications.timezone || "UTC";
|
||||
const { start, end: fullEnd } = monthBounds(id, tz);
|
||||
const bounds = periodBounds(id, tz);
|
||||
const { start, end: fullEnd } = bounds;
|
||||
const partial = fullEnd.getTime() > now.getTime();
|
||||
const end = partial ? now : fullEnd;
|
||||
const days = Math.max(1, Math.ceil((end.getTime() - start.getTime()) / DAY_MS));
|
||||
@@ -154,10 +228,8 @@ export async function buildMonthlyReport(id: string, now: Date = new Date()): Pr
|
||||
|
||||
// Endpoints: from the daily figures, for the days the month covers. The figures are per UTC day, so a day at either end can belong to a neighbouring month
|
||||
// by a few hours (more than the time zone is ahead of UTC); a month that is not over takes in today.
|
||||
const [yy, mm] = id.split("-").map(Number);
|
||||
const following = shift(yy, mm, 1);
|
||||
const firstDay = `${id}-01`;
|
||||
const lastDayExclusive = partial ? new Date(now.getTime() + DAY_MS).toISOString().slice(0, 10) : `${monthId(following.year, following.month)}-01`;
|
||||
const firstDay = bounds.firstDay;
|
||||
const lastDayExclusive = partial ? new Date(now.getTime() + DAY_MS).toISOString().slice(0, 10) : bounds.nextFirstDay;
|
||||
const dailyRows = await db.select().from(endpointDaily).where(and(gte(endpointDaily.day, firstDay), lt(endpointDaily.day, lastDayExclusive)));
|
||||
const checkNames = new Map((await db.select({ id: endpointChecks.id, name: endpointChecks.name }).from(endpointChecks)).map((c) => [c.id, c.name]));
|
||||
const perCheck = new Map<number, { total: number; ok: number; msSum: number; msCount: number }>();
|
||||
@@ -175,7 +247,7 @@ export async function buildMonthlyReport(id: string, now: Date = new Date()): Pr
|
||||
.sort((a, b) => (a.percent ?? 101) - (b.percent ?? 101) || a.name.localeCompare(b.name));
|
||||
|
||||
// Incidents.
|
||||
const all = await listIncidents({ now: end, limit: 3000 });
|
||||
const all = await listIncidents({ now: end, limit: 20000 });
|
||||
const inMonth = (iso: string) => iso >= startIso && iso < endIso;
|
||||
const began = all.filter((i) => inMonth(i.openedAt));
|
||||
const resolved = all.filter((i) => i.resolvedAt !== null && inMonth(i.resolvedAt));
|
||||
@@ -209,8 +281,9 @@ export async function buildMonthlyReport(id: string, now: Date = new Date()): Pr
|
||||
const renewalRows = await renewalsBetween(start, end);
|
||||
|
||||
return {
|
||||
period: periodKind(id),
|
||||
month: id,
|
||||
label: monthLabel(id),
|
||||
label: periodLabel(id),
|
||||
from: startIso,
|
||||
to: endIso,
|
||||
partial,
|
||||
@@ -218,6 +291,7 @@ export async function buildMonthlyReport(id: string, now: Date = new Date()): Pr
|
||||
servers: rowsOf(uptime.servers, serverNames),
|
||||
integrations: rowsOf(uptime.integrations, integrationNames),
|
||||
endpoints: endpointRows,
|
||||
serviceLevels: await levelResultsFor(start, fullEnd, now),
|
||||
incidents: {
|
||||
began: began.length,
|
||||
resolved: resolved.length,
|
||||
@@ -274,6 +348,13 @@ export function monthlyReportText(r: MonthlyReport): { title: string; message: s
|
||||
if (integrationAvg !== null) avail.push(`Integrations: ${integrationAvg}% on average.`);
|
||||
lines("Availability", avail);
|
||||
|
||||
if (r.serviceLevels.length > 0) {
|
||||
const missed = r.serviceLevels.filter((l) => !l.met);
|
||||
const body = [`${r.serviceLevels.length - missed.length} of ${plural(r.serviceLevels.length, "target")} met${r.partial ? " so far" : ""}.`];
|
||||
for (const l of r.serviceLevels.slice(0, 8)) body.push(`- ${l.name}: ${l.percent}% against ${l.target}% — ${l.met ? "met" : "MISSED"} (down ${l.downMinutes > 0 ? spanWords(l.downMinutes) : "not at all"}, ${l.allowedMinutes >= 1 ? spanWords(l.allowedMinutes) : "under a minute"} allowed)`);
|
||||
lines("Service levels", body);
|
||||
}
|
||||
|
||||
const inc = r.incidents;
|
||||
if (inc.began > 0 || inc.resolved > 0 || inc.stillOpen > 0) {
|
||||
const body = [`${plural(inc.began, "incident")} began, ${inc.resolved} ended, ${inc.stillOpen} still open.`];
|
||||
@@ -298,6 +379,7 @@ export function monthlyReportText(r: MonthlyReport): { title: string; message: s
|
||||
|
||||
lines("Renewed", r.renewals.slice(0, 10).map((x) => `- ${x.kind === "domain" ? "Domain" : "Certificate"} ${x.subject}: ${x.fromDate} → ${x.toDate}`));
|
||||
|
||||
const title = `Homelab monthly report — ${r.label}${r.partial ? " (so far)" : ""}`;
|
||||
return { title, message: sections.length > 0 ? sections.join("\n\n") : "Nothing was recorded for this month." };
|
||||
const noun = r.period === "year" ? "annual" : r.period === "quarter" ? "quarterly" : "monthly";
|
||||
const title = `Homelab ${noun} report — ${r.label}${r.partial ? " (so far)" : ""}`;
|
||||
return { title, message: sections.length > 0 ? sections.join("\n\n") : `Nothing was recorded for this ${r.period}.` };
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
import schedule from "node-schedule";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { notify } from "./notify.js";
|
||||
import { buildMonthlyReport, lastFullMonth, monthlyReportText } from "./monthlyReport.js";
|
||||
import { buildMonthlyReport, lastFullMonth, lastFullQuarter, lastFullYear, monthlyReportText } from "./monthlyReport.js";
|
||||
|
||||
let currentJob: schedule.Job | null = null;
|
||||
|
||||
@@ -12,11 +12,25 @@ export function cronForMonthlyReport(day: number, time: string): string {
|
||||
return `${Number.isFinite(m) ? m : 0} ${Number.isFinite(h) ? h : 8} ${d} * *`;
|
||||
}
|
||||
|
||||
/** Builds last month's report and sends it through the channels (held with other notifications during quiet hours). */
|
||||
/**
|
||||
* Builds last month's report and sends it through the channels (held with other notifications during quiet hours). When that month closed a quarter and the quarterly
|
||||
* report is on, the quarter's report follows it; when it closed the year and the annual one is on, the year's.
|
||||
*/
|
||||
export async function sendMonthlyReport(now: Date = new Date()): Promise<{ title: string; message: string }> {
|
||||
const tz = (await getSettings()).notifications.timezone || "UTC";
|
||||
const text = monthlyReportText(await buildMonthlyReport(lastFullMonth(now, tz), now));
|
||||
const settings = await getSettings();
|
||||
const tz = settings.notifications.timezone || "UTC";
|
||||
const month = lastFullMonth(now, tz);
|
||||
const text = monthlyReportText(await buildMonthlyReport(month, now));
|
||||
await notify(text.title, text.message);
|
||||
const closing = Number(month.slice(5));
|
||||
if (settings.monthlyReport.quarterly && closing % 3 === 0) {
|
||||
const q = monthlyReportText(await buildMonthlyReport(lastFullQuarter(now, tz), now));
|
||||
await notify(q.title, q.message);
|
||||
}
|
||||
if (settings.monthlyReport.annual && closing === 12) {
|
||||
const y = monthlyReportText(await buildMonthlyReport(lastFullYear(now, tz), now));
|
||||
await notify(y.title, y.message);
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,244 @@
|
||||
/**
|
||||
* Service-level targets: "99.9 % for Gitea", and what is left of the error budget this month. A target is set on something the app already keeps the record of — an
|
||||
* endpoint check, a server, an integration — and is measured over calendar months (on the clock of the notification time zone):
|
||||
*
|
||||
* allowed downtime = minutes in the month × (1 − target) — that is the error budget
|
||||
* used = minutes the thing was down so far
|
||||
* left = allowed − used
|
||||
*
|
||||
* An endpoint's downtime is worked out from its daily check figures (each day's share of failed checks, over the part of the day that is within the period); a server's and an integration's
|
||||
* from the moments the audit log recorded them going away and coming back, to the minute. Planned maintenance is *not* taken out: down is down. Nothing is guessed — a
|
||||
* thing with no record for the period has no figure, and says so.
|
||||
*
|
||||
* A target whose budget is mostly used (80 %) is a warning, and one that is overspent a critical alert, until the month turns over and the budget starts again.
|
||||
*/
|
||||
import { and, gte, lt } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { endpointChecks, endpointDaily, integrations, servers, serviceLevels } from "../db/schema.js";
|
||||
import { monthBounds, recentMonths } from "./monthlyReport.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { getUptimeRange, type UptimeRange } from "./uptimeHistory.js";
|
||||
import { spanWords } from "./weeklyDigest.js";
|
||||
|
||||
export const SUBJECT_TYPES = ["endpoint", "server", "integration"] as const;
|
||||
export type SubjectType = (typeof SUBJECT_TYPES)[number];
|
||||
export const MAX_LEVELS = 50;
|
||||
export const MIN_TARGET = 90;
|
||||
export const MAX_TARGET = 99.999;
|
||||
/** The share of the budget used at which it is a warning. */
|
||||
export const WARN_AT_PERCENT = 80;
|
||||
export const HISTORY_MONTHS = 6;
|
||||
|
||||
const DAY_MS = 86_400_000;
|
||||
|
||||
export type LevelRow = typeof serviceLevels.$inferSelect;
|
||||
export type BudgetState = "ok" | "at_risk" | "breached" | "no_data";
|
||||
|
||||
export interface Measure {
|
||||
/** Share of the known time spent up, in percent (three decimals); null when nothing is known. */
|
||||
percent: number | null;
|
||||
downMinutes: number;
|
||||
knownMinutes: number;
|
||||
}
|
||||
|
||||
export interface Budget {
|
||||
allowedMinutes: number;
|
||||
/** Of the budget, in percent (can pass 100). Null without a figure. */
|
||||
usedPercent: number | null;
|
||||
/** Negative when overspent. */
|
||||
remainingMinutes: number;
|
||||
state: BudgetState;
|
||||
/** Of the budget, at the pace so far, by the end of the period; null before a day has passed or without a figure. */
|
||||
projectedPercent: number | null;
|
||||
}
|
||||
|
||||
const round3 = (n: number) => Math.round(n * 1000) / 1000;
|
||||
|
||||
/** Pure: the error budget of a target over a period of `totalMinutes`, and where it stands after `elapsedMinutes` with `downMinutes` used. */
|
||||
export function judgeBudget(target: number, totalMinutes: number, elapsedMinutes: number, measure: Measure): Budget {
|
||||
const allowed = totalMinutes * (1 - target / 100);
|
||||
if (measure.percent === null) return { allowedMinutes: allowed, usedPercent: null, remainingMinutes: allowed, state: "no_data", projectedPercent: null };
|
||||
const used = measure.downMinutes;
|
||||
const usedPercent = allowed > 0 ? (used / allowed) * 100 : used > 0 ? Infinity : 0;
|
||||
const state: BudgetState = used > allowed ? "breached" : usedPercent >= WARN_AT_PERCENT ? "at_risk" : "ok";
|
||||
const projected = elapsedMinutes >= 1440 && allowed > 0 ? ((used / elapsedMinutes) * totalMinutes / allowed) * 100 : null;
|
||||
return { allowedMinutes: allowed, usedPercent: Number.isFinite(usedPercent) ? Math.round(usedPercent * 10) / 10 : 999, remainingMinutes: allowed - used, state, projectedPercent: projected === null ? null : Math.round(projected) };
|
||||
}
|
||||
|
||||
/** A budget in words: "43 min", "4 h". Below a minute it is "under a minute". */
|
||||
export const budgetWords = (minutes: number) => (minutes < 1 ? "under a minute" : spanWords(minutes));
|
||||
|
||||
/** Pure: what an alert about a budget says. */
|
||||
export function budgetMessage(name: string, target: number, periodLabel: string, b: Budget, m: Measure): string {
|
||||
if (b.state === "breached") return `${name} has missed its ${target}% target for ${periodLabel}: down ${budgetWords(m.downMinutes)}, ${budgetWords(b.allowedMinutes)} allowed`;
|
||||
return `${name}: ${Math.round(b.usedPercent ?? 0)}% of the error budget for its ${target}% target is used up for ${periodLabel} — down ${budgetWords(m.downMinutes)} of ${budgetWords(b.allowedMinutes)} allowed, ${budgetWords(b.remainingMinutes)} left`;
|
||||
}
|
||||
|
||||
// ─── Measuring ──────────────────────────────────────────────────────────────
|
||||
|
||||
/** What is shared between the measurements of one request, so each stretch of the audit log and of the daily figures is read once. */
|
||||
export interface MeasureContext {
|
||||
now: Date;
|
||||
uptime: Map<string, Promise<UptimeRange>>;
|
||||
daily: Map<string, Promise<{ checkId: number; day: string; total: number; ok: number }[]>>;
|
||||
}
|
||||
|
||||
export const newContext = (now: Date = new Date()): MeasureContext => ({ now, uptime: new Map(), daily: new Map() });
|
||||
|
||||
const dayOf = (ms: number) => new Date(ms).toISOString().slice(0, 10);
|
||||
|
||||
/** How a thing did between two moments (the end is clipped to now). */
|
||||
export async function measure(type: SubjectType, subjectId: number, start: Date, end: Date, ctx: MeasureContext): Promise<Measure> {
|
||||
const to = Math.min(end.getTime(), ctx.now.getTime());
|
||||
const from = start.getTime();
|
||||
if (to <= from) return { percent: null, downMinutes: 0, knownMinutes: 0 };
|
||||
if (type === "endpoint") {
|
||||
const key = `${dayOf(from)}|${dayOf(to + DAY_MS - 1)}`;
|
||||
let rows = ctx.daily.get(key);
|
||||
if (!rows) {
|
||||
rows = (async () => db.select({ checkId: endpointDaily.checkId, day: endpointDaily.day, total: endpointDaily.total, ok: endpointDaily.ok }).from(endpointDaily).where(and(gte(endpointDaily.day, dayOf(from)), lt(endpointDaily.day, dayOf(to + DAY_MS - 1)))))();
|
||||
ctx.daily.set(key, rows);
|
||||
}
|
||||
// Day by day: the share of that day's checks that failed, over the part of the day that is within the period. Days with no checks are not known, so they count neither way.
|
||||
let known = 0;
|
||||
let down = 0;
|
||||
for (const r of await rows) {
|
||||
if (r.checkId !== subjectId || r.total <= 0) continue;
|
||||
const dayStart = Date.parse(`${r.day}T00:00:00Z`);
|
||||
const overlap = Math.min(to, dayStart + DAY_MS) - Math.max(from, dayStart);
|
||||
if (overlap <= 0) continue;
|
||||
known += overlap / 60_000;
|
||||
down += (1 - r.ok / r.total) * (overlap / 60_000);
|
||||
}
|
||||
if (known <= 0) return { percent: null, downMinutes: 0, knownMinutes: 0 };
|
||||
return { percent: round3((1 - down / known) * 100), downMinutes: Math.round(down), knownMinutes: Math.round(known) };
|
||||
}
|
||||
const key = `${from}|${to}`;
|
||||
let range = ctx.uptime.get(key);
|
||||
if (!range) {
|
||||
range = getUptimeRange(start, new Date(to), ctx.now);
|
||||
ctx.uptime.set(key, range);
|
||||
}
|
||||
const t = (await range)[type === "server" ? "servers" : "integrations"][subjectId];
|
||||
if (!t || t.known <= 0) return { percent: null, downMinutes: 0, knownMinutes: 0 };
|
||||
return { percent: round3((1 - t.down / t.known) * 100), downMinutes: t.down, knownMinutes: t.known };
|
||||
}
|
||||
|
||||
export interface LevelStatus {
|
||||
id: number;
|
||||
name: string;
|
||||
subjectType: SubjectType;
|
||||
subjectId: number;
|
||||
/** The name of the thing measured; null when it has been removed. */
|
||||
subjectName: string | null;
|
||||
target: number;
|
||||
alert: boolean;
|
||||
createdBy: string | null;
|
||||
period: { id: string; label: string; from: string; to: string; totalMinutes: number; elapsedMinutes: number };
|
||||
measure: Measure;
|
||||
budget: Budget;
|
||||
/** The months before this one, newest first. */
|
||||
history: { id: string; label: string; percent: number | null; downMinutes: number; allowedMinutes: number; met: boolean | null }[];
|
||||
}
|
||||
|
||||
async function subjectNames(): Promise<Record<SubjectType, Map<number, string>>> {
|
||||
const [e, s, i] = await Promise.all([db.select({ id: endpointChecks.id, name: endpointChecks.name }).from(endpointChecks), db.select({ id: servers.id, name: servers.name }).from(servers), db.select({ id: integrations.id, name: integrations.name }).from(integrations)]);
|
||||
return { endpoint: new Map(e.map((r) => [r.id, r.name])), server: new Map(s.map((r) => [r.id, r.name])), integration: new Map(i.map((r) => [r.id, r.name])) };
|
||||
}
|
||||
|
||||
export async function listLevels(): Promise<LevelRow[]> {
|
||||
return (await db.select().from(serviceLevels)).sort((a, b) => a.name.localeCompare(b.name) || a.id - b.id);
|
||||
}
|
||||
|
||||
/** Every target, where this month's budget stands, and how each of the last months went. */
|
||||
export async function levelStatuses(now: Date = new Date(), options: { history?: boolean } = {}): Promise<LevelStatus[]> {
|
||||
const rows = await listLevels();
|
||||
if (rows.length === 0) return [];
|
||||
const tz = (await getSettings()).notifications.timezone || "UTC";
|
||||
const months = recentMonths(now, tz, HISTORY_MONTHS + 1);
|
||||
const current = months[0].id;
|
||||
const { start, end } = monthBounds(current, tz);
|
||||
const totalMinutes = (end.getTime() - start.getTime()) / 60_000;
|
||||
const elapsedMinutes = Math.max(0, (Math.min(now.getTime(), end.getTime()) - start.getTime()) / 60_000);
|
||||
const names = await subjectNames();
|
||||
const ctx = newContext(now);
|
||||
const out: LevelStatus[] = [];
|
||||
for (const r of rows) {
|
||||
const type = r.subjectType as SubjectType;
|
||||
const m = await measure(type, r.subjectId, start, end, ctx);
|
||||
const history: LevelStatus["history"] = [];
|
||||
if (options.history !== false) {
|
||||
for (const past of months.slice(1)) {
|
||||
const b = monthBounds(past.id, tz);
|
||||
const total = (b.end.getTime() - b.start.getTime()) / 60_000;
|
||||
const pm = await measure(type, r.subjectId, b.start, b.end, ctx);
|
||||
const allowed = total * (1 - r.target / 100);
|
||||
history.push({ id: past.id, label: past.label, percent: pm.percent, downMinutes: pm.downMinutes, allowedMinutes: allowed, met: pm.percent === null ? null : pm.downMinutes <= allowed });
|
||||
}
|
||||
}
|
||||
out.push({
|
||||
id: r.id,
|
||||
name: r.name,
|
||||
subjectType: type,
|
||||
subjectId: r.subjectId,
|
||||
subjectName: names[type].get(r.subjectId) ?? null,
|
||||
target: r.target,
|
||||
alert: r.alert,
|
||||
createdBy: r.createdBy,
|
||||
period: { id: current, label: months[0].label, from: start.toISOString(), to: end.toISOString(), totalMinutes: Math.round(totalMinutes), elapsedMinutes: Math.round(elapsedMinutes) },
|
||||
measure: m,
|
||||
budget: judgeBudget(r.target, totalMinutes, elapsedMinutes, m),
|
||||
history,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export interface LevelFinding {
|
||||
levelId: number;
|
||||
name: string;
|
||||
state: "at_risk" | "breached";
|
||||
severity: "warning" | "critical";
|
||||
message: string;
|
||||
}
|
||||
|
||||
export const levelKey = (f: Pick<LevelFinding, "levelId">) => `slo:${f.levelId}`;
|
||||
|
||||
/** The targets whose budget is mostly or wholly used this month, for the ones that are to alert. */
|
||||
export async function levelFindings(now: Date = new Date()): Promise<LevelFinding[]> {
|
||||
const out: LevelFinding[] = [];
|
||||
for (const s of await levelStatuses(now, { history: false })) {
|
||||
if (!s.alert || (s.budget.state !== "at_risk" && s.budget.state !== "breached")) continue;
|
||||
out.push({ levelId: s.id, name: s.name, state: s.budget.state, severity: s.budget.state === "breached" ? "critical" : "warning", message: budgetMessage(s.name, s.target, s.period.label, s.budget, s.measure) });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export interface PeriodResult {
|
||||
id: number;
|
||||
name: string;
|
||||
subjectType: SubjectType;
|
||||
target: number;
|
||||
percent: number | null;
|
||||
downMinutes: number;
|
||||
/** Over the whole period (even one that is not over yet). */
|
||||
allowedMinutes: number;
|
||||
usedPercent: number | null;
|
||||
/** Whether it stayed within its budget; null without a figure. For a period not over, "so far". */
|
||||
met: boolean | null;
|
||||
}
|
||||
|
||||
/** How each target did over a period (a month, a quarter, a year): for the reports. */
|
||||
export async function levelResultsFor(start: Date, end: Date, now: Date = new Date()): Promise<PeriodResult[]> {
|
||||
const rows = await listLevels();
|
||||
const ctx = newContext(now);
|
||||
const total = (end.getTime() - start.getTime()) / 60_000;
|
||||
const out: PeriodResult[] = [];
|
||||
for (const r of rows) {
|
||||
const m = await measure(r.subjectType as SubjectType, r.subjectId, start, end, ctx);
|
||||
if (m.percent === null) continue;
|
||||
const b = judgeBudget(r.target, total, (Math.min(now.getTime(), end.getTime()) - start.getTime()) / 60_000, m);
|
||||
out.push({ id: r.id, name: r.name, subjectType: r.subjectType as SubjectType, target: r.target, percent: m.percent, downMinutes: m.downMinutes, allowedMinutes: Math.round(b.allowedMinutes * 10) / 10, usedPercent: b.usedPercent, met: m.downMinutes <= b.allowedMinutes });
|
||||
}
|
||||
return out.sort((a, b) => Number(a.met) - Number(b.met) || (b.usedPercent ?? 0) - (a.usedPercent ?? 0) || a.name.localeCompare(b.name));
|
||||
}
|
||||
@@ -128,6 +128,10 @@ export interface HealthCheckSettings {
|
||||
/** The monthly report: last month's availability, incidents, updates and renewals, in one message on a day of the month. Off until switched on. */
|
||||
export interface MonthlyReportSettings {
|
||||
enabled: boolean;
|
||||
/** Also send the quarter's report after the month that ends it (March, June, September, December). */
|
||||
quarterly: boolean;
|
||||
/** Also send the year's report after December. */
|
||||
annual: boolean;
|
||||
/** Day of the month, 1–28 (every month has it), in notifications.timezone. */
|
||||
day: number;
|
||||
/** "HH:MM", in notifications.timezone. */
|
||||
@@ -293,7 +297,7 @@ const DEFAULTS: AppSettings = {
|
||||
logRetention: { enabled: false, retentionDays: 90, intervalHours: 24 },
|
||||
quietHours: { enabled: false, start: "22:00", end: "07:00" },
|
||||
digest: { enabled: false, day: 1, time: "08:00" },
|
||||
monthlyReport: { enabled: false, day: 1, time: "08:00" },
|
||||
monthlyReport: { enabled: false, quarterly: false, annual: false, day: 1, time: "08:00" },
|
||||
inventory: { currency: "", pricePerKwh: null },
|
||||
agents: { autoUpdate: false },
|
||||
alertActions: { enabled: false },
|
||||
|
||||
@@ -34,13 +34,17 @@ export interface UptimeSeries {
|
||||
percent: number;
|
||||
}
|
||||
|
||||
export interface Segment {
|
||||
from: number;
|
||||
to: number;
|
||||
down: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits [knownFrom, now] into up and down stretches from the recorded transitions and lays them over `days` consecutive 24-hour
|
||||
* buckets ending at `now`. Before the first transition the thing is taken to have been in the opposite state (a "went offline" means it
|
||||
* was up until then); with no transitions at all it has been up the whole time.
|
||||
* Splits [knownFrom, now] into up and down stretches from the recorded transitions. Before the first transition the thing is taken to have been in the opposite state
|
||||
* (a "went offline" means it was up until then); with no transitions at all it has been up the whole time.
|
||||
*/
|
||||
export function computeBuckets(transitions: Transition[], knownFrom: number, now: number, days: number): UptimeBucket[] {
|
||||
const start = now - days * DAY_MS;
|
||||
export function segmentsOf(transitions: Transition[], knownFrom: number, now: number): Segment[] {
|
||||
const sorted = [...transitions].sort((a, b) => a.at - b.at);
|
||||
|
||||
// State at knownFrom: after any transitions that happened before it, otherwise the opposite of the first one that follows.
|
||||
@@ -49,7 +53,7 @@ export function computeBuckets(transitions: Transition[], knownFrom: number, now
|
||||
while (i < sorted.length && sorted[i].at <= knownFrom) state = sorted[i++].down;
|
||||
if (i === 0 && sorted.length > 0) state = !sorted[0].down;
|
||||
|
||||
const segments: { from: number; to: number; down: boolean }[] = [];
|
||||
const segments: Segment[] = [];
|
||||
let cursor = Math.min(knownFrom, now);
|
||||
for (; i < sorted.length; i++) {
|
||||
const t = Math.min(sorted[i].at, now);
|
||||
@@ -58,6 +62,26 @@ export function computeBuckets(transitions: Transition[], knownFrom: number, now
|
||||
state = sorted[i].down;
|
||||
}
|
||||
if (now > cursor) segments.push({ from: cursor, to: now, down: state });
|
||||
return segments;
|
||||
}
|
||||
|
||||
/** Minutes known, and of those spent down, within [from, to]. */
|
||||
export function minutesWithin(segments: Segment[], from: number, to: number): { known: number; down: number } {
|
||||
let known = 0;
|
||||
let down = 0;
|
||||
for (const seg of segments) {
|
||||
const overlap = Math.min(seg.to, to) - Math.max(seg.from, from);
|
||||
if (overlap <= 0) continue;
|
||||
known += overlap;
|
||||
if (seg.down) down += overlap;
|
||||
}
|
||||
return { known: Math.round(known / 60_000), down: Math.round(down / 60_000) };
|
||||
}
|
||||
|
||||
/** The segments laid over `days` consecutive 24-hour buckets ending at `now`. */
|
||||
export function computeBuckets(transitions: Transition[], knownFrom: number, now: number, days: number): UptimeBucket[] {
|
||||
const start = now - days * DAY_MS;
|
||||
const segments = segmentsOf(transitions, knownFrom, now);
|
||||
|
||||
const known = new Array<number>(days).fill(0);
|
||||
const down = new Array<number>(days).fill(0);
|
||||
@@ -121,10 +145,49 @@ export interface UptimeReport {
|
||||
integrations: Record<number, UptimeSeries>;
|
||||
}
|
||||
|
||||
export async function getUptime(now: Date = new Date(), days: number = DEFAULT_DAYS): Promise<UptimeReport> {
|
||||
const nowMs = now.getTime();
|
||||
const rows = await db.select().from(auditLog).where(inArray(auditLog.action, ["went_offline", "back_online", "went_failing", "recovered"]));
|
||||
/** How long a server or integration was known, and down, between two moments. */
|
||||
export interface UptimeTotals {
|
||||
known: number;
|
||||
down: number;
|
||||
}
|
||||
|
||||
export interface UptimeRange {
|
||||
servers: Record<number, UptimeTotals>;
|
||||
integrations: Record<number, UptimeTotals>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Exactly how long each server and integration was down between `from` and `to` (the end is clipped to now), to the minute — for a target that must be measured over a
|
||||
* calendar period rather than in whole days back from now. Only the ones known for some of it are listed.
|
||||
*/
|
||||
export async function getUptimeRange(from: Date, to: Date, now: Date = new Date()): Promise<UptimeRange> {
|
||||
const nowMs = now.getTime();
|
||||
const end = Math.min(to.getTime(), nowMs);
|
||||
const start = from.getTime();
|
||||
const out: UptimeRange = { servers: {}, integrations: {} };
|
||||
if (end <= start) return out;
|
||||
const { serverEvents, sourceEvents } = await loadTransitions(nowMs);
|
||||
|
||||
for (const s of await db.select({ id: servers.id, createdAt: servers.createdAt, lastSeenAt: servers.lastSeenAt }).from(servers)) {
|
||||
const events = serverEvents.get(s.id) ?? [];
|
||||
if (!s.lastSeenAt && events.length === 0) continue;
|
||||
const created = Date.parse(auditTimeToIso(s.createdAt));
|
||||
const totals = minutesWithin(segmentsOf(events, Number.isFinite(created) ? created : nowMs, nowMs), start, end);
|
||||
if (totals.known > 0) out.servers[s.id] = totals;
|
||||
}
|
||||
const since = await ensureHistoryStart(nowMs);
|
||||
for (const i of await db.select({ id: integrations.id, type: integrations.type, createdAt: integrations.createdAt }).from(integrations)) {
|
||||
const created = Date.parse(auditTimeToIso(i.createdAt));
|
||||
const knownFrom = Math.max(Number.isFinite(created) ? created : 0, since);
|
||||
const totals = minutesWithin(segmentsOf(sourceEvents.get(i.type) ?? [], knownFrom, nowMs), start, end);
|
||||
if (totals.known > 0) out.integrations[i.id] = totals;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The recorded moments servers went away and came back, and integrations started failing and worked again. */
|
||||
async function loadTransitions(nowMs: number): Promise<{ serverEvents: Map<number, Transition[]>; sourceEvents: Map<string, Transition[]> }> {
|
||||
const rows = await db.select().from(auditLog).where(inArray(auditLog.action, ["went_offline", "back_online", "went_failing", "recovered"]));
|
||||
const serverEvents = new Map<number, Transition[]>();
|
||||
const sourceEvents = new Map<string, Transition[]>();
|
||||
for (const r of rows) {
|
||||
@@ -141,6 +204,12 @@ export async function getUptime(now: Date = new Date(), days: number = DEFAULT_D
|
||||
sourceEvents.set(r.targetId, list);
|
||||
}
|
||||
}
|
||||
return { serverEvents, sourceEvents };
|
||||
}
|
||||
|
||||
export async function getUptime(now: Date = new Date(), days: number = DEFAULT_DAYS): Promise<UptimeReport> {
|
||||
const nowMs = now.getTime();
|
||||
const { serverEvents, sourceEvents } = await loadTransitions(nowMs);
|
||||
|
||||
const out: UptimeReport = { days, endsAt: now.toISOString(), servers: {}, integrations: {} };
|
||||
|
||||
|
||||
@@ -0,0 +1,369 @@
|
||||
/**
|
||||
* Service-level targets: the error budget of a target over a month, how a thing is measured (an endpoint from its daily figures, a server and an integration from the audit
|
||||
* log, to the minute), how the months before it went, when it is an alert, and the quarterly and annual reports that carry the results.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer as createHttpServer, type Server } from "node:http";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-slo-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, integrations, endpointChecks, endpointDaily, auditLog, serviceLevels } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const slo = await import("../src/services/serviceLevels.js");
|
||||
const uptime = await import("../src/services/uptimeHistory.js");
|
||||
const report = await import("../src/services/monthlyReport.js");
|
||||
const { sendMonthlyReport } = await import("../src/services/monthlyReportScheduler.js");
|
||||
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
const { serviceLevelsRouter } = await import("../src/routes/serviceLevels.js");
|
||||
const { reportsRouter } = await import("../src/routes/reports.js");
|
||||
const { setInternalFlag, updateSettings } = await import("../src/services/settingsStore.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const MIN = 60_000;
|
||||
const DAY = 86_400_000;
|
||||
/** The 20th of October, noon: 19.5 days into a 31-day month. */
|
||||
const NOW = new Date("2026-10-20T12:00:00Z");
|
||||
const OCT_MINUTES = 31 * 1440;
|
||||
|
||||
describe("the error budget", () => {
|
||||
const m = (down: number, percent: number | null = 99): import("../src/services/serviceLevels.js").Measure => ({ percent, downMinutes: down, knownMinutes: 1000 });
|
||||
|
||||
test("a target allows the minutes it leaves over, and says how much of that is used", () => {
|
||||
const ok = slo.judgeBudget(99.9, OCT_MINUTES, 28_080, m(10));
|
||||
assert.equal(Math.round(ok.allowedMinutes * 100) / 100, 44.64);
|
||||
assert.deepEqual([ok.state, ok.usedPercent, Math.round(ok.remainingMinutes * 10) / 10], ["ok", 22.4, 34.6]);
|
||||
assert.equal(slo.judgeBudget(99.9, OCT_MINUTES, 28_080, m(36)).state, "at_risk", "80 % or more used");
|
||||
assert.equal(slo.judgeBudget(99.9, OCT_MINUTES, 28_080, m(35)).state, "ok", "78 %");
|
||||
const over = slo.judgeBudget(99.9, OCT_MINUTES, 28_080, m(50));
|
||||
assert.deepEqual([over.state, over.usedPercent, Math.round(over.remainingMinutes)], ["breached", 112, -5]);
|
||||
assert.equal(slo.judgeBudget(99.9, OCT_MINUTES, 28_080, m(44)).state, "at_risk", "within the budget, though barely");
|
||||
});
|
||||
|
||||
test("no figure is no data, and a budget of nothing is overspent by any downtime", () => {
|
||||
assert.equal(slo.judgeBudget(99.9, OCT_MINUTES, 100, m(0, null)).state, "no_data");
|
||||
assert.equal(slo.judgeBudget(100, OCT_MINUTES, 100, m(1)).state, "breached");
|
||||
assert.equal(slo.judgeBudget(100, OCT_MINUTES, 100, m(0)).state, "ok");
|
||||
});
|
||||
|
||||
test("the pace so far is projected to the end of the month, but not from the first hours", () => {
|
||||
assert.equal(slo.judgeBudget(99.9, OCT_MINUTES, 14_400, m(10)).projectedPercent, 69, "10 minutes in 10 days → 31 in the month → 69 % of 44.6");
|
||||
assert.equal(slo.judgeBudget(99.9, OCT_MINUTES, 600, m(10)).projectedPercent, null, "ten hours into the month says little");
|
||||
});
|
||||
|
||||
test("the words an alert uses", () => {
|
||||
const m1 = m(36);
|
||||
const b1 = slo.judgeBudget(99.9, OCT_MINUTES, 28_080, m1);
|
||||
assert.equal(slo.budgetMessage("Gitea", 99.9, "October 2026", b1, m1), "Gitea: 81% of the error budget for its 99.9% target is used up for October 2026 — down 36 min of 45 min allowed, 9 min left");
|
||||
const m2 = m(50);
|
||||
assert.equal(slo.budgetMessage("Gitea", 99.9, "October 2026", slo.judgeBudget(99.9, OCT_MINUTES, 28_080, m2), m2), "Gitea has missed its 99.9% target for October 2026: down 50 min, 45 min allowed");
|
||||
assert.equal(slo.budgetWords(0.4), "under a minute");
|
||||
});
|
||||
});
|
||||
|
||||
describe("measuring", () => {
|
||||
let checkId = 0;
|
||||
let emptyCheckId = 0;
|
||||
let serverId = 0;
|
||||
let integrationId = 0;
|
||||
|
||||
before(async () => {
|
||||
checkId = (await db.insert(endpointChecks).values({ name: "Gitea", kind: "http", target: "https://git.lab/" }).returning({ id: endpointChecks.id }))[0].id;
|
||||
emptyCheckId = (await db.insert(endpointChecks).values({ name: "Unasked", kind: "http", target: "https://x.lab/" }).returning({ id: endpointChecks.id }))[0].id;
|
||||
// October: 288 checks a day, ten of them failed on the 5th; the 20th is half done. September: a whole day failed.
|
||||
const day = (d: string, total: number, ok: number) => db.insert(endpointDaily).values({ checkId, day: d, total, ok, msSum: 0, msCount: 0 });
|
||||
for (let d = 1; d <= 19; d++) await day(`2026-10-${String(d).padStart(2, "0")}`, 288, d === 5 ? 278 : 288);
|
||||
await day("2026-10-20", 144, 144);
|
||||
for (let d = 1; d <= 30; d++) await day(`2026-09-${String(d).padStart(2, "0")}`, 288, d === 10 ? 0 : 288);
|
||||
|
||||
serverId = (await db.insert(servers).values({ name: "nas", apiTokenHash: "n", apiTokenPrefix: "n", createdAt: "2026-01-01 00:00:00", lastSeenAt: NOW.toISOString() }).returning({ id: servers.id }))[0].id;
|
||||
integrationId = (await db.insert(integrations).values({ type: "gitea", name: "Gitea", baseUrl: "", createdAt: "2026-01-01 00:00:00" }).returning({ id: integrations.id }))[0].id;
|
||||
const audit = (action: string, at: string, extra: Record<string, string>) => db.insert(auditLog).values({ category: "server", action, targetType: "server", createdAt: at, ...extra });
|
||||
await audit("went_offline", "2026-10-05 10:00:00", { targetId: String(serverId) });
|
||||
await audit("back_online", "2026-10-05 10:30:00", { targetId: String(serverId) });
|
||||
// an outage that straddles the start of the month: only the part in October counts
|
||||
await audit("went_offline", "2026-09-30 23:00:00", { targetId: String(serverId) });
|
||||
await audit("back_online", "2026-10-01 01:00:00", { targetId: String(serverId) });
|
||||
await db.insert(auditLog).values([
|
||||
{ category: "integration", action: "went_failing", targetType: "integration_type", targetId: "gitea", createdAt: "2026-10-12 08:00:00" },
|
||||
{ category: "integration", action: "recovered", targetType: "integration_type", targetId: "gitea", createdAt: "2026-10-12 08:20:00" },
|
||||
]);
|
||||
await setInternalFlag("integration_history_since", "2026-01-01T00:00:00.000Z");
|
||||
uptime.resetHistoryStartForTests();
|
||||
});
|
||||
|
||||
const oct = { start: new Date("2026-10-01T00:00:00Z"), end: new Date("2026-11-01T00:00:00Z") };
|
||||
|
||||
test("an endpoint: the share of checks that failed, over the time that has passed", async () => {
|
||||
const r = await slo.measure("endpoint", checkId, oct.start, oct.end, slo.newContext(NOW));
|
||||
assert.deepEqual([r.percent, r.downMinutes, r.knownMinutes], [99.822, 50, 28_080]);
|
||||
assert.deepEqual(await slo.measure("endpoint", emptyCheckId, oct.start, oct.end, slo.newContext(NOW)), { percent: null, downMinutes: 0, knownMinutes: 0 }, "no checks, no figure");
|
||||
assert.equal((await slo.measure("endpoint", checkId, new Date("2026-11-01T00:00:00Z"), new Date("2026-12-01T00:00:00Z"), slo.newContext(NOW))).percent, null, "a month that has not begun");
|
||||
});
|
||||
|
||||
test("a server: the minutes the audit log says it was away, only those within the period", async () => {
|
||||
const r = await slo.measure("server", serverId, oct.start, oct.end, slo.newContext(NOW));
|
||||
assert.deepEqual([r.downMinutes, r.knownMinutes], [90, 28_080], "30 on the 5th and the 60 of the night that began in September");
|
||||
assert.equal(r.percent, 99.679);
|
||||
const sept = await slo.measure("server", serverId, new Date("2026-09-01T00:00:00Z"), oct.start, slo.newContext(NOW));
|
||||
assert.equal(sept.downMinutes, 60, "and in September, the hour before midnight");
|
||||
});
|
||||
|
||||
test("an integration is measured by the kind of service it is", async () => {
|
||||
const r = await slo.measure("integration", integrationId, oct.start, oct.end, slo.newContext(NOW));
|
||||
assert.deepEqual([r.downMinutes, r.percent], [20, 99.929]);
|
||||
});
|
||||
|
||||
test("each stretch of the audit log is read once for everything measured over the same period", async () => {
|
||||
const ctx = slo.newContext(NOW);
|
||||
await Promise.all([slo.measure("server", serverId, oct.start, oct.end, ctx), slo.measure("integration", integrationId, oct.start, oct.end, ctx)]);
|
||||
assert.equal(ctx.uptime.size, 1);
|
||||
});
|
||||
|
||||
test("the exact range is what the whole-day figures are made of", async () => {
|
||||
const range = await uptime.getUptimeRange(new Date(NOW.getTime() - 3 * DAY), NOW, NOW);
|
||||
assert.deepEqual(Object.keys(range.servers).map(Number), [serverId]);
|
||||
assert.deepEqual(await uptime.getUptimeRange(NOW, new Date(NOW.getTime() - MIN), NOW), { servers: {}, integrations: {} }, "an empty range");
|
||||
const seg = uptime.segmentsOf([{ at: 100 * MIN, down: true }, { at: 130 * MIN, down: false }], 0, 200 * MIN);
|
||||
assert.deepEqual(uptime.minutesWithin(seg, 90 * MIN, 120 * MIN), { known: 30, down: 20 });
|
||||
assert.deepEqual(uptime.minutesWithin(seg, 300 * MIN, 400 * MIN), { known: 0, down: 0 });
|
||||
});
|
||||
|
||||
test("this month's budget, with how each of the months before it went", async () => {
|
||||
await db.insert(serviceLevels).values([
|
||||
{ name: "Gitea answers", subjectType: "endpoint", subjectId: checkId, target: 99.9 },
|
||||
{ name: "NAS up", subjectType: "server", subjectId: serverId, target: 99 },
|
||||
{ name: "Gitea service", subjectType: "integration", subjectId: integrationId, target: 99.9, alert: false },
|
||||
{ name: "Nobody asks", subjectType: "endpoint", subjectId: emptyCheckId, target: 99.9 },
|
||||
]);
|
||||
const all = await slo.levelStatuses(NOW);
|
||||
const by = Object.fromEntries(all.map((s) => [s.name, s]));
|
||||
assert.deepEqual([by["Gitea answers"].budget.state, by["Gitea answers"].measure.percent, by["Gitea answers"].subjectName], ["breached", 99.822, "Gitea"]);
|
||||
assert.deepEqual([by["NAS up"].budget.state, Math.round(by["NAS up"].budget.allowedMinutes)], ["ok", 446], "99 % of 31 days leaves 446 minutes");
|
||||
assert.equal(by["Nobody asks"].budget.state, "no_data");
|
||||
assert.deepEqual([by["Gitea answers"].period.id, by["Gitea answers"].period.label, by["Gitea answers"].period.totalMinutes, by["Gitea answers"].period.elapsedMinutes], ["2026-10", "October 2026", OCT_MINUTES, 28_080]);
|
||||
// the months before: September had a whole failed day; August has no figures
|
||||
const hist = by["Gitea answers"].history;
|
||||
assert.deepEqual(hist.map((h) => h.id), ["2026-09", "2026-08", "2026-07", "2026-06", "2026-05", "2026-04"]);
|
||||
assert.deepEqual([hist[0].percent, hist[0].met, Math.round(hist[0].downMinutes)], [96.667, false, 1440]);
|
||||
assert.deepEqual([hist[1].percent, hist[1].met], [null, null]);
|
||||
assert.equal((await slo.levelStatuses(NOW, { history: false }))[0].history.length, 0, "left out when it isn't wanted");
|
||||
});
|
||||
|
||||
test("a target whose thing has been removed has nothing to measure, and says so", async () => {
|
||||
await db.insert(serviceLevels).values({ name: "Gone", subjectType: "server", subjectId: 99999, target: 99.9 });
|
||||
const gone = (await slo.levelStatuses(NOW)).find((s) => s.name === "Gone")!;
|
||||
assert.deepEqual([gone.subjectName, gone.budget.state], [null, "no_data"]);
|
||||
await db.delete(serviceLevels).where(eq(serviceLevels.name, "Gone"));
|
||||
});
|
||||
|
||||
test("only targets that are to alert, and whose budget is mostly or wholly used, are findings", async () => {
|
||||
const found = await slo.levelFindings(NOW);
|
||||
assert.deepEqual(found.map((f) => [f.name, f.state, f.severity]), [["Gitea answers", "breached", "critical"]]);
|
||||
assert.match(found[0].message, /^Gitea answers has missed its 99.9% target for October 2026: down 50 min, 45 min allowed$/);
|
||||
assert.equal(slo.levelKey(found[0]), `slo:${found[0].levelId}`);
|
||||
});
|
||||
|
||||
test("how each did over a period, for a report: the missed first", async () => {
|
||||
const results = await slo.levelResultsFor(oct.start, oct.end, NOW);
|
||||
assert.deepEqual(results.map((r) => [r.name, r.met]), [["Gitea answers", false], ["Gitea service", true], ["NAS up", true]], "the missed first, then the most used");
|
||||
assert.ok(results.every((r) => r.name !== "Nobody asks"), "one with no figure is left out");
|
||||
assert.equal(results[0].allowedMinutes, 44.6);
|
||||
});
|
||||
});
|
||||
|
||||
describe("quarters and years", () => {
|
||||
test("which are which, and what can be asked for", () => {
|
||||
assert.equal(report.periodKind("2026-Q3"), "quarter");
|
||||
assert.equal(report.periodKind("2026"), "year");
|
||||
assert.equal(report.periodKind("2026-09"), "month");
|
||||
for (const ok of ["2026-09", "2026-Q1", "2026"]) assert.equal(report.isPeriodId(ok), true, ok);
|
||||
for (const bad of ["2026-Q5", "2026-Q0", "26", "2026-13", "Q3", "", undefined, 2026]) assert.equal(report.isPeriodId(bad), false, String(bad));
|
||||
assert.deepEqual([report.periodLabel("2026-Q3"), report.periodLabel("2026"), report.periodLabel("2026-09")], ["Q3 2026", "2026", "September 2026"]);
|
||||
assert.equal(report.lastFullQuarter(new Date("2026-10-02T09:00:00Z"), "UTC"), "2026-Q3");
|
||||
assert.equal(report.lastFullQuarter(new Date("2027-01-02T09:00:00Z"), "UTC"), "2026-Q4", "across the new year");
|
||||
assert.equal(report.lastFullYear(new Date("2027-01-02T09:00:00Z"), "UTC"), "2026");
|
||||
const options = report.recentPeriods(new Date("2026-10-19T12:00:00Z"), "UTC");
|
||||
assert.deepEqual([options.months.length, options.quarters.length, options.years.length], [12, 8, 3]);
|
||||
assert.deepEqual([options.quarters[0], options.quarters[1], options.quarters[7]], [{ id: "2026-Q4", label: "Q4 2026" }, { id: "2026-Q3", label: "Q3 2026" }, { id: "2025-Q1", label: "Q1 2025" }]);
|
||||
assert.deepEqual(options.years.map((y) => y.id), ["2026", "2025", "2024"]);
|
||||
});
|
||||
|
||||
test("a quarter and a year begin and end where the clock of the time zone says", () => {
|
||||
const q3 = report.periodBounds("2026-Q3", "UTC");
|
||||
assert.deepEqual([q3.start.toISOString(), q3.end.toISOString(), q3.firstDay, q3.nextFirstDay], ["2026-07-01T00:00:00.000Z", "2026-10-01T00:00:00.000Z", "2026-07-01", "2026-10-01"]);
|
||||
const q4 = report.periodBounds("2026-Q4", "Europe/Stockholm");
|
||||
assert.deepEqual([q4.start.toISOString(), q4.end.toISOString(), q4.nextFirstDay], ["2026-09-30T22:00:00.000Z", "2026-12-31T23:00:00.000Z", "2027-01-01"], "summer time at the start, winter at the end");
|
||||
const y = report.periodBounds("2026", "UTC");
|
||||
assert.deepEqual([y.start.toISOString(), y.end.toISOString(), y.nextFirstDay], ["2026-01-01T00:00:00.000Z", "2027-01-01T00:00:00.000Z", "2027-01-01"]);
|
||||
assert.equal(report.periodBounds("2026-09", "UTC").nextFirstDay, "2026-10-01", "a month is as it was");
|
||||
});
|
||||
|
||||
test("a quarter's report is the same report over longer, with the targets in it", async () => {
|
||||
const q = await report.buildMonthlyReport("2026-Q3", NOW);
|
||||
assert.deepEqual([q.period, q.month, q.label, q.partial, q.from, q.to], ["quarter", "2026-Q3", "Q3 2026", false, "2026-07-01T00:00:00.000Z", "2026-10-01T00:00:00.000Z"]);
|
||||
assert.deepEqual(q.serviceLevels.map((l) => [l.name, l.met]), [["Gitea answers", false], ["NAS up", true], ["Gitea service", true]], "only September has endpoint figures: a whole failed day");
|
||||
const text = report.monthlyReportText(q);
|
||||
assert.equal(text.title, "Homelab quarterly report — Q3 2026");
|
||||
assert.match(text.message, /SERVICE LEVELS\n2 of 3 targets met\.\n- Gitea answers: 96\.667% against 99\.9% — MISSED \(down 1 d, 2 h 13 min allowed\)/);
|
||||
const year = await report.buildMonthlyReport("2026", NOW);
|
||||
assert.deepEqual([year.period, year.label, year.partial], ["year", "2026", true]);
|
||||
assert.equal(report.monthlyReportText(year).title, "Homelab annual report — 2026 (so far)");
|
||||
const month = await report.buildMonthlyReport("2026-10", NOW);
|
||||
assert.equal(month.period, "month");
|
||||
assert.deepEqual(month.serviceLevels.map((l) => [l.name, l.met]), [["Gitea answers", false], ["Gitea service", true], ["NAS up", true]]);
|
||||
assert.equal(report.monthlyReportText(month).title, "Homelab monthly report — October 2026 (so far)");
|
||||
});
|
||||
});
|
||||
|
||||
describe("as alerts, in /metrics, in the routes and as scheduled messages", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "admin";
|
||||
let hook: Server;
|
||||
const messages: string[] = [];
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
||||
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
||||
]);
|
||||
hook = createHttpServer((req, res) => {
|
||||
const chunks: Buffer[] = [];
|
||||
req.on("data", (c) => chunks.push(c));
|
||||
req.on("end", () => {
|
||||
messages.push(String(JSON.parse(Buffer.concat(chunks).toString()).content));
|
||||
res.end("{}");
|
||||
});
|
||||
});
|
||||
await new Promise<void>((r) => hook.listen(0, "127.0.0.1", r));
|
||||
await updateSettings({ webhook: { enabled: true, url: `http://127.0.0.1:${(hook.address() as AddressInfo).port}/` } });
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/service-levels", serviceLevelsRouter);
|
||||
a.use("/api/reports", reportsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
|
||||
});
|
||||
after(() => {
|
||||
app.close();
|
||||
hook.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const send = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json", connection: "close" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
|
||||
test("a finding is an ordinary health condition", async () => {
|
||||
const found = await slo.levelFindings(NOW);
|
||||
const list = evaluateHealth({ servers: [], proxmox: [], synology: [], levels: found }, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, NOW.getTime());
|
||||
const c = list.find((x) => x.key === `slo:${found[0].levelId}`)!;
|
||||
assert.deepEqual([c.source, c.severity], ["slo", "critical"]);
|
||||
const { snapshot } = await collectSnapshot();
|
||||
assert.ok(Array.isArray(snapshot.levels), "the health pass looks");
|
||||
});
|
||||
|
||||
test("the Alerts page says it, with a link to the page", async () => {
|
||||
// measured against the real clock: put a target on something that is down now
|
||||
await db.insert(servers).values({ name: "down-now", apiTokenHash: "d", apiTokenPrefix: "d", createdAt: "2020-01-01 00:00:00", lastSeenAt: new Date().toISOString() });
|
||||
const [s] = await db.select().from(servers).where(eq(servers.name, "down-now"));
|
||||
const startOfMonth = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), 1)).toISOString().slice(0, 19).replace("T", " ");
|
||||
await db.insert(auditLog).values({ category: "server", action: "went_offline", targetType: "server", targetId: String(s.id), createdAt: startOfMonth });
|
||||
await db.insert(serviceLevels).values({ name: "Always up", subjectType: "server", subjectId: s.id, target: 99.9 });
|
||||
invalidateAlertsCache();
|
||||
const found = (await getAlerts(true)).alerts.find((a) => a.message.startsWith("Always up has missed"));
|
||||
assert.ok(found, "the alert is there");
|
||||
assert.deepEqual([found.severity, found.category, found.source, found.link], ["critical", "monitoring", "Service level", "/service-levels"]);
|
||||
});
|
||||
|
||||
test("/metrics has each target, how it is doing and what is left", async () => {
|
||||
const text = await buildMetrics();
|
||||
assert.match(text, /homelab_service_level_target_percent\{name="Gitea answers"[^}]*\} 99\.9/);
|
||||
assert.match(text, /homelab_service_level_error_budget_remaining_minutes\{name="Always up"[^}]*\} -\d/);
|
||||
});
|
||||
|
||||
test("anyone signed in can look; only an administrator sets, changes and removes targets, with a reason when it is wrong", async () => {
|
||||
who = "viewer";
|
||||
const list = (await (await send("GET", "/service-levels")).json()) as any;
|
||||
assert.ok(list.levels.length >= 4 && list.subjects.endpoint.length >= 1 && list.subjects.server.length >= 1);
|
||||
assert.equal((await send("POST", "/service-levels", { name: "x", subjectType: "server", subjectId: 1, target: 99.9 })).status, 403);
|
||||
who = "admin";
|
||||
const [e] = list.subjects.endpoint;
|
||||
const good = { name: "Wiki", subjectType: "endpoint", subjectId: e.id, target: 99.95 };
|
||||
const res = await send("POST", "/service-levels", good);
|
||||
assert.equal(res.status, 201);
|
||||
const id = ((await res.json()) as { id: number }).id;
|
||||
for (const bad of [{ ...good, target: 89.9 }, { ...good, target: 100 }, { ...good, name: " " }, { ...good, subjectType: "dns" }, { ...good, subjectId: 99999 }]) {
|
||||
const r = await send("POST", "/service-levels", bad);
|
||||
assert.equal(r.status, 400, JSON.stringify(bad));
|
||||
}
|
||||
assert.match(((await (await send("POST", "/service-levels", { ...good, target: 100 })).json()) as { message: string }).message, /100% leaves no room/);
|
||||
assert.equal((await send("PUT", `/service-levels/${id}`, { ...good, target: 99.5, name: "Wiki up" })).status, 200);
|
||||
const [row] = await db.select().from(serviceLevels).where(eq(serviceLevels.id, id));
|
||||
assert.deepEqual([row.name, row.target], ["Wiki up", 99.5]);
|
||||
assert.equal((await send("PUT", "/service-levels/99999", good)).status, 404);
|
||||
const audited = await db.select().from(auditLog).where(eq(auditLog.category, "service_level"));
|
||||
assert.deepEqual(audited.map((a) => a.action).sort(), ["create", "update"]);
|
||||
who = "viewer";
|
||||
assert.equal((await send("DELETE", `/service-levels/${id}`)).status, 403);
|
||||
who = "admin";
|
||||
assert.equal((await send("DELETE", `/service-levels/${id}`)).status, 204);
|
||||
assert.equal((await send("DELETE", `/service-levels/${id}`)).status, 404);
|
||||
});
|
||||
|
||||
test("the report route takes a month, a quarter or a year, and offers all three", async () => {
|
||||
who = "viewer";
|
||||
const body = (await (await send("GET", "/reports/monthly?month=2026-Q3")).json()) as any;
|
||||
assert.deepEqual([body.report.period, body.report.label], ["quarter", "Q3 2026"]);
|
||||
assert.deepEqual([body.months.length, body.quarters.length, body.years.length], [12, 8, 3]);
|
||||
assert.equal((await send("GET", `/reports/monthly?month=${new Date().getUTCFullYear()}`)).status, 200);
|
||||
for (const bad of ["2026-Q9", "1999", "1999-Q1", "abc"]) assert.equal((await send("GET", `/reports/monthly?month=${bad}`)).status, 400, bad);
|
||||
const text = (await (await send("GET", "/reports/monthly?month=2026-Q3&format=text")).json()) as { title: string };
|
||||
assert.equal(text.title, "Homelab quarterly report — Q3 2026");
|
||||
who = "admin";
|
||||
});
|
||||
|
||||
test("a quarter's and a year's report follow the monthly one when they are switched on, and only after the month that closes them", async () => {
|
||||
const titles = () => messages.map((m) => /Homelab (monthly|quarterly|annual) report — [^*]+/.exec(m)?.[0] ?? "").filter(Boolean);
|
||||
const base = { enabled: true, day: 1, time: "08:00" };
|
||||
messages.length = 0;
|
||||
await updateSettings({ monthlyReport: { ...base, quarterly: false, annual: false } });
|
||||
await sendMonthlyReport(new Date("2026-10-02T09:00:00Z"));
|
||||
assert.deepEqual(titles(), ["Homelab monthly report — September 2026"]);
|
||||
messages.length = 0;
|
||||
await updateSettings({ monthlyReport: { ...base, quarterly: true, annual: true } });
|
||||
await sendMonthlyReport(new Date("2026-10-02T09:00:00Z"));
|
||||
assert.deepEqual(titles(), ["Homelab monthly report — September 2026", "Homelab quarterly report — Q3 2026"], "September closes a quarter, not the year");
|
||||
messages.length = 0;
|
||||
await sendMonthlyReport(new Date("2026-11-02T09:00:00Z"));
|
||||
assert.deepEqual(titles(), ["Homelab monthly report — October 2026"], "October closes nothing");
|
||||
messages.length = 0;
|
||||
await sendMonthlyReport(new Date("2027-01-02T09:00:00Z"));
|
||||
assert.deepEqual(titles(), ["Homelab monthly report — December 2026", "Homelab quarterly report — Q4 2026", "Homelab annual report — 2026"]);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user