Add a since-you-last-looked summary, 30-day uptime ribbons, a maintenance timeline and hover timestamps
- Dashboard card on what changed since the user pressed Got it (users.last_looked_at, migration 0021), folding each problem and its recovery into one line. - Uptime ribbons on servers and integrations, built from the audit log; integrations' failing/working-again moments are now recorded. - Timeline of running and scheduled maintenance windows on the Maintenance page. - TimeAgo component: relative times with the exact time in the chosen format on hover. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3945c60681
commit
906220040e
32 files changed
+3380
-15
No files matched your search
@@ -0,0 +1 @@
|
||||
ALTER TABLE `users` ADD `last_looked_at` text;
|
||||
File diff suppressed because it is too large.
Load diff
@@ -148,6 +148,13 @@
|
||||
"when": 1792093015342,
|
||||
"tag": "0020_salty_black_tom",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 21,
|
||||
"version": "6",
|
||||
"when": 1792177953255,
|
||||
"tag": "0021_conscious_snowbird",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import { env } from "./env.js";
|
||||
import { resolveDataPath } from "./paths.js";
|
||||
import { authRouter } from "./auth/router.js";
|
||||
import { meRouter } from "./routes/me.js";
|
||||
import { uptimeRouter } from "./routes/uptime.js";
|
||||
import { usersRouter } from "./routes/users.js";
|
||||
import { auditLogRouter } from "./routes/auditLog.js";
|
||||
import { diagLogRouter } from "./routes/diagLog.js";
|
||||
@@ -87,6 +88,7 @@ export function createApp() {
|
||||
|
||||
app.use("/auth", authRouter);
|
||||
app.use("/api/me", meRouter);
|
||||
app.use("/api/uptime", uptimeRouter);
|
||||
app.use("/api/users", usersRouter);
|
||||
app.use("/api/audit-log", auditLogRouter);
|
||||
app.use("/api/diag-log", diagLogRouter);
|
||||
|
||||
@@ -18,6 +18,8 @@ export const users = sqliteTable("users", {
|
||||
lastLoginAt: text("last_login_at"),
|
||||
// The pages this person pinned to the top of their menu, as a JSON array of paths ("/servers", "/alerts"). It follows them to any browser.
|
||||
favorites: text("favorites"),
|
||||
// When this person last pressed "Got it" on the dashboard summary of what changed; the next summary starts from here (ISO).
|
||||
lastLookedAt: text("last_looked_at"),
|
||||
});
|
||||
|
||||
// ─── Audit log ──────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -5,6 +5,7 @@ import { db } from "../db/client.js";
|
||||
import { users } from "../db/schema.js";
|
||||
import { requireAuth } from "../auth/middleware.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
import { FIRST_LOOKBACK_MS, getChangesSince } from "../services/changesSince.js";
|
||||
|
||||
export const meRouter = Router();
|
||||
|
||||
@@ -43,3 +44,28 @@ meRouter.put("/favorites", requireAuth, asyncHandler(async (req, res) => {
|
||||
await db.update(users).set({ favorites: JSON.stringify(paths) }).where(eq(users.id, req.currentUser!.id));
|
||||
res.json({ favorites: paths });
|
||||
}));
|
||||
|
||||
const CHANGES_SHOWN = 8;
|
||||
|
||||
// What happened since this person last pressed "Got it" on the dashboard (or in the last day, the first time). Only reads.
|
||||
meRouter.get("/changes", requireAuth, asyncHandler(async (req, res) => {
|
||||
const user = req.currentUser!;
|
||||
const now = new Date();
|
||||
const stored = user.lastLookedAt ? Date.parse(user.lastLookedAt) : NaN;
|
||||
const since = new Date(Number.isFinite(stored) ? stored : now.getTime() - FIRST_LOOKBACK_MS);
|
||||
const report = await getChangesSince({ since, now, actorId: user.id, canSeeAudit: user.role === "admin" || user.role === "operator" });
|
||||
res.json({
|
||||
since: report.since,
|
||||
firstTime: !Number.isFinite(stored),
|
||||
events: report.events.slice(0, CHANGES_SHOWN),
|
||||
moreEvents: Math.max(0, report.events.length - CHANGES_SHOWN),
|
||||
changesByOthers: report.changesByOthers,
|
||||
});
|
||||
}));
|
||||
|
||||
// "Got it": the next summary starts from now. Only ever for the signed-in person.
|
||||
meRouter.post("/changes/seen", requireAuth, asyncHandler(async (req, res) => {
|
||||
const at = new Date().toISOString();
|
||||
await db.update(users).set({ lastLookedAt: at }).where(eq(users.id, req.currentUser!.id));
|
||||
res.json({ lastLookedAt: at });
|
||||
}));
|
||||
@@ -0,0 +1,11 @@
|
||||
import { Router } from "express";
|
||||
import { requireAuth } from "../auth/middleware.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
import { getUptime } from "../services/uptimeHistory.js";
|
||||
|
||||
export const uptimeRouter = Router();
|
||||
|
||||
// Anyone signed in who can see the server and integration lists can see how those have been doing.
|
||||
uptimeRouter.get("/", requireAuth, asyncHandler(async (_req, res) => {
|
||||
res.json(await getUptime());
|
||||
}));
|
||||
@@ -0,0 +1,194 @@
|
||||
/**
|
||||
* "Since you last looked": what happened to the homelab between one visit and the next, in the order a person would tell it —
|
||||
* servers that stopped reporting or filled a disk, services that stopped answering, maintenance that began or ended — with each
|
||||
* problem and its recovery folded into one line ("elsa was offline for 12 min"). Worked out from the audit log and the maintenance
|
||||
* windows, so nothing is stored but the moment the person last pressed "Got it".
|
||||
*/
|
||||
import { and, gte } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { auditLog, integrations, maintenanceWindows, servers } from "../db/schema.js";
|
||||
import { auditTimeToIso } from "./serverActivity.js";
|
||||
import { describeTarget } from "./maintenance.js";
|
||||
|
||||
export type ChangeTone = "good" | "bad" | "info" | "neutral";
|
||||
|
||||
export interface ChangeEvent {
|
||||
/** ISO — when it began (or, for something already under way before, when it ended). */
|
||||
at: string;
|
||||
tone: ChangeTone;
|
||||
kind: "server" | "integration" | "maintenance";
|
||||
title: string;
|
||||
detail: string | null;
|
||||
/** A page in the app to look at it, or null. */
|
||||
link: string | null;
|
||||
}
|
||||
|
||||
export interface ChangesReport {
|
||||
/** ISO — where this summary starts. */
|
||||
since: string;
|
||||
/** Everything worth a line, newest first, before it is cut down for display. */
|
||||
events: ChangeEvent[];
|
||||
/** How many entries other people (not the app itself, not the asker) wrote in the audit log in that time; null when the asker can't see the audit log. */
|
||||
changesByOthers: number | null;
|
||||
}
|
||||
|
||||
/** How far back a summary reaches at most, however long since the last visit. */
|
||||
export const MAX_LOOKBACK_MS = 14 * 24 * 3600_000;
|
||||
/** Where a first-ever summary starts. */
|
||||
export const FIRST_LOOKBACK_MS = 24 * 3600_000;
|
||||
|
||||
const SERVER_PROBLEM = { went_offline: "offline", disk_full: "disk" } as const;
|
||||
const SERVER_RECOVERY = { back_online: "offline", disk_ok: "disk" } as const;
|
||||
|
||||
function span(ms: number): string {
|
||||
const minutes = Math.max(1, Math.round(ms / 60_000));
|
||||
if (minutes < 60) return `${minutes} min`;
|
||||
const hours = Math.floor(minutes / 60);
|
||||
if (hours < 24) return minutes % 60 && hours < 10 ? `${hours} h ${minutes % 60} min` : `${hours} h`;
|
||||
const days = Math.floor(hours / 24);
|
||||
return hours % 24 ? `${days} d ${hours % 24} h` : `${days} d`;
|
||||
}
|
||||
|
||||
function parseDetail(detail: string | null): Record<string, unknown> {
|
||||
if (!detail) return {};
|
||||
try {
|
||||
const v = JSON.parse(detail);
|
||||
return v && typeof v === "object" ? (v as Record<string, unknown>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
interface Mark {
|
||||
at: number;
|
||||
problem: boolean;
|
||||
message: string | null;
|
||||
}
|
||||
|
||||
/** Folds an ordered run of problem / recovery marks into the story: each pair is one finished episode, a lone recovery began before, a lone problem is still going. */
|
||||
export function foldMarks(marks: Mark[]): { start: number | null; end: number | null; message: string | null }[] {
|
||||
const out: { start: number | null; end: number | null; message: string | null }[] = [];
|
||||
let open: Mark | null = null;
|
||||
for (const m of [...marks].sort((a, b) => a.at - b.at)) {
|
||||
if (m.problem) {
|
||||
if (!open) open = m; // a second "problem" while one is open (a restart repeating itself) adds nothing
|
||||
} else if (open) {
|
||||
out.push({ start: open.at, end: m.at, message: open.message });
|
||||
open = null;
|
||||
} else {
|
||||
out.push({ start: null, end: m.at, message: null });
|
||||
}
|
||||
}
|
||||
if (open) out.push({ start: open.at, end: null, message: open.message });
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function getChangesSince(params: { since: Date; now?: Date; actorId: number; canSeeAudit: boolean }): Promise<ChangesReport> {
|
||||
const now = params.now ?? new Date();
|
||||
const sinceMs = Math.max(params.since.getTime(), now.getTime() - MAX_LOOKBACK_MS);
|
||||
const nowMs = now.getTime();
|
||||
// The audit log stores "2026-10-03 14:05:09"; compare in that form, then filter exactly in JS.
|
||||
const sinceDb = new Date(sinceMs - 1000).toISOString().slice(0, 19).replace("T", " ");
|
||||
|
||||
const audits = (await db.select().from(auditLog).where(gte(auditLog.createdAt, sinceDb))).map((a) => ({ ...a, atMs: Date.parse(auditTimeToIso(a.createdAt)) })).filter((a) => a.atMs <= nowMs);
|
||||
const events: ChangeEvent[] = [];
|
||||
|
||||
// ── Servers: stopped reporting / disk full, and the ends of those ──
|
||||
const serverMarks = new Map<string, { id: number; kind: "offline" | "disk"; marks: Mark[] }>();
|
||||
for (const a of audits) {
|
||||
if (a.category !== "server" || a.targetType !== "server" || a.atMs < sinceMs) continue;
|
||||
const problem = SERVER_PROBLEM[a.action as keyof typeof SERVER_PROBLEM];
|
||||
const recovery = SERVER_RECOVERY[a.action as keyof typeof SERVER_RECOVERY];
|
||||
const kind = problem ?? recovery;
|
||||
if (!kind) continue;
|
||||
const mount = typeof parseDetail(a.detail).mount === "string" ? (parseDetail(a.detail).mount as string) : "";
|
||||
const key = `${a.targetId}:${kind}:${mount}`;
|
||||
const entry = serverMarks.get(key) ?? { id: Number(a.targetId), kind, marks: [] };
|
||||
const message = typeof parseDetail(a.detail).message === "string" ? (parseDetail(a.detail).message as string) : null;
|
||||
entry.marks.push({ at: a.atMs, problem: Boolean(problem), message });
|
||||
serverMarks.set(key, entry);
|
||||
}
|
||||
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((s) => [s.id, s.name]));
|
||||
for (const { id, kind, marks } of serverMarks.values()) {
|
||||
const name = serverNames.get(id);
|
||||
if (!name) continue; // since removed
|
||||
const link = `/servers/${id}`;
|
||||
for (const ep of foldMarks(marks)) {
|
||||
const noun = kind === "offline" ? "offline" : "had a nearly full disk";
|
||||
if (ep.start !== null && ep.end !== null) {
|
||||
events.push({ at: new Date(ep.start).toISOString(), tone: "good", kind: "server", link, detail: ep.message, title: kind === "offline" ? `${name} was offline for ${span(ep.end - ep.start)} — reporting again` : `${name} had a nearly full disk for ${span(ep.end - ep.start)} — fine again` });
|
||||
} else if (ep.start !== null) {
|
||||
events.push({ at: new Date(ep.start).toISOString(), tone: "bad", kind: "server", link, detail: ep.message, title: kind === "offline" ? `${name} went offline ${span(nowMs - ep.start)} ago — still offline` : `${name} ${noun} — still` });
|
||||
} else if (ep.end !== null) {
|
||||
events.push({ at: new Date(ep.end).toISOString(), tone: "good", kind: "server", link, detail: null, title: kind === "offline" ? `${name} is reporting again` : `${name}'s disk is no longer nearly full` });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Integrations: started failing / working again (kept per kind of service, as the health checks are) ──
|
||||
const sourceMarks = new Map<string, Mark[]>();
|
||||
for (const a of audits) {
|
||||
if (a.category !== "integration" || a.targetType !== "integration_type" || !a.targetId || a.atMs < sinceMs) continue;
|
||||
if (a.action !== "went_failing" && a.action !== "recovered") continue;
|
||||
const list = sourceMarks.get(a.targetId) ?? [];
|
||||
list.push({ at: a.atMs, problem: a.action === "went_failing", message: null });
|
||||
sourceMarks.set(a.targetId, list);
|
||||
}
|
||||
if (sourceMarks.size > 0) {
|
||||
const names = new Map<string, string[]>();
|
||||
for (const i of await db.select({ name: integrations.name, type: integrations.type }).from(integrations)) names.set(i.type, [...(names.get(i.type) ?? []), i.name]);
|
||||
for (const [source, marks] of sourceMarks) {
|
||||
const label = names.get(source)?.join(" / ");
|
||||
if (!label) continue;
|
||||
for (const ep of foldMarks(marks)) {
|
||||
if (ep.start !== null && ep.end !== null) {
|
||||
events.push({ at: new Date(ep.start).toISOString(), tone: "good", kind: "integration", link: "/integrations", detail: null, title: `${label} stopped answering for ${span(ep.end - ep.start)} — working again` });
|
||||
} else if (ep.start !== null) {
|
||||
events.push({ at: new Date(ep.start).toISOString(), tone: "bad", kind: "integration", link: "/integrations", detail: null, title: `${label} stopped answering ${span(nowMs - ep.start)} ago — still failing` });
|
||||
} else if (ep.end !== null) {
|
||||
events.push({ at: new Date(ep.end).toISOString(), tone: "good", kind: "integration", link: "/integrations", detail: null, title: `${label} is answering again` });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Maintenance: windows that began or finished in the time ──
|
||||
const sinceIso = new Date(sinceMs).toISOString();
|
||||
const nowIso = now.toISOString();
|
||||
const windows = (await db.select().from(maintenanceWindows).where(and(gte(maintenanceWindows.endsAt, sinceIso)))).filter(
|
||||
(w) => w.endsAt > w.startedAt && w.startedAt <= nowIso && (w.startedAt >= sinceIso || w.endsAt <= nowIso),
|
||||
);
|
||||
const groups = new Map<string, typeof windows>();
|
||||
for (const w of windows) {
|
||||
const key = `${w.startedAt.slice(0, 16)}|${w.endsAt.slice(0, 16)}|${w.createdBy ?? ""}|${w.reason ?? ""}`;
|
||||
groups.set(key, [...(groups.get(key) ?? []), w]);
|
||||
}
|
||||
for (const group of groups.values()) {
|
||||
const names: string[] = [];
|
||||
for (const w of group) {
|
||||
const t = await describeTarget(w.targetType, w.targetId);
|
||||
if (t) names.push(t.name);
|
||||
}
|
||||
if (names.length === 0) continue;
|
||||
const ended = group[0].endsAt <= nowIso;
|
||||
const subject = names.length >= 4 ? `${names.length} items` : names.join(", ");
|
||||
const verb = names.length === 1 ? (ended ? "was" : "is") : ended ? "were" : "are";
|
||||
const lasted = span(Date.parse(group[0].endsAt) - Date.parse(group[0].startedAt));
|
||||
events.push({
|
||||
at: group[0].startedAt >= sinceIso ? group[0].startedAt : group[0].endsAt,
|
||||
tone: "info",
|
||||
kind: "maintenance",
|
||||
link: "/maintenance",
|
||||
title: ended ? `${subject} ${verb} in maintenance for ${lasted}` : `${subject} ${verb} in maintenance — alerts silenced`,
|
||||
detail: group[0].reason,
|
||||
});
|
||||
}
|
||||
|
||||
events.sort((a, b) => b.at.localeCompare(a.at));
|
||||
|
||||
let changesByOthers: number | null = null;
|
||||
if (params.canSeeAudit) {
|
||||
changesByOthers = audits.filter((a) => a.atMs >= sinceMs && a.actorUserId !== params.actorId && a.actorLabel !== "system").length;
|
||||
}
|
||||
return { since: sinceIso, events, changesByOthers };
|
||||
}
|
||||
@@ -1,3 +1,8 @@
|
||||
import { desc, eq, and } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { auditLog } from "../db/schema.js";
|
||||
import { recordAudit } from "./audit.js";
|
||||
import { ensureHistoryStart } from "./uptimeHistory.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { isSourceInMaintenance } from "./maintenance.js";
|
||||
import { notifyIntegrationDown, notifyIntegrationRecovered } from "./notify.js";
|
||||
@@ -6,6 +11,36 @@ interface SourceHealth {
|
||||
consecutiveFailures: number;
|
||||
/** True once a "down" notification has been sent for the current failure streak, so it isn't repeated on every subsequent failure. */
|
||||
alerted: boolean;
|
||||
/** True once the audit log has been told this streak reached the failure threshold, so it is written once and its end can be written too. */
|
||||
recorded?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes a service's "started failing" / "working again" moment in the audit log (never letting a problem there hold up the call
|
||||
* that triggered it), so the dashboard can say what changed and the integration pages can draw how it has done over the weeks.
|
||||
*/
|
||||
async function recordTransition(source: string, action: "went_failing" | "recovered", detail?: unknown): Promise<void> {
|
||||
try {
|
||||
await ensureHistoryStart();
|
||||
await recordAudit({ category: "integration", action, targetType: "integration_type", targetId: source, detail });
|
||||
} catch (err) {
|
||||
console.error("[integration-health] couldn't record a transition:", err);
|
||||
}
|
||||
}
|
||||
|
||||
/** What the audit log last says about a service, for the first call after a restart (the live streak is only in memory). */
|
||||
async function lastRecordedTransition(source: string): Promise<"went_failing" | "recovered" | null> {
|
||||
try {
|
||||
const [row] = await db
|
||||
.select({ action: auditLog.action })
|
||||
.from(auditLog)
|
||||
.where(and(eq(auditLog.category, "integration"), eq(auditLog.targetType, "integration_type"), eq(auditLog.targetId, source)))
|
||||
.orderBy(desc(auditLog.createdAt), desc(auditLog.id))
|
||||
.limit(1);
|
||||
return row?.action === "went_failing" || row?.action === "recovered" ? row.action : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -38,18 +73,26 @@ export function getSourceStatus(source: string): { status: "ok" | "failing" | "u
|
||||
|
||||
/** Called after every diagnostic-log entry is recorded, to track consecutive failures per source and alert on threshold-cross / recovery. */
|
||||
export async function trackIntegrationHealth(source: string, ok: boolean): Promise<void> {
|
||||
const seenBefore = health.has(source);
|
||||
const state = health.get(source) ?? { consecutiveFailures: 0, alerted: false };
|
||||
|
||||
if (ok) {
|
||||
if (state.alerted) {
|
||||
await notifyIntegrationRecovered(source);
|
||||
}
|
||||
if (state.recorded || (!seenBefore && (await lastRecordedTransition(source)) === "went_failing")) {
|
||||
await recordTransition(source, "recovered");
|
||||
}
|
||||
health.set(source, { consecutiveFailures: 0, alerted: false });
|
||||
return;
|
||||
}
|
||||
|
||||
state.consecutiveFailures += 1;
|
||||
const { notifications } = await getSettings();
|
||||
if (!state.recorded && state.consecutiveFailures >= notifications.integrationFailureThreshold) {
|
||||
state.recorded = true;
|
||||
await recordTransition(source, "went_failing", { failures: state.consecutiveFailures });
|
||||
}
|
||||
if (notifications.integrationFailureAlerts && !state.alerted && state.consecutiveFailures >= notifications.integrationFailureThreshold) {
|
||||
// During maintenance the failures keep being counted but `alerted` stays false, so if the service is
|
||||
// still failing once the window ends, the very next failed call alerts — a real outage isn't swallowed.
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
/**
|
||||
* How each server and integration has done over the last weeks, as one figure per day: how long it was up, how long down, and for
|
||||
* how much of the day anything is known. It is worked out from the moments the health checks wrote down in the audit log (a
|
||||
* server stopped reporting / reported again; an integration's calls started failing / worked again), so it needs no table of its
|
||||
* own — and it can only be as old as those records.
|
||||
*/
|
||||
import { inArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { auditLog, integrations, servers } from "../db/schema.js";
|
||||
import { auditTimeToIso } from "./serverActivity.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
const DAY_MS = 24 * 3600_000;
|
||||
const HISTORY_FLAG = "integration_history_since";
|
||||
|
||||
export const DEFAULT_DAYS = 30;
|
||||
|
||||
export interface Transition {
|
||||
/** Epoch ms. */
|
||||
at: number;
|
||||
down: boolean;
|
||||
}
|
||||
|
||||
/** Minutes of one 24-hour stretch that are known, and of those, minutes spent down. */
|
||||
export interface UptimeBucket {
|
||||
k: number;
|
||||
d: number;
|
||||
}
|
||||
|
||||
export interface UptimeSeries {
|
||||
/** Oldest first; the last bucket ends now. */
|
||||
buckets: UptimeBucket[];
|
||||
/** Share of the known time spent up over the whole period. */
|
||||
percent: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits [knownFrom, now] into up and down stretches from the recorded transitions and lays them over `days` consecutive 24-hour
|
||||
* buckets ending at `now`. Before the first transition the thing is taken to have been in the opposite state (a "went offline" means it
|
||||
* was up until then); with no transitions at all it has been up the whole time.
|
||||
*/
|
||||
export function computeBuckets(transitions: Transition[], knownFrom: number, now: number, days: number): UptimeBucket[] {
|
||||
const start = now - days * DAY_MS;
|
||||
const sorted = [...transitions].sort((a, b) => a.at - b.at);
|
||||
|
||||
// State at knownFrom: after any transitions that happened before it, otherwise the opposite of the first one that follows.
|
||||
let state = false;
|
||||
let i = 0;
|
||||
while (i < sorted.length && sorted[i].at <= knownFrom) state = sorted[i++].down;
|
||||
if (i === 0 && sorted.length > 0) state = !sorted[0].down;
|
||||
|
||||
const segments: { from: number; to: number; down: boolean }[] = [];
|
||||
let cursor = Math.min(knownFrom, now);
|
||||
for (; i < sorted.length; i++) {
|
||||
const t = Math.min(sorted[i].at, now);
|
||||
if (t > cursor) segments.push({ from: cursor, to: t, down: state });
|
||||
cursor = Math.max(cursor, t);
|
||||
state = sorted[i].down;
|
||||
}
|
||||
if (now > cursor) segments.push({ from: cursor, to: now, down: state });
|
||||
|
||||
const known = new Array<number>(days).fill(0);
|
||||
const down = new Array<number>(days).fill(0);
|
||||
for (const seg of segments) {
|
||||
if (seg.to <= start) continue;
|
||||
const first = Math.max(0, Math.floor((Math.max(seg.from, start) - start) / DAY_MS));
|
||||
for (let b = first; b < days; b++) {
|
||||
const bStart = start + b * DAY_MS;
|
||||
if (bStart >= seg.to) break;
|
||||
const overlap = Math.min(seg.to, bStart + DAY_MS) - Math.max(seg.from, bStart);
|
||||
if (overlap <= 0) continue;
|
||||
known[b] += overlap;
|
||||
if (seg.down) down[b] += overlap;
|
||||
}
|
||||
}
|
||||
return known.map((k, b) => ({ k: Math.round(k / 60_000), d: Math.round(down[b] / 60_000) }));
|
||||
}
|
||||
|
||||
/** Share of the known time spent up, to one decimal; null when nothing is known. */
|
||||
export function uptimePercent(buckets: UptimeBucket[]): number | null {
|
||||
const known = buckets.reduce((n, b) => n + b.k, 0);
|
||||
if (known === 0) return null;
|
||||
const down = buckets.reduce((n, b) => n + b.d, 0);
|
||||
return Math.round((1 - down / known) * 1000) / 10;
|
||||
}
|
||||
|
||||
function series(transitions: Transition[], knownFrom: number, now: number, days: number): UptimeSeries | null {
|
||||
const buckets = computeBuckets(transitions, knownFrom, now, days);
|
||||
const percent = uptimePercent(buckets);
|
||||
return percent === null ? null : { buckets, percent };
|
||||
}
|
||||
|
||||
/**
|
||||
* Integrations' failing / working-again moments only exist from the version that began writing them, so nothing earlier can be claimed.
|
||||
* The first time anything asks, that moment is written down and kept.
|
||||
*/
|
||||
let historyStart: number | null = null;
|
||||
export async function ensureHistoryStart(now: number = Date.now()): Promise<number> {
|
||||
if (historyStart !== null) return historyStart;
|
||||
const stored = await getInternalFlag(HISTORY_FLAG);
|
||||
const parsed = stored ? Date.parse(stored) : NaN;
|
||||
if (Number.isFinite(parsed)) {
|
||||
historyStart = parsed;
|
||||
return parsed;
|
||||
}
|
||||
await setInternalFlag(HISTORY_FLAG, new Date(now).toISOString());
|
||||
historyStart = now;
|
||||
return now;
|
||||
}
|
||||
|
||||
/** For tests: forget the remembered start. */
|
||||
export function resetHistoryStartForTests(): void {
|
||||
historyStart = null;
|
||||
}
|
||||
|
||||
export interface UptimeReport {
|
||||
days: number;
|
||||
/** ISO — the end of the last bucket. */
|
||||
endsAt: string;
|
||||
servers: Record<number, UptimeSeries>;
|
||||
integrations: Record<number, UptimeSeries>;
|
||||
}
|
||||
|
||||
export async function getUptime(now: Date = new Date(), days: number = DEFAULT_DAYS): Promise<UptimeReport> {
|
||||
const nowMs = now.getTime();
|
||||
const rows = await db.select().from(auditLog).where(inArray(auditLog.action, ["went_offline", "back_online", "went_failing", "recovered"]));
|
||||
|
||||
const serverEvents = new Map<number, Transition[]>();
|
||||
const sourceEvents = new Map<string, Transition[]>();
|
||||
for (const r of rows) {
|
||||
const at = Date.parse(auditTimeToIso(r.createdAt));
|
||||
if (!Number.isFinite(at) || at > nowMs) continue;
|
||||
if (r.category === "server" && r.targetType === "server" && (r.action === "went_offline" || r.action === "back_online")) {
|
||||
const id = Number(r.targetId);
|
||||
const list = serverEvents.get(id) ?? [];
|
||||
list.push({ at, down: r.action === "went_offline" });
|
||||
serverEvents.set(id, list);
|
||||
} else if (r.category === "integration" && r.targetType === "integration_type" && r.targetId && (r.action === "went_failing" || r.action === "recovered")) {
|
||||
const list = sourceEvents.get(r.targetId) ?? [];
|
||||
list.push({ at, down: r.action === "went_failing" });
|
||||
sourceEvents.set(r.targetId, list);
|
||||
}
|
||||
}
|
||||
|
||||
const out: UptimeReport = { days, endsAt: now.toISOString(), servers: {}, integrations: {} };
|
||||
|
||||
for (const s of await db.select({ id: servers.id, createdAt: servers.createdAt, lastSeenAt: servers.lastSeenAt }).from(servers)) {
|
||||
const events = serverEvents.get(s.id) ?? [];
|
||||
// A server that has never reported and never been seen to stop has no history to speak of.
|
||||
if (!s.lastSeenAt && events.length === 0) continue;
|
||||
const created = Date.parse(auditTimeToIso(s.createdAt));
|
||||
const result = series(events, Number.isFinite(created) ? created : nowMs, nowMs, days);
|
||||
if (result) out.servers[s.id] = result;
|
||||
}
|
||||
|
||||
const since = await ensureHistoryStart(nowMs);
|
||||
for (const i of await db.select({ id: integrations.id, type: integrations.type, createdAt: integrations.createdAt }).from(integrations)) {
|
||||
const created = Date.parse(auditTimeToIso(i.createdAt));
|
||||
const knownFrom = Math.max(Number.isFinite(created) ? created : 0, since);
|
||||
const result = series(sourceEvents.get(i.type) ?? [], knownFrom, nowMs, days);
|
||||
if (result) out.integrations[i.id] = result;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,299 @@
|
||||
/**
|
||||
* The 30-day uptime ribbons and the dashboard's "since you last looked" summary — both built from what the health checks wrote
|
||||
* down in the audit log.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-uptime-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, integrations, auditLog, maintenanceWindows } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { meRouter } = await import("../src/routes/me.js");
|
||||
const { uptimeRouter } = await import("../src/routes/uptime.js");
|
||||
const { computeBuckets, uptimePercent, getUptime } = await import("../src/services/uptimeHistory.js");
|
||||
const { foldMarks, getChangesSince } = await import("../src/services/changesSince.js");
|
||||
const { trackIntegrationHealth } = await import("../src/services/integrationHealthMonitor.js");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const HOUR = 3600_000;
|
||||
const DAY = 24 * HOUR;
|
||||
const NOW = Math.floor(Date.now() / 1000) * 1000;
|
||||
const dbTime = (ms: number) => new Date(ms).toISOString().slice(0, 19).replace("T", " ");
|
||||
|
||||
describe("computeBuckets", () => {
|
||||
test("a thing with no problems recorded has been up for as long as it has been known", () => {
|
||||
const buckets = computeBuckets([], NOW - 10 * DAY, NOW, 30);
|
||||
assert.equal(buckets.length, 30);
|
||||
assert.deepEqual(buckets.slice(0, 20).map((b) => b.k), new Array(20).fill(0), "nothing is known before it existed");
|
||||
assert.ok(buckets.slice(20).every((b) => b.k === 1440 && b.d === 0));
|
||||
assert.equal(uptimePercent(buckets), 100);
|
||||
});
|
||||
|
||||
test("a stretch offline shows as down minutes in the day it happened", () => {
|
||||
const buckets = computeBuckets(
|
||||
[
|
||||
{ at: NOW - 2 * HOUR, down: true },
|
||||
{ at: NOW - HOUR, down: false },
|
||||
],
|
||||
NOW - 30 * DAY,
|
||||
NOW,
|
||||
30,
|
||||
);
|
||||
assert.equal(buckets[29].d, 60);
|
||||
assert.equal(buckets[29].k, 1440);
|
||||
assert.equal(buckets.slice(0, 29).reduce((n, b) => n + b.d, 0), 0);
|
||||
assert.equal(uptimePercent(buckets), Math.round((1 - 60 / (30 * 1440)) * 1000) / 10);
|
||||
});
|
||||
|
||||
test("an outage that spans midnight is split between the days, and one still going counts up to now", () => {
|
||||
const buckets = computeBuckets([{ at: NOW - 36 * HOUR, down: true }], NOW - 30 * DAY, NOW, 30);
|
||||
assert.equal(buckets[29].d, 1440);
|
||||
assert.equal(buckets[28].d, 12 * 60);
|
||||
});
|
||||
|
||||
test("a first record of recovery means it was down before that", () => {
|
||||
const buckets = computeBuckets([{ at: NOW - 12 * HOUR, down: false }], NOW - 3 * DAY, NOW, 30);
|
||||
assert.equal(buckets[28].d, 1440);
|
||||
assert.equal(buckets[27].d, 1440);
|
||||
assert.equal(buckets[29].d, 12 * 60, "down until the recovery 12 hours ago");
|
||||
assert.equal(buckets[26].k, 0, "nothing known before it existed");
|
||||
});
|
||||
|
||||
test("repeated 'down' marks don't double anything", () => {
|
||||
const once = computeBuckets([{ at: NOW - 5 * HOUR, down: true }, { at: NOW - 3 * HOUR, down: false }], NOW - 5 * DAY, NOW, 30);
|
||||
const twice = computeBuckets([{ at: NOW - 5 * HOUR, down: true }, { at: NOW - 4 * HOUR, down: true }, { at: NOW - 3 * HOUR, down: false }], NOW - 5 * DAY, NOW, 30);
|
||||
assert.deepEqual(once, twice);
|
||||
});
|
||||
});
|
||||
|
||||
describe("foldMarks", () => {
|
||||
test("pairs, lone recoveries and open problems", () => {
|
||||
const folded = foldMarks([
|
||||
{ at: 1, problem: false, message: null },
|
||||
{ at: 5, problem: true, message: "a" },
|
||||
{ at: 6, problem: true, message: "dup" },
|
||||
{ at: 9, problem: false, message: null },
|
||||
{ at: 20, problem: true, message: "b" },
|
||||
]);
|
||||
assert.deepEqual(folded, [
|
||||
{ start: null, end: 1, message: null },
|
||||
{ start: 5, end: 9, message: "a" },
|
||||
{ start: 20, end: null, message: "b" },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("recording integration transitions", () => {
|
||||
test("the third failure is written down once, and the next success ends it", async () => {
|
||||
await trackIntegrationHealth("ribbon-a", false);
|
||||
await trackIntegrationHealth("ribbon-a", false);
|
||||
assert.equal((await db.select().from(auditLog)).filter((e) => e.targetId === "ribbon-a").length, 0, "two failures are not an outage yet");
|
||||
await trackIntegrationHealth("ribbon-a", false);
|
||||
await trackIntegrationHealth("ribbon-a", false);
|
||||
await trackIntegrationHealth("ribbon-a", true);
|
||||
await trackIntegrationHealth("ribbon-a", true);
|
||||
const rows = (await db.select().from(auditLog)).filter((e) => e.targetId === "ribbon-a");
|
||||
assert.deepEqual(rows.map((r) => r.action), ["went_failing", "recovered"]);
|
||||
assert.ok(rows.every((r) => r.category === "integration" && r.targetType === "integration_type"));
|
||||
});
|
||||
|
||||
test("a service the log last saw failing is closed off by its first success after a restart", async () => {
|
||||
await db.insert(auditLog).values({ category: "integration", action: "went_failing", targetType: "integration_type", targetId: "ribbon-b", actorLabel: "system" });
|
||||
await trackIntegrationHealth("ribbon-b", true); // nothing in memory about ribbon-b: this is the first call since "start"
|
||||
const rows = (await db.select().from(auditLog)).filter((e) => e.targetId === "ribbon-b");
|
||||
assert.deepEqual(rows.map((r) => r.action), ["went_failing", "recovered"]);
|
||||
await trackIntegrationHealth("ribbon-b", true);
|
||||
assert.equal((await db.select().from(auditLog)).filter((e) => e.targetId === "ribbon-b").length, 2, "and only once");
|
||||
});
|
||||
|
||||
test("a healthy service writes nothing", async () => {
|
||||
await trackIntegrationHealth("ribbon-c", true);
|
||||
await trackIntegrationHealth("ribbon-c", true);
|
||||
assert.equal((await db.select().from(auditLog)).filter((e) => e.targetId === "ribbon-c").length, 0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("uptime and changes from the audit log", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "admin";
|
||||
let elsaId = 0;
|
||||
let annaId = 0;
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
||||
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
||||
]);
|
||||
const made = await db
|
||||
.insert(servers)
|
||||
.values([
|
||||
{ name: "elsa", apiTokenHash: "a", apiTokenPrefix: "a", createdAt: dbTime(NOW - 40 * DAY), lastSeenAt: new Date(NOW).toISOString() },
|
||||
{ name: "anna", apiTokenHash: "b", apiTokenPrefix: "b", createdAt: dbTime(NOW - 40 * DAY), lastSeenAt: new Date(NOW).toISOString() },
|
||||
{ name: "never", apiTokenHash: "c", apiTokenPrefix: "c", createdAt: dbTime(NOW - 40 * DAY) },
|
||||
])
|
||||
.returning({ id: servers.id });
|
||||
elsaId = made[0].id;
|
||||
annaId = made[1].id;
|
||||
|
||||
await db.insert(integrations).values({ type: "proxmox", name: "Felicia", baseUrl: "https://pve.lab", config: "{}", createdAt: dbTime(NOW - 40 * DAY) });
|
||||
|
||||
const entry = (offsetMs: number, category: string, action: string, targetType: string, targetId: string | number, extra: Record<string, unknown> = {}) => ({
|
||||
category,
|
||||
action,
|
||||
targetType,
|
||||
targetId: String(targetId),
|
||||
actorLabel: "system",
|
||||
createdAt: dbTime(NOW + offsetMs),
|
||||
...extra,
|
||||
});
|
||||
await db.insert(auditLog).values([
|
||||
// elsa: offline for an hour, a while ago — finished
|
||||
entry(-5 * HOUR, "server", "went_offline", "server", elsaId, { detail: JSON.stringify({ message: "No report for 5 minutes" }) }),
|
||||
entry(-4 * HOUR, "server", "back_online", "server", elsaId),
|
||||
// anna: went offline and has not come back
|
||||
entry(-3 * HOUR, "server", "went_offline", "server", annaId),
|
||||
// a service that stopped answering and came back
|
||||
entry(-6 * HOUR, "integration", "went_failing", "integration_type", "proxmox"),
|
||||
entry(-5.5 * HOUR, "integration", "recovered", "integration_type", "proxmox"),
|
||||
// somebody else's edits, and one of Ada's own
|
||||
{ category: "secret", action: "update", targetType: "secret", targetId: "1", actorLabel: "Olga", createdAt: dbTime(NOW - 2 * HOUR) },
|
||||
{ category: "secret", action: "update", targetType: "secret", targetId: "2", actorLabel: "Olga", createdAt: dbTime(NOW - 2 * HOUR) },
|
||||
// before the window we ask about
|
||||
entry(-3 * DAY, "server", "went_offline", "server", elsaId),
|
||||
]);
|
||||
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/me", meRouter);
|
||||
a.use("/api/uptime", uptimeRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
test("uptime covers servers that have reported, gives them a percentage, and skips one that never has", async () => {
|
||||
const report = await getUptime(new Date(NOW), 30);
|
||||
assert.ok(report.servers[elsaId]);
|
||||
assert.ok(report.servers[annaId]);
|
||||
assert.equal(Object.keys(report.servers).length, 2);
|
||||
assert.equal(report.servers[annaId].buckets.length, 30);
|
||||
// anna stopped reporting 3 hours ago and the log has not seen her come back
|
||||
assert.equal(report.servers[annaId].buckets[29].d, 180);
|
||||
// elsa went offline 3 days ago, was recorded offline again and then back 4 hours ago: down from then until then
|
||||
assert.equal(report.servers[elsaId].buckets[29].d, 20 * 60);
|
||||
assert.equal(report.servers[elsaId].buckets[28].d, 1440);
|
||||
assert.equal(report.servers[elsaId].buckets[27].d, 1440);
|
||||
assert.equal(report.servers[elsaId].buckets[26].d, 0);
|
||||
assert.ok(report.servers[elsaId].percent < 100 && report.servers[elsaId].percent > 90);
|
||||
});
|
||||
|
||||
test("the uptime route answers anyone signed in", async () => {
|
||||
who = "viewer";
|
||||
const res = await fetch(`${base}/uptime`);
|
||||
assert.equal(res.status, 200);
|
||||
const body = await res.json();
|
||||
assert.equal(body.days, 30);
|
||||
who = "admin";
|
||||
});
|
||||
|
||||
test("changes since: pairs, an open problem, a service blip, and what others did", async () => {
|
||||
const adaId = (await db.select().from(users)).find((u) => u.oidcSub === "admin")!.id;
|
||||
const report = await getChangesSince({ since: new Date(NOW - 12 * HOUR), now: new Date(NOW), actorId: adaId, canSeeAudit: true });
|
||||
const titles = report.events.map((e) => e.title);
|
||||
assert.ok(titles.some((t) => /^elsa was offline for 1 h — reporting again/.test(t)), titles.join(" | "));
|
||||
assert.ok(titles.some((t) => /^anna went offline 3 h ago — still offline/.test(t)), titles.join(" | "));
|
||||
assert.ok(titles.some((t) => /^Felicia stopped answering for 30 min — working again/.test(t)), titles.join(" | "));
|
||||
const anna = report.events.find((e) => e.title.startsWith("anna"));
|
||||
assert.equal(anna?.tone, "bad");
|
||||
assert.equal(anna?.link, `/servers/${annaId}`);
|
||||
assert.equal(report.events.find((e) => e.title.startsWith("elsa"))?.detail, "No report for 5 minutes");
|
||||
// Olga's two edits count; the app's own entries don't.
|
||||
assert.equal(report.changesByOthers, 2);
|
||||
});
|
||||
|
||||
test("a viewer isn't told about the audit log", async () => {
|
||||
const viewerId = (await db.select().from(users)).find((u) => u.oidcSub === "viewer")!.id;
|
||||
const report = await getChangesSince({ since: new Date(NOW - 12 * HOUR), now: new Date(NOW), actorId: viewerId, canSeeAudit: false });
|
||||
assert.equal(report.changesByOthers, null);
|
||||
});
|
||||
|
||||
test("nothing before the start of the summary is in it", async () => {
|
||||
const report = await getChangesSince({ since: new Date(NOW - 2 * HOUR), now: new Date(NOW), actorId: 0, canSeeAudit: true });
|
||||
assert.equal(report.events.filter((e) => e.kind !== "maintenance").length, 0, "the problems were all earlier than that");
|
||||
});
|
||||
|
||||
test("maintenance that began, ended or was started in bulk is told in few lines", async () => {
|
||||
const mk = (hoursAgoStart: number, hoursAgoEnd: number, target: number, reason: string) => ({
|
||||
targetType: "server" as const,
|
||||
targetId: target,
|
||||
reason,
|
||||
startedAt: new Date(NOW - hoursAgoStart * HOUR).toISOString(),
|
||||
endsAt: new Date(NOW - hoursAgoEnd * HOUR).toISOString(),
|
||||
createdBy: "Ada",
|
||||
});
|
||||
const extra = await db
|
||||
.insert(servers)
|
||||
.values([1, 2, 3, 4].map((n) => ({ name: `bulk${n}`, apiTokenHash: `h${n}`, apiTokenPrefix: `p${n}` })))
|
||||
.returning({ id: servers.id });
|
||||
await db.insert(maintenanceWindows).values([
|
||||
mk(10, 8, elsaId, "Kernel update"), // finished
|
||||
...extra.map((s) => mk(7, -2, s.id, "Power work")), // four at once, still running
|
||||
mk(1, 1, annaId, "withdrawn"), // cancelled before it began
|
||||
]);
|
||||
const report = await getChangesSince({ since: new Date(NOW - 12 * HOUR), now: new Date(NOW), actorId: 0, canSeeAudit: false });
|
||||
const maintenance = report.events.filter((e) => e.kind === "maintenance");
|
||||
assert.equal(maintenance.length, 2, maintenance.map((m) => m.title).join(" | "));
|
||||
assert.ok(maintenance.some((m) => m.title === "elsa was in maintenance for 2 h" && m.detail === "Kernel update"));
|
||||
assert.ok(maintenance.some((m) => m.title === "4 items are in maintenance — alerts silenced" && m.detail === "Power work"));
|
||||
});
|
||||
|
||||
test("the route starts from a day ago the first time, then from where 'Got it' was pressed", async () => {
|
||||
let res = await fetch(`${base}/me/changes`);
|
||||
let body = await res.json();
|
||||
assert.equal(body.firstTime, true);
|
||||
assert.ok(Array.isArray(body.events));
|
||||
|
||||
assert.equal((await fetch(`${base}/me/changes/seen`, { method: "POST" })).status, 200);
|
||||
res = await fetch(`${base}/me/changes`);
|
||||
body = await res.json();
|
||||
assert.equal(body.firstTime, false);
|
||||
assert.equal(body.events.length, 0, "nothing has happened since just now");
|
||||
assert.ok(Math.abs(Date.parse(body.since) - Date.now()) < 60_000);
|
||||
});
|
||||
|
||||
test("'Got it' only moves the signed-in person's own mark", async () => {
|
||||
who = "viewer";
|
||||
const res = await fetch(`${base}/me/changes`);
|
||||
assert.equal((await res.json()).firstTime, true);
|
||||
who = "admin";
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user