Add a since-you-last-looked summary, 30-day uptime ribbons, a maintenance timeline and hover timestamps

- Dashboard card on what changed since the user pressed Got it (users.last_looked_at,
  migration 0021), folding each problem and its recovery into one line.
- Uptime ribbons on servers and integrations, built from the audit log; integrations'
  failing/working-again moments are now recorded.
- Timeline of running and scheduled maintenance windows on the Maintenance page.
- TimeAgo component: relative times with the exact time in the chosen format on hover.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-16 21:55:04 +02:00
1 parent 3945c60681
commit 906220040e
32 files changed
+3380 -15

No files matched your search

@@ -0,0 +1 @@
ALTER TABLE `users` ADD `last_looked_at` text;
File diff suppressed because it is too large. Load diff
+7
View File
@@ -148,6 +148,13 @@
"when": 1792093015342,
"tag": "0020_salty_black_tom",
"breakpoints": true
},
{
"idx": 21,
"version": "6",
"when": 1792177953255,
"tag": "0021_conscious_snowbird",
"breakpoints": true
}
]
}
+2
View File
@@ -8,6 +8,7 @@ import { env } from "./env.js";
import { resolveDataPath } from "./paths.js";
import { authRouter } from "./auth/router.js";
import { meRouter } from "./routes/me.js";
import { uptimeRouter } from "./routes/uptime.js";
import { usersRouter } from "./routes/users.js";
import { auditLogRouter } from "./routes/auditLog.js";
import { diagLogRouter } from "./routes/diagLog.js";
@@ -87,6 +88,7 @@ export function createApp() {
app.use("/auth", authRouter);
app.use("/api/me", meRouter);
app.use("/api/uptime", uptimeRouter);
app.use("/api/users", usersRouter);
app.use("/api/audit-log", auditLogRouter);
app.use("/api/diag-log", diagLogRouter);
+2
View File
@@ -18,6 +18,8 @@ export const users = sqliteTable("users", {
lastLoginAt: text("last_login_at"),
// The pages this person pinned to the top of their menu, as a JSON array of paths ("/servers", "/alerts"). It follows them to any browser.
favorites: text("favorites"),
// When this person last pressed "Got it" on the dashboard summary of what changed; the next summary starts from here (ISO).
lastLookedAt: text("last_looked_at"),
});
// ─── Audit log ──────────────────────────────────────────────────────────────
+26
View File
@@ -5,6 +5,7 @@ import { db } from "../db/client.js";
import { users } from "../db/schema.js";
import { requireAuth } from "../auth/middleware.js";
import { asyncHandler } from "../utils/asyncHandler.js";
import { FIRST_LOOKBACK_MS, getChangesSince } from "../services/changesSince.js";
export const meRouter = Router();
@@ -43,3 +44,28 @@ meRouter.put("/favorites", requireAuth, asyncHandler(async (req, res) => {
await db.update(users).set({ favorites: JSON.stringify(paths) }).where(eq(users.id, req.currentUser!.id));
res.json({ favorites: paths });
}));
const CHANGES_SHOWN = 8;
// What happened since this person last pressed "Got it" on the dashboard (or in the last day, the first time). Only reads.
meRouter.get("/changes", requireAuth, asyncHandler(async (req, res) => {
const user = req.currentUser!;
const now = new Date();
const stored = user.lastLookedAt ? Date.parse(user.lastLookedAt) : NaN;
const since = new Date(Number.isFinite(stored) ? stored : now.getTime() - FIRST_LOOKBACK_MS);
const report = await getChangesSince({ since, now, actorId: user.id, canSeeAudit: user.role === "admin" || user.role === "operator" });
res.json({
since: report.since,
firstTime: !Number.isFinite(stored),
events: report.events.slice(0, CHANGES_SHOWN),
moreEvents: Math.max(0, report.events.length - CHANGES_SHOWN),
changesByOthers: report.changesByOthers,
});
}));
// "Got it": the next summary starts from now. Only ever for the signed-in person.
meRouter.post("/changes/seen", requireAuth, asyncHandler(async (req, res) => {
const at = new Date().toISOString();
await db.update(users).set({ lastLookedAt: at }).where(eq(users.id, req.currentUser!.id));
res.json({ lastLookedAt: at });
}));
+11
View File
@@ -0,0 +1,11 @@
import { Router } from "express";
import { requireAuth } from "../auth/middleware.js";
import { asyncHandler } from "../utils/asyncHandler.js";
import { getUptime } from "../services/uptimeHistory.js";
export const uptimeRouter = Router();
// Anyone signed in who can see the server and integration lists can see how those have been doing.
uptimeRouter.get("/", requireAuth, asyncHandler(async (_req, res) => {
res.json(await getUptime());
}));
+194
View File
@@ -0,0 +1,194 @@
/**
* "Since you last looked": what happened to the homelab between one visit and the next, in the order a person would tell it —
* servers that stopped reporting or filled a disk, services that stopped answering, maintenance that began or ended — with each
* problem and its recovery folded into one line ("elsa was offline for 12 min"). Worked out from the audit log and the maintenance
* windows, so nothing is stored but the moment the person last pressed "Got it".
*/
import { and, gte } from "drizzle-orm";
import { db } from "../db/client.js";
import { auditLog, integrations, maintenanceWindows, servers } from "../db/schema.js";
import { auditTimeToIso } from "./serverActivity.js";
import { describeTarget } from "./maintenance.js";
export type ChangeTone = "good" | "bad" | "info" | "neutral";
export interface ChangeEvent {
/** ISO — when it began (or, for something already under way before, when it ended). */
at: string;
tone: ChangeTone;
kind: "server" | "integration" | "maintenance";
title: string;
detail: string | null;
/** A page in the app to look at it, or null. */
link: string | null;
}
export interface ChangesReport {
/** ISO — where this summary starts. */
since: string;
/** Everything worth a line, newest first, before it is cut down for display. */
events: ChangeEvent[];
/** How many entries other people (not the app itself, not the asker) wrote in the audit log in that time; null when the asker can't see the audit log. */
changesByOthers: number | null;
}
/** How far back a summary reaches at most, however long since the last visit. */
export const MAX_LOOKBACK_MS = 14 * 24 * 3600_000;
/** Where a first-ever summary starts. */
export const FIRST_LOOKBACK_MS = 24 * 3600_000;
const SERVER_PROBLEM = { went_offline: "offline", disk_full: "disk" } as const;
const SERVER_RECOVERY = { back_online: "offline", disk_ok: "disk" } as const;
function span(ms: number): string {
const minutes = Math.max(1, Math.round(ms / 60_000));
if (minutes < 60) return `${minutes} min`;
const hours = Math.floor(minutes / 60);
if (hours < 24) return minutes % 60 && hours < 10 ? `${hours} h ${minutes % 60} min` : `${hours} h`;
const days = Math.floor(hours / 24);
return hours % 24 ? `${days} d ${hours % 24} h` : `${days} d`;
}
function parseDetail(detail: string | null): Record<string, unknown> {
if (!detail) return {};
try {
const v = JSON.parse(detail);
return v && typeof v === "object" ? (v as Record<string, unknown>) : {};
} catch {
return {};
}
}
interface Mark {
at: number;
problem: boolean;
message: string | null;
}
/** Folds an ordered run of problem / recovery marks into the story: each pair is one finished episode, a lone recovery began before, a lone problem is still going. */
export function foldMarks(marks: Mark[]): { start: number | null; end: number | null; message: string | null }[] {
const out: { start: number | null; end: number | null; message: string | null }[] = [];
let open: Mark | null = null;
for (const m of [...marks].sort((a, b) => a.at - b.at)) {
if (m.problem) {
if (!open) open = m; // a second "problem" while one is open (a restart repeating itself) adds nothing
} else if (open) {
out.push({ start: open.at, end: m.at, message: open.message });
open = null;
} else {
out.push({ start: null, end: m.at, message: null });
}
}
if (open) out.push({ start: open.at, end: null, message: open.message });
return out;
}
export async function getChangesSince(params: { since: Date; now?: Date; actorId: number; canSeeAudit: boolean }): Promise<ChangesReport> {
const now = params.now ?? new Date();
const sinceMs = Math.max(params.since.getTime(), now.getTime() - MAX_LOOKBACK_MS);
const nowMs = now.getTime();
// The audit log stores "2026-10-03 14:05:09"; compare in that form, then filter exactly in JS.
const sinceDb = new Date(sinceMs - 1000).toISOString().slice(0, 19).replace("T", " ");
const audits = (await db.select().from(auditLog).where(gte(auditLog.createdAt, sinceDb))).map((a) => ({ ...a, atMs: Date.parse(auditTimeToIso(a.createdAt)) })).filter((a) => a.atMs <= nowMs);
const events: ChangeEvent[] = [];
// ── Servers: stopped reporting / disk full, and the ends of those ──
const serverMarks = new Map<string, { id: number; kind: "offline" | "disk"; marks: Mark[] }>();
for (const a of audits) {
if (a.category !== "server" || a.targetType !== "server" || a.atMs < sinceMs) continue;
const problem = SERVER_PROBLEM[a.action as keyof typeof SERVER_PROBLEM];
const recovery = SERVER_RECOVERY[a.action as keyof typeof SERVER_RECOVERY];
const kind = problem ?? recovery;
if (!kind) continue;
const mount = typeof parseDetail(a.detail).mount === "string" ? (parseDetail(a.detail).mount as string) : "";
const key = `${a.targetId}:${kind}:${mount}`;
const entry = serverMarks.get(key) ?? { id: Number(a.targetId), kind, marks: [] };
const message = typeof parseDetail(a.detail).message === "string" ? (parseDetail(a.detail).message as string) : null;
entry.marks.push({ at: a.atMs, problem: Boolean(problem), message });
serverMarks.set(key, entry);
}
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((s) => [s.id, s.name]));
for (const { id, kind, marks } of serverMarks.values()) {
const name = serverNames.get(id);
if (!name) continue; // since removed
const link = `/servers/${id}`;
for (const ep of foldMarks(marks)) {
const noun = kind === "offline" ? "offline" : "had a nearly full disk";
if (ep.start !== null && ep.end !== null) {
events.push({ at: new Date(ep.start).toISOString(), tone: "good", kind: "server", link, detail: ep.message, title: kind === "offline" ? `${name} was offline for ${span(ep.end - ep.start)} — reporting again` : `${name} had a nearly full disk for ${span(ep.end - ep.start)} — fine again` });
} else if (ep.start !== null) {
events.push({ at: new Date(ep.start).toISOString(), tone: "bad", kind: "server", link, detail: ep.message, title: kind === "offline" ? `${name} went offline ${span(nowMs - ep.start)} ago — still offline` : `${name} ${noun} — still` });
} else if (ep.end !== null) {
events.push({ at: new Date(ep.end).toISOString(), tone: "good", kind: "server", link, detail: null, title: kind === "offline" ? `${name} is reporting again` : `${name}'s disk is no longer nearly full` });
}
}
}
// ── Integrations: started failing / working again (kept per kind of service, as the health checks are) ──
const sourceMarks = new Map<string, Mark[]>();
for (const a of audits) {
if (a.category !== "integration" || a.targetType !== "integration_type" || !a.targetId || a.atMs < sinceMs) continue;
if (a.action !== "went_failing" && a.action !== "recovered") continue;
const list = sourceMarks.get(a.targetId) ?? [];
list.push({ at: a.atMs, problem: a.action === "went_failing", message: null });
sourceMarks.set(a.targetId, list);
}
if (sourceMarks.size > 0) {
const names = new Map<string, string[]>();
for (const i of await db.select({ name: integrations.name, type: integrations.type }).from(integrations)) names.set(i.type, [...(names.get(i.type) ?? []), i.name]);
for (const [source, marks] of sourceMarks) {
const label = names.get(source)?.join(" / ");
if (!label) continue;
for (const ep of foldMarks(marks)) {
if (ep.start !== null && ep.end !== null) {
events.push({ at: new Date(ep.start).toISOString(), tone: "good", kind: "integration", link: "/integrations", detail: null, title: `${label} stopped answering for ${span(ep.end - ep.start)} — working again` });
} else if (ep.start !== null) {
events.push({ at: new Date(ep.start).toISOString(), tone: "bad", kind: "integration", link: "/integrations", detail: null, title: `${label} stopped answering ${span(nowMs - ep.start)} ago — still failing` });
} else if (ep.end !== null) {
events.push({ at: new Date(ep.end).toISOString(), tone: "good", kind: "integration", link: "/integrations", detail: null, title: `${label} is answering again` });
}
}
}
}
// ── Maintenance: windows that began or finished in the time ──
const sinceIso = new Date(sinceMs).toISOString();
const nowIso = now.toISOString();
const windows = (await db.select().from(maintenanceWindows).where(and(gte(maintenanceWindows.endsAt, sinceIso)))).filter(
(w) => w.endsAt > w.startedAt && w.startedAt <= nowIso && (w.startedAt >= sinceIso || w.endsAt <= nowIso),
);
const groups = new Map<string, typeof windows>();
for (const w of windows) {
const key = `${w.startedAt.slice(0, 16)}|${w.endsAt.slice(0, 16)}|${w.createdBy ?? ""}|${w.reason ?? ""}`;
groups.set(key, [...(groups.get(key) ?? []), w]);
}
for (const group of groups.values()) {
const names: string[] = [];
for (const w of group) {
const t = await describeTarget(w.targetType, w.targetId);
if (t) names.push(t.name);
}
if (names.length === 0) continue;
const ended = group[0].endsAt <= nowIso;
const subject = names.length >= 4 ? `${names.length} items` : names.join(", ");
const verb = names.length === 1 ? (ended ? "was" : "is") : ended ? "were" : "are";
const lasted = span(Date.parse(group[0].endsAt) - Date.parse(group[0].startedAt));
events.push({
at: group[0].startedAt >= sinceIso ? group[0].startedAt : group[0].endsAt,
tone: "info",
kind: "maintenance",
link: "/maintenance",
title: ended ? `${subject} ${verb} in maintenance for ${lasted}` : `${subject} ${verb} in maintenance — alerts silenced`,
detail: group[0].reason,
});
}
events.sort((a, b) => b.at.localeCompare(a.at));
let changesByOthers: number | null = null;
if (params.canSeeAudit) {
changesByOthers = audits.filter((a) => a.atMs >= sinceMs && a.actorUserId !== params.actorId && a.actorLabel !== "system").length;
}
return { since: sinceIso, events, changesByOthers };
}
@@ -1,3 +1,8 @@
import { desc, eq, and } from "drizzle-orm";
import { db } from "../db/client.js";
import { auditLog } from "../db/schema.js";
import { recordAudit } from "./audit.js";
import { ensureHistoryStart } from "./uptimeHistory.js";
import { getSettings } from "./settingsStore.js";
import { isSourceInMaintenance } from "./maintenance.js";
import { notifyIntegrationDown, notifyIntegrationRecovered } from "./notify.js";
@@ -6,6 +11,36 @@ interface SourceHealth {
consecutiveFailures: number;
/** True once a "down" notification has been sent for the current failure streak, so it isn't repeated on every subsequent failure. */
alerted: boolean;
/** True once the audit log has been told this streak reached the failure threshold, so it is written once and its end can be written too. */
recorded?: boolean;
}
/**
* Writes a service's "started failing" / "working again" moment in the audit log (never letting a problem there hold up the call
* that triggered it), so the dashboard can say what changed and the integration pages can draw how it has done over the weeks.
*/
async function recordTransition(source: string, action: "went_failing" | "recovered", detail?: unknown): Promise<void> {
try {
await ensureHistoryStart();
await recordAudit({ category: "integration", action, targetType: "integration_type", targetId: source, detail });
} catch (err) {
console.error("[integration-health] couldn't record a transition:", err);
}
}
/** What the audit log last says about a service, for the first call after a restart (the live streak is only in memory). */
async function lastRecordedTransition(source: string): Promise<"went_failing" | "recovered" | null> {
try {
const [row] = await db
.select({ action: auditLog.action })
.from(auditLog)
.where(and(eq(auditLog.category, "integration"), eq(auditLog.targetType, "integration_type"), eq(auditLog.targetId, source)))
.orderBy(desc(auditLog.createdAt), desc(auditLog.id))
.limit(1);
return row?.action === "went_failing" || row?.action === "recovered" ? row.action : null;
} catch {
return null;
}
}
/**
@@ -38,18 +73,26 @@ export function getSourceStatus(source: string): { status: "ok" | "failing" | "u
/** Called after every diagnostic-log entry is recorded, to track consecutive failures per source and alert on threshold-cross / recovery. */
export async function trackIntegrationHealth(source: string, ok: boolean): Promise<void> {
const seenBefore = health.has(source);
const state = health.get(source) ?? { consecutiveFailures: 0, alerted: false };
if (ok) {
if (state.alerted) {
await notifyIntegrationRecovered(source);
}
if (state.recorded || (!seenBefore && (await lastRecordedTransition(source)) === "went_failing")) {
await recordTransition(source, "recovered");
}
health.set(source, { consecutiveFailures: 0, alerted: false });
return;
}
state.consecutiveFailures += 1;
const { notifications } = await getSettings();
if (!state.recorded && state.consecutiveFailures >= notifications.integrationFailureThreshold) {
state.recorded = true;
await recordTransition(source, "went_failing", { failures: state.consecutiveFailures });
}
if (notifications.integrationFailureAlerts && !state.alerted && state.consecutiveFailures >= notifications.integrationFailureThreshold) {
// During maintenance the failures keep being counted but `alerted` stays false, so if the service is
// still failing once the window ends, the very next failed call alerts — a real outage isn't swallowed.
+164
View File
@@ -0,0 +1,164 @@
/**
* How each server and integration has done over the last weeks, as one figure per day: how long it was up, how long down, and for
* how much of the day anything is known. It is worked out from the moments the health checks wrote down in the audit log (a
* server stopped reporting / reported again; an integration's calls started failing / worked again), so it needs no table of its
* own — and it can only be as old as those records.
*/
import { inArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { auditLog, integrations, servers } from "../db/schema.js";
import { auditTimeToIso } from "./serverActivity.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
const DAY_MS = 24 * 3600_000;
const HISTORY_FLAG = "integration_history_since";
export const DEFAULT_DAYS = 30;
export interface Transition {
/** Epoch ms. */
at: number;
down: boolean;
}
/** Minutes of one 24-hour stretch that are known, and of those, minutes spent down. */
export interface UptimeBucket {
k: number;
d: number;
}
export interface UptimeSeries {
/** Oldest first; the last bucket ends now. */
buckets: UptimeBucket[];
/** Share of the known time spent up over the whole period. */
percent: number;
}
/**
* Splits [knownFrom, now] into up and down stretches from the recorded transitions and lays them over `days` consecutive 24-hour
* buckets ending at `now`. Before the first transition the thing is taken to have been in the opposite state (a "went offline" means it
* was up until then); with no transitions at all it has been up the whole time.
*/
export function computeBuckets(transitions: Transition[], knownFrom: number, now: number, days: number): UptimeBucket[] {
const start = now - days * DAY_MS;
const sorted = [...transitions].sort((a, b) => a.at - b.at);
// State at knownFrom: after any transitions that happened before it, otherwise the opposite of the first one that follows.
let state = false;
let i = 0;
while (i < sorted.length && sorted[i].at <= knownFrom) state = sorted[i++].down;
if (i === 0 && sorted.length > 0) state = !sorted[0].down;
const segments: { from: number; to: number; down: boolean }[] = [];
let cursor = Math.min(knownFrom, now);
for (; i < sorted.length; i++) {
const t = Math.min(sorted[i].at, now);
if (t > cursor) segments.push({ from: cursor, to: t, down: state });
cursor = Math.max(cursor, t);
state = sorted[i].down;
}
if (now > cursor) segments.push({ from: cursor, to: now, down: state });
const known = new Array<number>(days).fill(0);
const down = new Array<number>(days).fill(0);
for (const seg of segments) {
if (seg.to <= start) continue;
const first = Math.max(0, Math.floor((Math.max(seg.from, start) - start) / DAY_MS));
for (let b = first; b < days; b++) {
const bStart = start + b * DAY_MS;
if (bStart >= seg.to) break;
const overlap = Math.min(seg.to, bStart + DAY_MS) - Math.max(seg.from, bStart);
if (overlap <= 0) continue;
known[b] += overlap;
if (seg.down) down[b] += overlap;
}
}
return known.map((k, b) => ({ k: Math.round(k / 60_000), d: Math.round(down[b] / 60_000) }));
}
/** Share of the known time spent up, to one decimal; null when nothing is known. */
export function uptimePercent(buckets: UptimeBucket[]): number | null {
const known = buckets.reduce((n, b) => n + b.k, 0);
if (known === 0) return null;
const down = buckets.reduce((n, b) => n + b.d, 0);
return Math.round((1 - down / known) * 1000) / 10;
}
function series(transitions: Transition[], knownFrom: number, now: number, days: number): UptimeSeries | null {
const buckets = computeBuckets(transitions, knownFrom, now, days);
const percent = uptimePercent(buckets);
return percent === null ? null : { buckets, percent };
}
/**
* Integrations' failing / working-again moments only exist from the version that began writing them, so nothing earlier can be claimed.
* The first time anything asks, that moment is written down and kept.
*/
let historyStart: number | null = null;
export async function ensureHistoryStart(now: number = Date.now()): Promise<number> {
if (historyStart !== null) return historyStart;
const stored = await getInternalFlag(HISTORY_FLAG);
const parsed = stored ? Date.parse(stored) : NaN;
if (Number.isFinite(parsed)) {
historyStart = parsed;
return parsed;
}
await setInternalFlag(HISTORY_FLAG, new Date(now).toISOString());
historyStart = now;
return now;
}
/** For tests: forget the remembered start. */
export function resetHistoryStartForTests(): void {
historyStart = null;
}
export interface UptimeReport {
days: number;
/** ISO — the end of the last bucket. */
endsAt: string;
servers: Record<number, UptimeSeries>;
integrations: Record<number, UptimeSeries>;
}
export async function getUptime(now: Date = new Date(), days: number = DEFAULT_DAYS): Promise<UptimeReport> {
const nowMs = now.getTime();
const rows = await db.select().from(auditLog).where(inArray(auditLog.action, ["went_offline", "back_online", "went_failing", "recovered"]));
const serverEvents = new Map<number, Transition[]>();
const sourceEvents = new Map<string, Transition[]>();
for (const r of rows) {
const at = Date.parse(auditTimeToIso(r.createdAt));
if (!Number.isFinite(at) || at > nowMs) continue;
if (r.category === "server" && r.targetType === "server" && (r.action === "went_offline" || r.action === "back_online")) {
const id = Number(r.targetId);
const list = serverEvents.get(id) ?? [];
list.push({ at, down: r.action === "went_offline" });
serverEvents.set(id, list);
} else if (r.category === "integration" && r.targetType === "integration_type" && r.targetId && (r.action === "went_failing" || r.action === "recovered")) {
const list = sourceEvents.get(r.targetId) ?? [];
list.push({ at, down: r.action === "went_failing" });
sourceEvents.set(r.targetId, list);
}
}
const out: UptimeReport = { days, endsAt: now.toISOString(), servers: {}, integrations: {} };
for (const s of await db.select({ id: servers.id, createdAt: servers.createdAt, lastSeenAt: servers.lastSeenAt }).from(servers)) {
const events = serverEvents.get(s.id) ?? [];
// A server that has never reported and never been seen to stop has no history to speak of.
if (!s.lastSeenAt && events.length === 0) continue;
const created = Date.parse(auditTimeToIso(s.createdAt));
const result = series(events, Number.isFinite(created) ? created : nowMs, nowMs, days);
if (result) out.servers[s.id] = result;
}
const since = await ensureHistoryStart(nowMs);
for (const i of await db.select({ id: integrations.id, type: integrations.type, createdAt: integrations.createdAt }).from(integrations)) {
const created = Date.parse(auditTimeToIso(i.createdAt));
const knownFrom = Math.max(Number.isFinite(created) ? created : 0, since);
const result = series(sourceEvents.get(i.type) ?? [], knownFrom, nowMs, days);
if (result) out.integrations[i.id] = result;
}
return out;
}
+299
View File
@@ -0,0 +1,299 @@
/**
* The 30-day uptime ribbons and the dashboard's "since you last looked" summary — both built from what the health checks wrote
* down in the audit log.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-uptime-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, integrations, auditLog, maintenanceWindows } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { meRouter } = await import("../src/routes/me.js");
const { uptimeRouter } = await import("../src/routes/uptime.js");
const { computeBuckets, uptimePercent, getUptime } = await import("../src/services/uptimeHistory.js");
const { foldMarks, getChangesSince } = await import("../src/services/changesSince.js");
const { trackIntegrationHealth } = await import("../src/services/integrationHealthMonitor.js");
await runMigrations();
const HOUR = 3600_000;
const DAY = 24 * HOUR;
const NOW = Math.floor(Date.now() / 1000) * 1000;
const dbTime = (ms: number) => new Date(ms).toISOString().slice(0, 19).replace("T", " ");
describe("computeBuckets", () => {
test("a thing with no problems recorded has been up for as long as it has been known", () => {
const buckets = computeBuckets([], NOW - 10 * DAY, NOW, 30);
assert.equal(buckets.length, 30);
assert.deepEqual(buckets.slice(0, 20).map((b) => b.k), new Array(20).fill(0), "nothing is known before it existed");
assert.ok(buckets.slice(20).every((b) => b.k === 1440 && b.d === 0));
assert.equal(uptimePercent(buckets), 100);
});
test("a stretch offline shows as down minutes in the day it happened", () => {
const buckets = computeBuckets(
[
{ at: NOW - 2 * HOUR, down: true },
{ at: NOW - HOUR, down: false },
],
NOW - 30 * DAY,
NOW,
30,
);
assert.equal(buckets[29].d, 60);
assert.equal(buckets[29].k, 1440);
assert.equal(buckets.slice(0, 29).reduce((n, b) => n + b.d, 0), 0);
assert.equal(uptimePercent(buckets), Math.round((1 - 60 / (30 * 1440)) * 1000) / 10);
});
test("an outage that spans midnight is split between the days, and one still going counts up to now", () => {
const buckets = computeBuckets([{ at: NOW - 36 * HOUR, down: true }], NOW - 30 * DAY, NOW, 30);
assert.equal(buckets[29].d, 1440);
assert.equal(buckets[28].d, 12 * 60);
});
test("a first record of recovery means it was down before that", () => {
const buckets = computeBuckets([{ at: NOW - 12 * HOUR, down: false }], NOW - 3 * DAY, NOW, 30);
assert.equal(buckets[28].d, 1440);
assert.equal(buckets[27].d, 1440);
assert.equal(buckets[29].d, 12 * 60, "down until the recovery 12 hours ago");
assert.equal(buckets[26].k, 0, "nothing known before it existed");
});
test("repeated 'down' marks don't double anything", () => {
const once = computeBuckets([{ at: NOW - 5 * HOUR, down: true }, { at: NOW - 3 * HOUR, down: false }], NOW - 5 * DAY, NOW, 30);
const twice = computeBuckets([{ at: NOW - 5 * HOUR, down: true }, { at: NOW - 4 * HOUR, down: true }, { at: NOW - 3 * HOUR, down: false }], NOW - 5 * DAY, NOW, 30);
assert.deepEqual(once, twice);
});
});
describe("foldMarks", () => {
test("pairs, lone recoveries and open problems", () => {
const folded = foldMarks([
{ at: 1, problem: false, message: null },
{ at: 5, problem: true, message: "a" },
{ at: 6, problem: true, message: "dup" },
{ at: 9, problem: false, message: null },
{ at: 20, problem: true, message: "b" },
]);
assert.deepEqual(folded, [
{ start: null, end: 1, message: null },
{ start: 5, end: 9, message: "a" },
{ start: 20, end: null, message: "b" },
]);
});
});
describe("recording integration transitions", () => {
test("the third failure is written down once, and the next success ends it", async () => {
await trackIntegrationHealth("ribbon-a", false);
await trackIntegrationHealth("ribbon-a", false);
assert.equal((await db.select().from(auditLog)).filter((e) => e.targetId === "ribbon-a").length, 0, "two failures are not an outage yet");
await trackIntegrationHealth("ribbon-a", false);
await trackIntegrationHealth("ribbon-a", false);
await trackIntegrationHealth("ribbon-a", true);
await trackIntegrationHealth("ribbon-a", true);
const rows = (await db.select().from(auditLog)).filter((e) => e.targetId === "ribbon-a");
assert.deepEqual(rows.map((r) => r.action), ["went_failing", "recovered"]);
assert.ok(rows.every((r) => r.category === "integration" && r.targetType === "integration_type"));
});
test("a service the log last saw failing is closed off by its first success after a restart", async () => {
await db.insert(auditLog).values({ category: "integration", action: "went_failing", targetType: "integration_type", targetId: "ribbon-b", actorLabel: "system" });
await trackIntegrationHealth("ribbon-b", true); // nothing in memory about ribbon-b: this is the first call since "start"
const rows = (await db.select().from(auditLog)).filter((e) => e.targetId === "ribbon-b");
assert.deepEqual(rows.map((r) => r.action), ["went_failing", "recovered"]);
await trackIntegrationHealth("ribbon-b", true);
assert.equal((await db.select().from(auditLog)).filter((e) => e.targetId === "ribbon-b").length, 2, "and only once");
});
test("a healthy service writes nothing", async () => {
await trackIntegrationHealth("ribbon-c", true);
await trackIntegrationHealth("ribbon-c", true);
assert.equal((await db.select().from(auditLog)).filter((e) => e.targetId === "ribbon-c").length, 0);
});
});
describe("uptime and changes from the audit log", () => {
let app: Server;
let base = "";
let who = "admin";
let elsaId = 0;
let annaId = 0;
before(async () => {
await db.insert(users).values([
{ oidcSub: "admin", name: "Ada", role: "admin" },
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
]);
const made = await db
.insert(servers)
.values([
{ name: "elsa", apiTokenHash: "a", apiTokenPrefix: "a", createdAt: dbTime(NOW - 40 * DAY), lastSeenAt: new Date(NOW).toISOString() },
{ name: "anna", apiTokenHash: "b", apiTokenPrefix: "b", createdAt: dbTime(NOW - 40 * DAY), lastSeenAt: new Date(NOW).toISOString() },
{ name: "never", apiTokenHash: "c", apiTokenPrefix: "c", createdAt: dbTime(NOW - 40 * DAY) },
])
.returning({ id: servers.id });
elsaId = made[0].id;
annaId = made[1].id;
await db.insert(integrations).values({ type: "proxmox", name: "Felicia", baseUrl: "https://pve.lab", config: "{}", createdAt: dbTime(NOW - 40 * DAY) });
const entry = (offsetMs: number, category: string, action: string, targetType: string, targetId: string | number, extra: Record<string, unknown> = {}) => ({
category,
action,
targetType,
targetId: String(targetId),
actorLabel: "system",
createdAt: dbTime(NOW + offsetMs),
...extra,
});
await db.insert(auditLog).values([
// elsa: offline for an hour, a while ago — finished
entry(-5 * HOUR, "server", "went_offline", "server", elsaId, { detail: JSON.stringify({ message: "No report for 5 minutes" }) }),
entry(-4 * HOUR, "server", "back_online", "server", elsaId),
// anna: went offline and has not come back
entry(-3 * HOUR, "server", "went_offline", "server", annaId),
// a service that stopped answering and came back
entry(-6 * HOUR, "integration", "went_failing", "integration_type", "proxmox"),
entry(-5.5 * HOUR, "integration", "recovered", "integration_type", "proxmox"),
// somebody else's edits, and one of Ada's own
{ category: "secret", action: "update", targetType: "secret", targetId: "1", actorLabel: "Olga", createdAt: dbTime(NOW - 2 * HOUR) },
{ category: "secret", action: "update", targetType: "secret", targetId: "2", actorLabel: "Olga", createdAt: dbTime(NOW - 2 * HOUR) },
// before the window we ask about
entry(-3 * DAY, "server", "went_offline", "server", elsaId),
]);
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/me", meRouter);
a.use("/api/uptime", uptimeRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("uptime covers servers that have reported, gives them a percentage, and skips one that never has", async () => {
const report = await getUptime(new Date(NOW), 30);
assert.ok(report.servers[elsaId]);
assert.ok(report.servers[annaId]);
assert.equal(Object.keys(report.servers).length, 2);
assert.equal(report.servers[annaId].buckets.length, 30);
// anna stopped reporting 3 hours ago and the log has not seen her come back
assert.equal(report.servers[annaId].buckets[29].d, 180);
// elsa went offline 3 days ago, was recorded offline again and then back 4 hours ago: down from then until then
assert.equal(report.servers[elsaId].buckets[29].d, 20 * 60);
assert.equal(report.servers[elsaId].buckets[28].d, 1440);
assert.equal(report.servers[elsaId].buckets[27].d, 1440);
assert.equal(report.servers[elsaId].buckets[26].d, 0);
assert.ok(report.servers[elsaId].percent < 100 && report.servers[elsaId].percent > 90);
});
test("the uptime route answers anyone signed in", async () => {
who = "viewer";
const res = await fetch(`${base}/uptime`);
assert.equal(res.status, 200);
const body = await res.json();
assert.equal(body.days, 30);
who = "admin";
});
test("changes since: pairs, an open problem, a service blip, and what others did", async () => {
const adaId = (await db.select().from(users)).find((u) => u.oidcSub === "admin")!.id;
const report = await getChangesSince({ since: new Date(NOW - 12 * HOUR), now: new Date(NOW), actorId: adaId, canSeeAudit: true });
const titles = report.events.map((e) => e.title);
assert.ok(titles.some((t) => /^elsa was offline for 1 h — reporting again/.test(t)), titles.join(" | "));
assert.ok(titles.some((t) => /^anna went offline 3 h ago — still offline/.test(t)), titles.join(" | "));
assert.ok(titles.some((t) => /^Felicia stopped answering for 30 min — working again/.test(t)), titles.join(" | "));
const anna = report.events.find((e) => e.title.startsWith("anna"));
assert.equal(anna?.tone, "bad");
assert.equal(anna?.link, `/servers/${annaId}`);
assert.equal(report.events.find((e) => e.title.startsWith("elsa"))?.detail, "No report for 5 minutes");
// Olga's two edits count; the app's own entries don't.
assert.equal(report.changesByOthers, 2);
});
test("a viewer isn't told about the audit log", async () => {
const viewerId = (await db.select().from(users)).find((u) => u.oidcSub === "viewer")!.id;
const report = await getChangesSince({ since: new Date(NOW - 12 * HOUR), now: new Date(NOW), actorId: viewerId, canSeeAudit: false });
assert.equal(report.changesByOthers, null);
});
test("nothing before the start of the summary is in it", async () => {
const report = await getChangesSince({ since: new Date(NOW - 2 * HOUR), now: new Date(NOW), actorId: 0, canSeeAudit: true });
assert.equal(report.events.filter((e) => e.kind !== "maintenance").length, 0, "the problems were all earlier than that");
});
test("maintenance that began, ended or was started in bulk is told in few lines", async () => {
const mk = (hoursAgoStart: number, hoursAgoEnd: number, target: number, reason: string) => ({
targetType: "server" as const,
targetId: target,
reason,
startedAt: new Date(NOW - hoursAgoStart * HOUR).toISOString(),
endsAt: new Date(NOW - hoursAgoEnd * HOUR).toISOString(),
createdBy: "Ada",
});
const extra = await db
.insert(servers)
.values([1, 2, 3, 4].map((n) => ({ name: `bulk${n}`, apiTokenHash: `h${n}`, apiTokenPrefix: `p${n}` })))
.returning({ id: servers.id });
await db.insert(maintenanceWindows).values([
mk(10, 8, elsaId, "Kernel update"), // finished
...extra.map((s) => mk(7, -2, s.id, "Power work")), // four at once, still running
mk(1, 1, annaId, "withdrawn"), // cancelled before it began
]);
const report = await getChangesSince({ since: new Date(NOW - 12 * HOUR), now: new Date(NOW), actorId: 0, canSeeAudit: false });
const maintenance = report.events.filter((e) => e.kind === "maintenance");
assert.equal(maintenance.length, 2, maintenance.map((m) => m.title).join(" | "));
assert.ok(maintenance.some((m) => m.title === "elsa was in maintenance for 2 h" && m.detail === "Kernel update"));
assert.ok(maintenance.some((m) => m.title === "4 items are in maintenance — alerts silenced" && m.detail === "Power work"));
});
test("the route starts from a day ago the first time, then from where 'Got it' was pressed", async () => {
let res = await fetch(`${base}/me/changes`);
let body = await res.json();
assert.equal(body.firstTime, true);
assert.ok(Array.isArray(body.events));
assert.equal((await fetch(`${base}/me/changes/seen`, { method: "POST" })).status, 200);
res = await fetch(`${base}/me/changes`);
body = await res.json();
assert.equal(body.firstTime, false);
assert.equal(body.events.length, 0, "nothing has happened since just now");
assert.ok(Math.abs(Date.parse(body.since) - Date.now()) < 60_000);
});
test("'Got it' only moves the signed-in person's own mark", async () => {
who = "viewer";
const res = await fetch(`${base}/me/changes`);
assert.equal((await res.json()).firstTime, true);
who = "admin";
});
});