Add a since-you-last-looked summary, 30-day uptime ribbons, a maintenance timeline and hover timestamps

- Dashboard card on what changed since the user pressed Got it (users.last_looked_at,
  migration 0021), folding each problem and its recovery into one line.
- Uptime ribbons on servers and integrations, built from the audit log; integrations'
  failing/working-again moments are now recorded.
- Timeline of running and scheduled maintenance windows on the Maintenance page.
- TimeAgo component: relative times with the exact time in the chosen format on hover.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-16 21:55:04 +02:00
1 parent 3945c60681
commit 906220040e
32 files changed
+3380 -15

No files matched your search

+194
View File
@@ -0,0 +1,194 @@
/**
* "Since you last looked": what happened to the homelab between one visit and the next, in the order a person would tell it —
* servers that stopped reporting or filled a disk, services that stopped answering, maintenance that began or ended — with each
* problem and its recovery folded into one line ("elsa was offline for 12 min"). Worked out from the audit log and the maintenance
* windows, so nothing is stored but the moment the person last pressed "Got it".
*/
import { and, gte } from "drizzle-orm";
import { db } from "../db/client.js";
import { auditLog, integrations, maintenanceWindows, servers } from "../db/schema.js";
import { auditTimeToIso } from "./serverActivity.js";
import { describeTarget } from "./maintenance.js";
export type ChangeTone = "good" | "bad" | "info" | "neutral";
export interface ChangeEvent {
/** ISO — when it began (or, for something already under way before, when it ended). */
at: string;
tone: ChangeTone;
kind: "server" | "integration" | "maintenance";
title: string;
detail: string | null;
/** A page in the app to look at it, or null. */
link: string | null;
}
export interface ChangesReport {
/** ISO — where this summary starts. */
since: string;
/** Everything worth a line, newest first, before it is cut down for display. */
events: ChangeEvent[];
/** How many entries other people (not the app itself, not the asker) wrote in the audit log in that time; null when the asker can't see the audit log. */
changesByOthers: number | null;
}
/** How far back a summary reaches at most, however long since the last visit. */
export const MAX_LOOKBACK_MS = 14 * 24 * 3600_000;
/** Where a first-ever summary starts. */
export const FIRST_LOOKBACK_MS = 24 * 3600_000;
const SERVER_PROBLEM = { went_offline: "offline", disk_full: "disk" } as const;
const SERVER_RECOVERY = { back_online: "offline", disk_ok: "disk" } as const;
function span(ms: number): string {
const minutes = Math.max(1, Math.round(ms / 60_000));
if (minutes < 60) return `${minutes} min`;
const hours = Math.floor(minutes / 60);
if (hours < 24) return minutes % 60 && hours < 10 ? `${hours} h ${minutes % 60} min` : `${hours} h`;
const days = Math.floor(hours / 24);
return hours % 24 ? `${days} d ${hours % 24} h` : `${days} d`;
}
function parseDetail(detail: string | null): Record<string, unknown> {
if (!detail) return {};
try {
const v = JSON.parse(detail);
return v && typeof v === "object" ? (v as Record<string, unknown>) : {};
} catch {
return {};
}
}
interface Mark {
at: number;
problem: boolean;
message: string | null;
}
/** Folds an ordered run of problem / recovery marks into the story: each pair is one finished episode, a lone recovery began before, a lone problem is still going. */
export function foldMarks(marks: Mark[]): { start: number | null; end: number | null; message: string | null }[] {
const out: { start: number | null; end: number | null; message: string | null }[] = [];
let open: Mark | null = null;
for (const m of [...marks].sort((a, b) => a.at - b.at)) {
if (m.problem) {
if (!open) open = m; // a second "problem" while one is open (a restart repeating itself) adds nothing
} else if (open) {
out.push({ start: open.at, end: m.at, message: open.message });
open = null;
} else {
out.push({ start: null, end: m.at, message: null });
}
}
if (open) out.push({ start: open.at, end: null, message: open.message });
return out;
}
export async function getChangesSince(params: { since: Date; now?: Date; actorId: number; canSeeAudit: boolean }): Promise<ChangesReport> {
const now = params.now ?? new Date();
const sinceMs = Math.max(params.since.getTime(), now.getTime() - MAX_LOOKBACK_MS);
const nowMs = now.getTime();
// The audit log stores "2026-10-03 14:05:09"; compare in that form, then filter exactly in JS.
const sinceDb = new Date(sinceMs - 1000).toISOString().slice(0, 19).replace("T", " ");
const audits = (await db.select().from(auditLog).where(gte(auditLog.createdAt, sinceDb))).map((a) => ({ ...a, atMs: Date.parse(auditTimeToIso(a.createdAt)) })).filter((a) => a.atMs <= nowMs);
const events: ChangeEvent[] = [];
// ── Servers: stopped reporting / disk full, and the ends of those ──
const serverMarks = new Map<string, { id: number; kind: "offline" | "disk"; marks: Mark[] }>();
for (const a of audits) {
if (a.category !== "server" || a.targetType !== "server" || a.atMs < sinceMs) continue;
const problem = SERVER_PROBLEM[a.action as keyof typeof SERVER_PROBLEM];
const recovery = SERVER_RECOVERY[a.action as keyof typeof SERVER_RECOVERY];
const kind = problem ?? recovery;
if (!kind) continue;
const mount = typeof parseDetail(a.detail).mount === "string" ? (parseDetail(a.detail).mount as string) : "";
const key = `${a.targetId}:${kind}:${mount}`;
const entry = serverMarks.get(key) ?? { id: Number(a.targetId), kind, marks: [] };
const message = typeof parseDetail(a.detail).message === "string" ? (parseDetail(a.detail).message as string) : null;
entry.marks.push({ at: a.atMs, problem: Boolean(problem), message });
serverMarks.set(key, entry);
}
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((s) => [s.id, s.name]));
for (const { id, kind, marks } of serverMarks.values()) {
const name = serverNames.get(id);
if (!name) continue; // since removed
const link = `/servers/${id}`;
for (const ep of foldMarks(marks)) {
const noun = kind === "offline" ? "offline" : "had a nearly full disk";
if (ep.start !== null && ep.end !== null) {
events.push({ at: new Date(ep.start).toISOString(), tone: "good", kind: "server", link, detail: ep.message, title: kind === "offline" ? `${name} was offline for ${span(ep.end - ep.start)} — reporting again` : `${name} had a nearly full disk for ${span(ep.end - ep.start)} — fine again` });
} else if (ep.start !== null) {
events.push({ at: new Date(ep.start).toISOString(), tone: "bad", kind: "server", link, detail: ep.message, title: kind === "offline" ? `${name} went offline ${span(nowMs - ep.start)} ago — still offline` : `${name} ${noun} — still` });
} else if (ep.end !== null) {
events.push({ at: new Date(ep.end).toISOString(), tone: "good", kind: "server", link, detail: null, title: kind === "offline" ? `${name} is reporting again` : `${name}'s disk is no longer nearly full` });
}
}
}
// ── Integrations: started failing / working again (kept per kind of service, as the health checks are) ──
const sourceMarks = new Map<string, Mark[]>();
for (const a of audits) {
if (a.category !== "integration" || a.targetType !== "integration_type" || !a.targetId || a.atMs < sinceMs) continue;
if (a.action !== "went_failing" && a.action !== "recovered") continue;
const list = sourceMarks.get(a.targetId) ?? [];
list.push({ at: a.atMs, problem: a.action === "went_failing", message: null });
sourceMarks.set(a.targetId, list);
}
if (sourceMarks.size > 0) {
const names = new Map<string, string[]>();
for (const i of await db.select({ name: integrations.name, type: integrations.type }).from(integrations)) names.set(i.type, [...(names.get(i.type) ?? []), i.name]);
for (const [source, marks] of sourceMarks) {
const label = names.get(source)?.join(" / ");
if (!label) continue;
for (const ep of foldMarks(marks)) {
if (ep.start !== null && ep.end !== null) {
events.push({ at: new Date(ep.start).toISOString(), tone: "good", kind: "integration", link: "/integrations", detail: null, title: `${label} stopped answering for ${span(ep.end - ep.start)} — working again` });
} else if (ep.start !== null) {
events.push({ at: new Date(ep.start).toISOString(), tone: "bad", kind: "integration", link: "/integrations", detail: null, title: `${label} stopped answering ${span(nowMs - ep.start)} ago — still failing` });
} else if (ep.end !== null) {
events.push({ at: new Date(ep.end).toISOString(), tone: "good", kind: "integration", link: "/integrations", detail: null, title: `${label} is answering again` });
}
}
}
}
// ── Maintenance: windows that began or finished in the time ──
const sinceIso = new Date(sinceMs).toISOString();
const nowIso = now.toISOString();
const windows = (await db.select().from(maintenanceWindows).where(and(gte(maintenanceWindows.endsAt, sinceIso)))).filter(
(w) => w.endsAt > w.startedAt && w.startedAt <= nowIso && (w.startedAt >= sinceIso || w.endsAt <= nowIso),
);
const groups = new Map<string, typeof windows>();
for (const w of windows) {
const key = `${w.startedAt.slice(0, 16)}|${w.endsAt.slice(0, 16)}|${w.createdBy ?? ""}|${w.reason ?? ""}`;
groups.set(key, [...(groups.get(key) ?? []), w]);
}
for (const group of groups.values()) {
const names: string[] = [];
for (const w of group) {
const t = await describeTarget(w.targetType, w.targetId);
if (t) names.push(t.name);
}
if (names.length === 0) continue;
const ended = group[0].endsAt <= nowIso;
const subject = names.length >= 4 ? `${names.length} items` : names.join(", ");
const verb = names.length === 1 ? (ended ? "was" : "is") : ended ? "were" : "are";
const lasted = span(Date.parse(group[0].endsAt) - Date.parse(group[0].startedAt));
events.push({
at: group[0].startedAt >= sinceIso ? group[0].startedAt : group[0].endsAt,
tone: "info",
kind: "maintenance",
link: "/maintenance",
title: ended ? `${subject} ${verb} in maintenance for ${lasted}` : `${subject} ${verb} in maintenance — alerts silenced`,
detail: group[0].reason,
});
}
events.sort((a, b) => b.at.localeCompare(a.at));
let changesByOthers: number | null = null;
if (params.canSeeAudit) {
changesByOthers = audits.filter((a) => a.atMs >= sinceMs && a.actorUserId !== params.actorId && a.actorLabel !== "system").length;
}
return { since: sinceIso, events, changesByOthers };
}
@@ -1,3 +1,8 @@
import { desc, eq, and } from "drizzle-orm";
import { db } from "../db/client.js";
import { auditLog } from "../db/schema.js";
import { recordAudit } from "./audit.js";
import { ensureHistoryStart } from "./uptimeHistory.js";
import { getSettings } from "./settingsStore.js";
import { isSourceInMaintenance } from "./maintenance.js";
import { notifyIntegrationDown, notifyIntegrationRecovered } from "./notify.js";
@@ -6,6 +11,36 @@ interface SourceHealth {
consecutiveFailures: number;
/** True once a "down" notification has been sent for the current failure streak, so it isn't repeated on every subsequent failure. */
alerted: boolean;
/** True once the audit log has been told this streak reached the failure threshold, so it is written once and its end can be written too. */
recorded?: boolean;
}
/**
* Writes a service's "started failing" / "working again" moment in the audit log (never letting a problem there hold up the call
* that triggered it), so the dashboard can say what changed and the integration pages can draw how it has done over the weeks.
*/
async function recordTransition(source: string, action: "went_failing" | "recovered", detail?: unknown): Promise<void> {
try {
await ensureHistoryStart();
await recordAudit({ category: "integration", action, targetType: "integration_type", targetId: source, detail });
} catch (err) {
console.error("[integration-health] couldn't record a transition:", err);
}
}
/** What the audit log last says about a service, for the first call after a restart (the live streak is only in memory). */
async function lastRecordedTransition(source: string): Promise<"went_failing" | "recovered" | null> {
try {
const [row] = await db
.select({ action: auditLog.action })
.from(auditLog)
.where(and(eq(auditLog.category, "integration"), eq(auditLog.targetType, "integration_type"), eq(auditLog.targetId, source)))
.orderBy(desc(auditLog.createdAt), desc(auditLog.id))
.limit(1);
return row?.action === "went_failing" || row?.action === "recovered" ? row.action : null;
} catch {
return null;
}
}
/**
@@ -38,18 +73,26 @@ export function getSourceStatus(source: string): { status: "ok" | "failing" | "u
/** Called after every diagnostic-log entry is recorded, to track consecutive failures per source and alert on threshold-cross / recovery. */
export async function trackIntegrationHealth(source: string, ok: boolean): Promise<void> {
const seenBefore = health.has(source);
const state = health.get(source) ?? { consecutiveFailures: 0, alerted: false };
if (ok) {
if (state.alerted) {
await notifyIntegrationRecovered(source);
}
if (state.recorded || (!seenBefore && (await lastRecordedTransition(source)) === "went_failing")) {
await recordTransition(source, "recovered");
}
health.set(source, { consecutiveFailures: 0, alerted: false });
return;
}
state.consecutiveFailures += 1;
const { notifications } = await getSettings();
if (!state.recorded && state.consecutiveFailures >= notifications.integrationFailureThreshold) {
state.recorded = true;
await recordTransition(source, "went_failing", { failures: state.consecutiveFailures });
}
if (notifications.integrationFailureAlerts && !state.alerted && state.consecutiveFailures >= notifications.integrationFailureThreshold) {
// During maintenance the failures keep being counted but `alerted` stays false, so if the service is
// still failing once the window ends, the very next failed call alerts — a real outage isn't swallowed.
+164
View File
@@ -0,0 +1,164 @@
/**
* How each server and integration has done over the last weeks, as one figure per day: how long it was up, how long down, and for
* how much of the day anything is known. It is worked out from the moments the health checks wrote down in the audit log (a
* server stopped reporting / reported again; an integration's calls started failing / worked again), so it needs no table of its
* own — and it can only be as old as those records.
*/
import { inArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { auditLog, integrations, servers } from "../db/schema.js";
import { auditTimeToIso } from "./serverActivity.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
const DAY_MS = 24 * 3600_000;
const HISTORY_FLAG = "integration_history_since";
export const DEFAULT_DAYS = 30;
export interface Transition {
/** Epoch ms. */
at: number;
down: boolean;
}
/** Minutes of one 24-hour stretch that are known, and of those, minutes spent down. */
export interface UptimeBucket {
k: number;
d: number;
}
export interface UptimeSeries {
/** Oldest first; the last bucket ends now. */
buckets: UptimeBucket[];
/** Share of the known time spent up over the whole period. */
percent: number;
}
/**
* Splits [knownFrom, now] into up and down stretches from the recorded transitions and lays them over `days` consecutive 24-hour
* buckets ending at `now`. Before the first transition the thing is taken to have been in the opposite state (a "went offline" means it
* was up until then); with no transitions at all it has been up the whole time.
*/
export function computeBuckets(transitions: Transition[], knownFrom: number, now: number, days: number): UptimeBucket[] {
const start = now - days * DAY_MS;
const sorted = [...transitions].sort((a, b) => a.at - b.at);
// State at knownFrom: after any transitions that happened before it, otherwise the opposite of the first one that follows.
let state = false;
let i = 0;
while (i < sorted.length && sorted[i].at <= knownFrom) state = sorted[i++].down;
if (i === 0 && sorted.length > 0) state = !sorted[0].down;
const segments: { from: number; to: number; down: boolean }[] = [];
let cursor = Math.min(knownFrom, now);
for (; i < sorted.length; i++) {
const t = Math.min(sorted[i].at, now);
if (t > cursor) segments.push({ from: cursor, to: t, down: state });
cursor = Math.max(cursor, t);
state = sorted[i].down;
}
if (now > cursor) segments.push({ from: cursor, to: now, down: state });
const known = new Array<number>(days).fill(0);
const down = new Array<number>(days).fill(0);
for (const seg of segments) {
if (seg.to <= start) continue;
const first = Math.max(0, Math.floor((Math.max(seg.from, start) - start) / DAY_MS));
for (let b = first; b < days; b++) {
const bStart = start + b * DAY_MS;
if (bStart >= seg.to) break;
const overlap = Math.min(seg.to, bStart + DAY_MS) - Math.max(seg.from, bStart);
if (overlap <= 0) continue;
known[b] += overlap;
if (seg.down) down[b] += overlap;
}
}
return known.map((k, b) => ({ k: Math.round(k / 60_000), d: Math.round(down[b] / 60_000) }));
}
/** Share of the known time spent up, to one decimal; null when nothing is known. */
export function uptimePercent(buckets: UptimeBucket[]): number | null {
const known = buckets.reduce((n, b) => n + b.k, 0);
if (known === 0) return null;
const down = buckets.reduce((n, b) => n + b.d, 0);
return Math.round((1 - down / known) * 1000) / 10;
}
function series(transitions: Transition[], knownFrom: number, now: number, days: number): UptimeSeries | null {
const buckets = computeBuckets(transitions, knownFrom, now, days);
const percent = uptimePercent(buckets);
return percent === null ? null : { buckets, percent };
}
/**
* Integrations' failing / working-again moments only exist from the version that began writing them, so nothing earlier can be claimed.
* The first time anything asks, that moment is written down and kept.
*/
let historyStart: number | null = null;
export async function ensureHistoryStart(now: number = Date.now()): Promise<number> {
if (historyStart !== null) return historyStart;
const stored = await getInternalFlag(HISTORY_FLAG);
const parsed = stored ? Date.parse(stored) : NaN;
if (Number.isFinite(parsed)) {
historyStart = parsed;
return parsed;
}
await setInternalFlag(HISTORY_FLAG, new Date(now).toISOString());
historyStart = now;
return now;
}
/** For tests: forget the remembered start. */
export function resetHistoryStartForTests(): void {
historyStart = null;
}
export interface UptimeReport {
days: number;
/** ISO — the end of the last bucket. */
endsAt: string;
servers: Record<number, UptimeSeries>;
integrations: Record<number, UptimeSeries>;
}
export async function getUptime(now: Date = new Date(), days: number = DEFAULT_DAYS): Promise<UptimeReport> {
const nowMs = now.getTime();
const rows = await db.select().from(auditLog).where(inArray(auditLog.action, ["went_offline", "back_online", "went_failing", "recovered"]));
const serverEvents = new Map<number, Transition[]>();
const sourceEvents = new Map<string, Transition[]>();
for (const r of rows) {
const at = Date.parse(auditTimeToIso(r.createdAt));
if (!Number.isFinite(at) || at > nowMs) continue;
if (r.category === "server" && r.targetType === "server" && (r.action === "went_offline" || r.action === "back_online")) {
const id = Number(r.targetId);
const list = serverEvents.get(id) ?? [];
list.push({ at, down: r.action === "went_offline" });
serverEvents.set(id, list);
} else if (r.category === "integration" && r.targetType === "integration_type" && r.targetId && (r.action === "went_failing" || r.action === "recovered")) {
const list = sourceEvents.get(r.targetId) ?? [];
list.push({ at, down: r.action === "went_failing" });
sourceEvents.set(r.targetId, list);
}
}
const out: UptimeReport = { days, endsAt: now.toISOString(), servers: {}, integrations: {} };
for (const s of await db.select({ id: servers.id, createdAt: servers.createdAt, lastSeenAt: servers.lastSeenAt }).from(servers)) {
const events = serverEvents.get(s.id) ?? [];
// A server that has never reported and never been seen to stop has no history to speak of.
if (!s.lastSeenAt && events.length === 0) continue;
const created = Date.parse(auditTimeToIso(s.createdAt));
const result = series(events, Number.isFinite(created) ? created : nowMs, nowMs, days);
if (result) out.servers[s.id] = result;
}
const since = await ensureHistoryStart(nowMs);
for (const i of await db.select({ id: integrations.id, type: integrations.type, createdAt: integrations.createdAt }).from(integrations)) {
const created = Date.parse(auditTimeToIso(i.createdAt));
const knownFrom = Math.max(Number.isFinite(created) ? created : 0, since);
const result = series(sourceEvents.get(i.type) ?? [], knownFrom, nowMs, days);
if (result) out.integrations[i.id] = result;
}
return out;
}