Dependency map, impact preview in maintenance, and a calmer Alerts page
- Map: a Dependencies view showing what would go down with a picked box and what it relies on - Maintenance: lists what depends on a target and can silence those too (same time, audited); the recurring form and update drawer show the same note - Alerts: a headline on where things stand, held-back alerts nested under their cause, severity edge on rows; caused-by now carries the explaining alert's id Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ec9c2f73f0
commit
9caa414829
19 files changed
+944
-40
No files matched your search
@@ -2,11 +2,14 @@ import { Router } from "express";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { z } from "zod";
|
||||
import { db } from "../db/client.js";
|
||||
import { dependencies } from "../db/schema.js";
|
||||
import { dependencies, servers } from "../db/schema.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { loadGraph, nodeKey, nodeTypes, wouldLoop } from "../services/dependencies.js";
|
||||
import { dependentsOf, impactOf, loadGraph, nodeKey, nodeTypes, wouldLoop } from "../services/dependencies.js";
|
||||
import { getSettings } from "../services/settingsStore.js";
|
||||
import { getSourceStatus } from "../services/integrationHealthMonitor.js";
|
||||
import { serverStatus, type MapStatus } from "../services/topology.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const dependenciesRouter = Router();
|
||||
@@ -41,6 +44,56 @@ dependenciesRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
});
|
||||
}));
|
||||
|
||||
// What a thing would take with it, and what it needs itself: for the maintenance and update pages, which say so before it is done.
|
||||
dependenciesRouter.get("/impact", asyncHandler(async (req, res) => {
|
||||
const type = nodeTypes.find((t) => t === req.query.type);
|
||||
const id = typeof req.query.id === "string" && /^\d+$/.test(req.query.id) ? Number(req.query.id) : NaN;
|
||||
if (!type || !Number.isInteger(id)) return res.status(400).json({ error: "invalid_query", message: "Say which server or integration." });
|
||||
const impact = await impactOf(type, id);
|
||||
if (!impact) return res.status(404).json({ error: "not_found" });
|
||||
res.json(impact);
|
||||
}));
|
||||
|
||||
// The graph for the Map page: the servers and integrations that take part in any dependency, how each is doing, and who needs whom. An edge runs
|
||||
// from the thing that is needed to the thing that needs it, so the foundations come first.
|
||||
dependenciesRouter.get("/graph", asyncHandler(async (_req, res) => {
|
||||
const graph = await loadGraph();
|
||||
const written = new Set(graph.rows.map((r) => `${nodeKey(r.fromType, r.fromId)}>${nodeKey(r.toType, r.toId)}`));
|
||||
const edges: { from: string; to: string; kind: "written" | "runs-on" }[] = [];
|
||||
const taking = new Set<string>();
|
||||
for (const [needing, needed] of graph.edges) {
|
||||
for (const n of needed) {
|
||||
edges.push({ from: n, to: needing, kind: written.has(`${needing}>${n}`) ? "written" : "runs-on" });
|
||||
taking.add(needing);
|
||||
taking.add(n);
|
||||
}
|
||||
}
|
||||
const { healthChecks } = await getSettings();
|
||||
const now = new Date();
|
||||
const serverRows = await db.select({ id: servers.id, lastSeenAt: servers.lastSeenAt, thresholds: servers.thresholds }).from(servers);
|
||||
const serverById = new Map(serverRows.map((s) => [s.id, s]));
|
||||
const intById = new Map(graph.intRows.map((i) => [i.id, i]));
|
||||
const nodes = [...taking]
|
||||
.map((key) => {
|
||||
const [type, rawId] = key.split(":");
|
||||
const id = Number(rawId);
|
||||
let status: MapStatus = "unknown";
|
||||
if (type === "server") {
|
||||
const s = serverById.get(id);
|
||||
if (s) status = serverStatus(s, healthChecks.serverOfflineMinutes, now);
|
||||
} else {
|
||||
const i = intById.get(id);
|
||||
if (i?.enabled) {
|
||||
const st = getSourceStatus(i.type).status;
|
||||
status = st === "failing" ? "bad" : st === "ok" ? "ok" : "unknown";
|
||||
}
|
||||
}
|
||||
return { key, type: type as "server" | "integration", id, name: graph.names.get(key) ?? key, status, href: type === "server" ? `/servers/${id}` : "/integrations" };
|
||||
})
|
||||
.sort((a, b) => a.name.localeCompare(b.name));
|
||||
res.json({ nodes, edges });
|
||||
}));
|
||||
|
||||
dependenciesRouter.post("/", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const parsed = edgeSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Say what needs what: a server or an integration each.", details: parsed.error.flatten() });
|
||||
|
||||
@@ -9,6 +9,7 @@ import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { describeTarget, listActiveWindows, listUpcomingWindows, startOrExtendWindow } from "../services/maintenance.js";
|
||||
import { impactOf } from "../services/dependencies.js";
|
||||
import { KumaSyncError, syncKumaMaintenance } from "../services/kumaMaintenanceSync.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createUptimeKumaAdapter } from "../integrations/uptimekuma/adapter.js";
|
||||
@@ -56,6 +57,8 @@ const startSchema = z.object({
|
||||
// Required and capped: an open-ended window is the way this feature could quietly hide a real outage.
|
||||
minutes: z.number().int().min(5).max(7 * 24 * 60),
|
||||
reason: z.string().max(200).optional(),
|
||||
// Also put what depends on the target under maintenance, for the same time: it will look down while the target is.
|
||||
withDependents: z.boolean().optional(),
|
||||
});
|
||||
|
||||
maintenanceRouter.post("/", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
@@ -63,7 +66,7 @@ maintenanceRouter.post("/", requireRole("operator"), asyncHandler(async (req, re
|
||||
if (!parsed.success) {
|
||||
return res.status(400).json({ error: "invalid_body", details: parsed.error.flatten() });
|
||||
}
|
||||
const { targetType, targetId, minutes, reason } = parsed.data;
|
||||
const { targetType, targetId, minutes, reason, withDependents } = parsed.data;
|
||||
|
||||
const actor = req.currentUser!;
|
||||
const createdBy = actor.name ?? actor.email ?? actor.oidcSub;
|
||||
@@ -73,6 +76,19 @@ maintenanceRouter.post("/", requireRole("operator"), asyncHandler(async (req, re
|
||||
return res.status(404).json({ error: "not_found", message: "That target doesn't exist." });
|
||||
}
|
||||
|
||||
// What depends on it goes down with it, so it is silenced for the same time (and says why).
|
||||
const alsoSilenced: { type: "server" | "integration"; id: number; name: string }[] = [];
|
||||
if (withDependents && (targetType === "server" || targetType === "integration")) {
|
||||
const impact = await impactOf(targetType, targetId);
|
||||
for (const d of impact?.dependents ?? []) {
|
||||
const also = await startOrExtendWindow(d.type, d.id, minutes, `Depends on ${result.target.name}${reason ? ` — ${reason}` : ""}`.slice(0, 200), createdBy);
|
||||
if (also) {
|
||||
alsoSilenced.push({ type: d.type, id: d.id, name: also.target.name });
|
||||
await recordAudit({ actor, category: "maintenance", action: also.extended ? "extend" : "start", targetType: d.type, targetId: d.id, detail: { name: also.target.name, minutes, reason: `depends on ${result.target.name}`, dependsOn: result.target.name } });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await recordAudit({
|
||||
actor,
|
||||
category: "maintenance",
|
||||
@@ -83,7 +99,7 @@ maintenanceRouter.post("/", requireRole("operator"), asyncHandler(async (req, re
|
||||
});
|
||||
|
||||
invalidateAlertsCache();
|
||||
res.status(201).json({ window: { ...result.window, targetName: result.target.name, targetKind: result.target.kind } });
|
||||
res.status(201).json({ window: { ...result.window, targetName: result.target.name, targetKind: result.target.kind }, alsoSilenced });
|
||||
}));
|
||||
|
||||
const importUptimeKumaSchema = z.object({
|
||||
|
||||
@@ -19,7 +19,8 @@ export interface Alert {
|
||||
/** Under an active maintenance window: still a real problem, but notifications for it are held back. */
|
||||
silenced: boolean;
|
||||
/** Set when this problem is explained by another one (a guest down because its host is): it is held back like a silenced one, but the reason is this, not maintenance. */
|
||||
causedBy?: { name: string };
|
||||
/** What explains it: the name of the thing that is down, and the id of the alert about that when there is one. */
|
||||
causedBy?: { name: string; alertId?: string };
|
||||
/** On a host's alert: the servers on it whose own alerts are folded under this one. */
|
||||
affects?: string[];
|
||||
/** What a maintenance window would have to be on to silence it, when there is such a thing — which is what "Silence" acts on. */
|
||||
|
||||
@@ -33,7 +33,7 @@ import { computeSecretStatus } from "./secretStatus.js";
|
||||
import { getFailingSources } from "./integrationHealthMonitor.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { sourceLabel } from "./notify.js";
|
||||
import { dependencyCover, integrationTypeCause, takenByType } from "./dependencies.js";
|
||||
import { alertIdOfNode, dependencyCover, integrationTypeCause, takenByType } from "./dependencies.js";
|
||||
import type { Alert, AlertCategory, AlertSeverity, SourceFailure } from "./alertTypes.js";
|
||||
|
||||
export interface AlertsReport {
|
||||
@@ -189,7 +189,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
const offlineId = /^offline:server:(\d+)$/.exec(c.key)?.[1];
|
||||
const parentOf = offlineId ? cover.cause.get(`server:${offlineId}`) : undefined;
|
||||
push({
|
||||
causedBy: parentOf !== undefined ? { name: cover.graph.names.get(parentOf) ?? parentOf } : undefined,
|
||||
causedBy: parentOf !== undefined ? { name: cover.graph.names.get(parentOf) ?? parentOf, alertId: alertIdOfNode(parentOf, cover.graph) ?? undefined } : undefined,
|
||||
affects: offlineId ? cover.taken.get(`server:${offlineId}`)?.sort((a, b) => a.localeCompare(b)) : undefined,
|
||||
category: origin.category,
|
||||
severity: c.severity ?? "warning",
|
||||
@@ -458,7 +458,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
const cause = cover ? integrationTypeCause(f.source, cover) : null;
|
||||
const affects = cover ? takenByType(f.source, cover) : [];
|
||||
push({
|
||||
causedBy: cause ? { name: cause.name } : undefined,
|
||||
causedBy: cause ? { name: cause.name, alertId: (cover && alertIdOfNode(cause.key, cover.graph)) || undefined } : undefined,
|
||||
affects: affects.length > 0 ? affects.sort((a, b) => a.localeCompare(b)) : undefined,
|
||||
category: "integration",
|
||||
severity: f.alerted ? "critical" : "warning",
|
||||
|
||||
@@ -12,6 +12,7 @@ import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { dependencies, integrations, servers } from "../db/schema.js";
|
||||
import { getInternalFlag } from "./settingsStore.js";
|
||||
import { listActiveWindows } from "./maintenance.js";
|
||||
import { loadParentRows, resolveParents, type ParentLink } from "./serverParents.js";
|
||||
|
||||
/** Where the health monitor keeps the problems it is announcing; read here so a service's alert can tell a server it needs is down. */
|
||||
@@ -89,6 +90,33 @@ export function wouldLoop(edges: Edges, from: string, to: string): boolean {
|
||||
return from === to || ancestorsOf(to, edges).order.includes(from) || ancestorsOf(to, edges).looped;
|
||||
}
|
||||
|
||||
/** Everything that needs `key`, directly or through others, nearest first. */
|
||||
export function dependentsOf(edges: Edges, key: string): { key: string; distance: number }[] {
|
||||
const reverse = new Map<string, string[]>();
|
||||
for (const [from, tos] of edges) for (const to of tos) reverse.set(to, [...(reverse.get(to) ?? []), from]);
|
||||
const out: { key: string; distance: number }[] = [];
|
||||
const seen = new Set<string>([key]);
|
||||
let layer = [key];
|
||||
for (let distance = 1; layer.length > 0; distance++) {
|
||||
const next: string[] = [];
|
||||
for (const k of layer) {
|
||||
for (const d of (reverse.get(k) ?? []).sort()) {
|
||||
if (seen.has(d)) continue;
|
||||
seen.add(d);
|
||||
out.push({ key: d, distance });
|
||||
next.push(d);
|
||||
}
|
||||
}
|
||||
layer = next;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Everything `key` needs, directly or through others, nearest first. */
|
||||
export function needsOfNode(edges: Edges, key: string): string[] {
|
||||
return ancestorsOf(key, edges).order;
|
||||
}
|
||||
|
||||
export interface Graph {
|
||||
edges: Edges;
|
||||
/** node → its name, for every node that exists. */
|
||||
@@ -115,6 +143,45 @@ export async function loadGraph(): Promise<Graph> {
|
||||
return { edges: buildEdges(resolveParents(parentRows), rows, (k) => names.has(k)), names, rows, instances, intRows };
|
||||
}
|
||||
|
||||
/** The id of the alert about a node being down, when it has one (a server's "offline", a kind of service's "failing"). */
|
||||
export function alertIdOfNode(key: string, graph: Graph): string | null {
|
||||
const m = /^(server|integration):(\d+)$/.exec(key);
|
||||
if (!m) return null;
|
||||
if (m[1] === "server") return `offline:offline:server:${m[2]}`;
|
||||
const row = graph.intRows.find((r) => r.id === Number(m[2]));
|
||||
return row ? `integration:failing:${row.type}` : null;
|
||||
}
|
||||
|
||||
export interface ImpactItem {
|
||||
key: string;
|
||||
type: NodeType;
|
||||
id: number;
|
||||
name: string;
|
||||
/** 1 = needs it directly, 2 = through one other, … */
|
||||
distance: number;
|
||||
/** Already under a maintenance window. */
|
||||
silenced: boolean;
|
||||
}
|
||||
|
||||
/** What would be affected if `type`/`id` went down (everything that needs it) and what it needs itself. Null if it doesn't exist. */
|
||||
export async function impactOf(type: NodeType, id: number, now: Date = new Date()): Promise<{ target: { key: string; type: NodeType; id: number; name: string }; dependents: ImpactItem[]; needs: ImpactItem[] } | null> {
|
||||
const graph = await loadGraph();
|
||||
const key = nodeKey(type, id);
|
||||
const name = graph.names.get(key);
|
||||
if (name === undefined) return null;
|
||||
const windows = await listActiveWindows(now);
|
||||
const silenced = new Set(windows.map((w) => `${w.targetType}:${w.targetId}`));
|
||||
const item = (k: string, distance: number): ImpactItem => {
|
||||
const [t, i] = k.split(":");
|
||||
return { key: k, type: t as NodeType, id: Number(i), name: graph.names.get(k) ?? k, distance, silenced: silenced.has(k) };
|
||||
};
|
||||
return {
|
||||
target: { key, type, id, name },
|
||||
dependents: dependentsOf(graph.edges, key).map((d) => item(d.key, d.distance)),
|
||||
needs: needsOfNode(graph.edges, key).map((k, i) => item(k, i + 1)),
|
||||
};
|
||||
}
|
||||
|
||||
export interface Cover {
|
||||
/** Each node that is down because of another → the root cause. */
|
||||
cause: Map<string, string>;
|
||||
|
||||
@@ -59,6 +59,13 @@ function parseIps(stored: string | null): string[] {
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether a server is reporting: "ok", "bad" once it has been silent longer than its limit, "unknown" if it has never reported. */
|
||||
export function serverStatus(row: { lastSeenAt: string | null; thresholds: string | null }, offlineMinutes: number, now: Date): MapStatus {
|
||||
const lastSeen = row.lastSeenAt ? Date.parse(row.lastSeenAt) : null;
|
||||
const offlineAfterMs = (parseThresholds(row.thresholds)?.serverOfflineMinutes ?? offlineMinutes) * 60_000;
|
||||
return lastSeen === null ? "unknown" : now.getTime() - lastSeen > offlineAfterMs ? "bad" : "ok";
|
||||
}
|
||||
|
||||
export async function buildTopology(now: Date = new Date()): Promise<MapData> {
|
||||
const nodes: MapNode[] = [];
|
||||
const edges: MapEdge[] = [];
|
||||
@@ -72,9 +79,7 @@ export async function buildTopology(now: Date = new Date()): Promise<MapData> {
|
||||
// Servers, and the Proxmox hosts they run on.
|
||||
const hostIds = new Set<string>();
|
||||
for (const s of serverRows) {
|
||||
const lastSeen = s.lastSeenAt ? Date.parse(s.lastSeenAt) : null;
|
||||
const offlineAfterMs = (parseThresholds(s.thresholds)?.serverOfflineMinutes ?? healthChecks.serverOfflineMinutes) * 60_000;
|
||||
const status: MapStatus = lastSeen === null ? "unknown" : now.getTime() - lastSeen > offlineAfterMs ? "bad" : "ok";
|
||||
const status = serverStatus(s, healthChecks.serverOfflineMinutes, now);
|
||||
nodes.push({ id: `server:${s.id}`, kind: "server", label: s.name, sub: s.hostname, status, href: `/servers/${s.id}` });
|
||||
if (s.proxmoxIntegrationId && s.proxmoxNode) {
|
||||
const hostId = `host:${s.proxmoxIntegrationId}:${s.proxmoxNode}`;
|
||||
|
||||
@@ -164,7 +164,7 @@ describe("through the routes, the alerts and the notifications", () => {
|
||||
const failing = report.alerts.filter((a) => a.category === "integration");
|
||||
assert.equal(failing.length, 2);
|
||||
for (const f of failing) {
|
||||
assert.deepEqual(f.causedBy, { name: "box" });
|
||||
assert.deepEqual(f.causedBy, { name: "box", alertId: report.alerts.find((a) => a.category === "offline" && a.message.startsWith("box"))!.id });
|
||||
assert.equal(f.silenced, true);
|
||||
}
|
||||
assert.equal(failing.find((f) => f.id.includes("gitea"))!.affects, undefined, "only the root says what it takes with it");
|
||||
@@ -178,7 +178,7 @@ describe("through the routes, the alerts and the notifications", () => {
|
||||
const report = await getAlerts(true);
|
||||
const failing = report.alerts.filter((a) => a.category === "integration");
|
||||
assert.equal(failing.find((f) => f.id.includes("gitea"))!.causedBy, undefined);
|
||||
assert.deepEqual(failing.find((f) => f.id.includes("semaphore"))!.causedBy, { name: "Gitea" });
|
||||
assert.deepEqual(failing.find((f) => f.id.includes("semaphore"))!.causedBy, { name: "Gitea", alertId: "integration:failing:gitea" });
|
||||
});
|
||||
|
||||
test("a kind of service is folded only if every enabled instance of it is explained", async () => {
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
/**
|
||||
* What a thing would take with it: the impact lookup, the graph for the map, silencing dependents along with the target, and an alert
|
||||
* pointing at the alert that explains it.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-impact-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, integrations, auditLog } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { buildEdges, dependentsOf, needsOfNode } = await import("../src/services/dependencies.js");
|
||||
const { dependenciesRouter } = await import("../src/routes/dependencies.js");
|
||||
const { maintenanceRouter } = await import("../src/routes/maintenance.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { trackIntegrationHealth } = await import("../src/services/integrationHealthMonitor.js");
|
||||
const { listActiveWindows } = await import("../src/services/maintenance.js");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const everything = () => true;
|
||||
const edge = (fromType: "server" | "integration", fromId: number, toType: "server" | "integration", toId: number) => ({ fromType, fromId, toType, toId });
|
||||
|
||||
describe("walking the graph", () => {
|
||||
const edges = buildEdges(new Map(), [edge("integration", 1, "server", 1), edge("integration", 2, "integration", 1), edge("server", 2, "integration", 1), edge("server", 3, "server", 2)], everything);
|
||||
|
||||
test("what needs a thing, nearest first, each once, through others too", () => {
|
||||
assert.deepEqual(dependentsOf(edges, "server:1"), [
|
||||
{ key: "integration:1", distance: 1 },
|
||||
{ key: "integration:2", distance: 2 },
|
||||
{ key: "server:2", distance: 2 },
|
||||
{ key: "server:3", distance: 3 },
|
||||
]);
|
||||
assert.deepEqual(dependentsOf(edges, "server:3"), [], "nothing needs it");
|
||||
});
|
||||
|
||||
test("what a thing needs, nearest first", () => {
|
||||
assert.deepEqual(needsOfNode(edges, "server:3"), ["server:2", "integration:1", "server:1"]);
|
||||
assert.deepEqual(needsOfNode(edges, "server:1"), []);
|
||||
});
|
||||
});
|
||||
|
||||
describe("through the routes", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "operator";
|
||||
let box = 0;
|
||||
let app1 = 0;
|
||||
let gitea = 0;
|
||||
let semaphore = 0;
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "operator", name: "Olga", role: "operator" },
|
||||
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
||||
]);
|
||||
const made = await db
|
||||
.insert(servers)
|
||||
.values([
|
||||
{ name: "box", apiTokenHash: "b", apiTokenPrefix: "b", lastSeenAt: new Date(Date.now() - 3 * 3600_000).toISOString() },
|
||||
{ name: "app", apiTokenHash: "a", apiTokenPrefix: "a", lastSeenAt: new Date().toISOString() },
|
||||
])
|
||||
.returning({ id: servers.id });
|
||||
box = made[0].id;
|
||||
app1 = made[1].id;
|
||||
const ints = await db
|
||||
.insert(integrations)
|
||||
.values([
|
||||
{ type: "gitea", name: "Gitea", baseUrl: "https://git.lab", config: "{}" },
|
||||
{ type: "semaphore", name: "Semaphore", baseUrl: "https://sem.lab", config: "{}" },
|
||||
])
|
||||
.returning({ id: integrations.id });
|
||||
gitea = ints[0].id;
|
||||
semaphore = ints[1].id;
|
||||
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/dependencies", dependenciesRouter);
|
||||
a.use("/api/maintenance", maintenanceRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
|
||||
|
||||
const post = (body: unknown) => fetch(`${base}/dependencies`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) });
|
||||
assert.equal((await post(edge("integration", gitea, "server", box))).status, 201);
|
||||
assert.equal((await post(edge("integration", semaphore, "integration", gitea))).status, 201);
|
||||
assert.equal((await post(edge("server", app1, "integration", gitea))).status, 201);
|
||||
});
|
||||
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const get = (path: string) => fetch(`${base}${path}`);
|
||||
const startMaintenance = (body: unknown) => fetch(`${base}/maintenance`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) });
|
||||
|
||||
test("the impact of the server: everything that needs it, with how far away it is; what Semaphore needs, nearest first", async () => {
|
||||
const impact = await (await get(`/dependencies/impact?type=server&id=${box}`)).json();
|
||||
assert.equal(impact.target.name, "box");
|
||||
assert.deepEqual(impact.dependents.map((d: { name: string; distance: number }) => [d.name, d.distance]).sort(), [["Gitea", 1], ["Semaphore", 2], ["app", 2]]);
|
||||
assert.deepEqual(impact.needs, []);
|
||||
const sem = await (await get(`/dependencies/impact?type=integration&id=${semaphore}`)).json();
|
||||
assert.deepEqual(sem.needs.map((n: { name: string }) => n.name), ["Gitea", "box"]);
|
||||
assert.equal((await get(`/dependencies/impact?type=server&id=99999`)).status, 404);
|
||||
assert.equal((await get(`/dependencies/impact?type=dns&id=1`)).status, 400);
|
||||
assert.equal((await get(`/dependencies/impact?type=server`)).status, 400);
|
||||
});
|
||||
|
||||
test("the graph for the map: only what takes part, the foundations first, with how each is doing", async () => {
|
||||
const graph = await (await get("/dependencies/graph")).json();
|
||||
const byName = Object.fromEntries(graph.nodes.map((n: { name: string; status: string; href: string }) => [n.name, n]));
|
||||
assert.deepEqual(Object.keys(byName).sort(), ["Gitea", "Semaphore", "app", "box"]);
|
||||
assert.equal(byName.box.status, "bad", "silent for 3 hours");
|
||||
assert.equal(byName.app.status, "ok");
|
||||
assert.equal(byName.Gitea.status, "unknown", "nothing has called it yet");
|
||||
assert.equal(byName.box.href, `/servers/${box}`);
|
||||
assert.deepEqual(
|
||||
graph.edges.map((e: { from: string; to: string; kind: string }) => `${e.from}>${e.to}:${e.kind}`).sort(),
|
||||
[`integration:${gitea}>integration:${semaphore}:written`, `integration:${gitea}>server:${app1}:written`, `server:${box}>integration:${gitea}:written`].sort(),
|
||||
);
|
||||
});
|
||||
|
||||
test("an alert points at the alert that explains it", async () => {
|
||||
await trackIntegrationHealth("gitea", false);
|
||||
invalidateAlertsCache();
|
||||
const report = await getAlerts(true);
|
||||
const offline = report.alerts.find((a) => a.category === "offline" && a.message.startsWith("box"))!;
|
||||
const failing = report.alerts.find((a) => a.id === "integration:failing:gitea")!;
|
||||
assert.deepEqual(failing.causedBy, { name: "box", alertId: offline.id });
|
||||
assert.ok(report.alerts.some((a) => a.id === failing.causedBy!.alertId), "and that alert exists");
|
||||
});
|
||||
|
||||
test("silencing a server for maintenance leaves what depends on it alone unless asked", async () => {
|
||||
const res = await startMaintenance({ targetType: "server", targetId: box, minutes: 30, reason: "reboot" });
|
||||
assert.equal(res.status, 201);
|
||||
assert.deepEqual((await res.json()).alsoSilenced, []);
|
||||
assert.equal((await listActiveWindows()).length, 1);
|
||||
});
|
||||
|
||||
test("asked to, it silences everything that depends on it too, for the same time, and says why", async () => {
|
||||
const res = await startMaintenance({ targetType: "server", targetId: box, minutes: 30, reason: "reboot", withDependents: true });
|
||||
assert.equal(res.status, 201);
|
||||
const body = await res.json();
|
||||
assert.deepEqual(body.alsoSilenced.map((s: { name: string }) => s.name).sort(), ["Gitea", "Semaphore", "app"]);
|
||||
const windows = await listActiveWindows();
|
||||
assert.equal(windows.length, 4);
|
||||
const gitea1 = windows.find((w) => w.targetType === "integration" && w.targetId === gitea)!;
|
||||
assert.equal(gitea1.reason, "Depends on box — reboot");
|
||||
assert.equal(Math.round((Date.parse(gitea1.endsAt) - Date.parse(body.window.endsAt)) / 1000), 0, "the same end");
|
||||
const audits = (await db.select().from(auditLog)).filter((e) => e.category === "maintenance" && /depends on box/.test(e.detail ?? ""));
|
||||
assert.equal(audits.length, 3);
|
||||
});
|
||||
|
||||
test("a viewer can look at the impact but not start maintenance", async () => {
|
||||
who = "viewer";
|
||||
assert.equal((await get(`/dependencies/impact?type=server&id=${box}`)).status, 200);
|
||||
assert.equal((await startMaintenance({ targetType: "server", targetId: box, minutes: 30, withDependents: true })).status, 403);
|
||||
who = "operator";
|
||||
});
|
||||
});
|
||||
@@ -165,7 +165,7 @@ describe("the alerts, folded", () => {
|
||||
assert.deepEqual(host.affects, ["db", "web"]);
|
||||
assert.equal(host.silenced, false);
|
||||
const web = offline.find((a) => a.message.startsWith("web"))!;
|
||||
assert.deepEqual(web.causedBy, { name: "pve1" });
|
||||
assert.deepEqual(web.causedBy, { name: "pve1", alertId: host.id });
|
||||
assert.equal(web.silenced, true, "held back like a silenced alert");
|
||||
assert.equal(offline.find((a) => a.message.startsWith("elsewhere"))!.causedBy, undefined);
|
||||
assert.equal(report.counts.critical, 2, "pve1 and the unrelated one");
|
||||
|
||||
Reference in new issue
Block a user