Add runbooks, backup coverage, inventory, roles from groups, Prometheus metrics and event webhooks
- Runbooks: notes per server/integration, shown next to matching alerts - Backup coverage page across Proxmox and PBS, with a digest section - Hardware inventory with warranty, power and running cost - Authentik group to role mapping with preview and safeguards - /metrics (bearer token) and signed outgoing event webhooks - Docs: README, DATABASE, NOTIFICATIONS, ROLES and new API.md Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c36e2092c5
commit
c17e9aa140
55 files changed
+7177
-27
No files matched your search
@@ -0,0 +1,57 @@
|
||||
CREATE TABLE `assets` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`name` text NOT NULL,
|
||||
`kind` text DEFAULT 'other' NOT NULL,
|
||||
`manufacturer` text,
|
||||
`model` text,
|
||||
`serial` text,
|
||||
`purchase_date` text,
|
||||
`purchase_price` real,
|
||||
`warranty_until` text,
|
||||
`location` text,
|
||||
`rack_unit` text,
|
||||
`power_watts` integer,
|
||||
`server_id` integer,
|
||||
`notes` text,
|
||||
`created_at` text NOT NULL,
|
||||
`updated_at` text NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE `event_webhooks` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`label` text NOT NULL,
|
||||
`url` text NOT NULL,
|
||||
`secret` text,
|
||||
`events` text NOT NULL,
|
||||
`enabled` integer DEFAULT true NOT NULL,
|
||||
`created_by` text,
|
||||
`created_at` text DEFAULT (current_timestamp) NOT NULL,
|
||||
`last_attempt_at` text,
|
||||
`last_result` text
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE `metrics_tokens` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`label` text NOT NULL,
|
||||
`token_hash` text NOT NULL,
|
||||
`token_prefix` text NOT NULL,
|
||||
`created_by` text,
|
||||
`created_at` text DEFAULT (current_timestamp) NOT NULL,
|
||||
`last_used_at` text
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX `metrics_tokens_token_hash_unique` ON `metrics_tokens` (`token_hash`);--> statement-breakpoint
|
||||
CREATE TABLE `runbooks` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`target_type` text NOT NULL,
|
||||
`target_id` integer NOT NULL,
|
||||
`title` text NOT NULL,
|
||||
`body` text NOT NULL,
|
||||
`category` text,
|
||||
`updated_by` text,
|
||||
`created_at` text NOT NULL,
|
||||
`updated_at` text NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX `runbooks_target_idx` ON `runbooks` (`target_type`,`target_id`);--> statement-breakpoint
|
||||
ALTER TABLE `users` ADD `groups` text;
|
||||
File diff suppressed because it is too large.
Load diff
@@ -190,6 +190,13 @@
|
||||
"when": 1792258587758,
|
||||
"tag": "0026_reflective_northstar",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 27,
|
||||
"version": "6",
|
||||
"when": 1792260718106,
|
||||
"tag": "0027_bizarre_magus",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -10,6 +10,11 @@ import { authRouter } from "./auth/router.js";
|
||||
import { meRouter } from "./routes/me.js";
|
||||
import { uptimeRouter } from "./routes/uptime.js";
|
||||
import { shareLinksRouter } from "./routes/shareLinks.js";
|
||||
import { runbooksRouter } from "./routes/runbooks.js";
|
||||
import { inventoryRouter } from "./routes/inventory.js";
|
||||
import { backupsRouter } from "./routes/backups.js";
|
||||
import { metricsRouter, metricsTokensRouter } from "./routes/metrics.js";
|
||||
import { eventWebhooksRouter } from "./routes/eventWebhooks.js";
|
||||
import { publicWallRouter } from "./routes/publicWall.js";
|
||||
import { usersRouter } from "./routes/users.js";
|
||||
import { auditLogRouter } from "./routes/auditLog.js";
|
||||
@@ -92,6 +97,13 @@ export function createApp() {
|
||||
app.use("/api/me", meRouter);
|
||||
app.use("/api/uptime", uptimeRouter);
|
||||
app.use("/api/share-links", shareLinksRouter);
|
||||
app.use("/api/runbooks", runbooksRouter);
|
||||
app.use("/api/inventory", inventoryRouter);
|
||||
app.use("/api/backups", backupsRouter);
|
||||
app.use("/api/metrics-tokens", metricsTokensRouter);
|
||||
app.use("/api/event-webhooks", eventWebhooksRouter);
|
||||
// Prometheus scrapes this with a bearer token, not a session.
|
||||
app.use("/metrics", metricsRouter);
|
||||
app.use("/api/public/wall", publicWallRouter);
|
||||
app.use("/api/users", usersRouter);
|
||||
app.use("/api/audit-log", auditLogRouter);
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
/**
|
||||
* Roles from the groups the identity provider says a person is in. An administrator names the groups that mean "admin" and "operator";
|
||||
* at each sign-in the person's role follows from them. Kept free of the database and the sign-in flow so every rule can be tested exactly.
|
||||
*/
|
||||
import type { UserRole } from "../db/schema.js";
|
||||
|
||||
export interface SignInRoleSettings {
|
||||
/** Roles follow the groups. Off, roles are set by hand on the Users page, as they always were. */
|
||||
roleFromGroups: boolean;
|
||||
/** The name of the claim that carries the groups ("groups" in Authentik). */
|
||||
groupsClaim: string;
|
||||
adminGroups: string[];
|
||||
operatorGroups: string[];
|
||||
/** What someone in none of the named groups gets: viewer, or whatever they already have. */
|
||||
otherwise: "viewer" | "keep";
|
||||
}
|
||||
|
||||
const MAX_GROUPS = 200;
|
||||
const MAX_GROUP_LENGTH = 200;
|
||||
|
||||
/**
|
||||
* The groups named in a claim, from every place the provider might have put it (the ID token, the user-info answer). Returns null when
|
||||
* none of them has the claim at all — which is "we weren't told", not "in no groups", and must never be mistaken for it.
|
||||
*/
|
||||
export function extractGroups(sources: Record<string, unknown>[], claim: string): string[] | null {
|
||||
let found = false;
|
||||
const groups = new Set<string>();
|
||||
for (const source of sources) {
|
||||
if (!source || !Object.prototype.hasOwnProperty.call(source, claim)) continue;
|
||||
const value = source[claim];
|
||||
if (value === null || value === undefined) continue;
|
||||
found = true;
|
||||
const list = Array.isArray(value) ? value : typeof value === "string" ? [value] : [];
|
||||
for (const g of list) {
|
||||
if (typeof g !== "string") continue;
|
||||
const name = g.trim().slice(0, MAX_GROUP_LENGTH);
|
||||
if (name) groups.add(name);
|
||||
}
|
||||
}
|
||||
return found ? [...groups].slice(0, MAX_GROUPS) : null;
|
||||
}
|
||||
|
||||
const norm = (s: string) => s.trim().toLowerCase();
|
||||
|
||||
/** The role these groups give: admin if any is an admin group, else operator if any is an operator group, else what `otherwise` says. */
|
||||
export function roleFromGroups(settings: SignInRoleSettings, groups: string[], current: UserRole): UserRole {
|
||||
const have = new Set(groups.map(norm));
|
||||
if (settings.adminGroups.some((g) => have.has(norm(g)))) return "admin";
|
||||
if (settings.operatorGroups.some((g) => have.has(norm(g)))) return "operator";
|
||||
return settings.otherwise === "keep" ? current : "viewer";
|
||||
}
|
||||
@@ -3,6 +3,8 @@ import * as client from "openid-client";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { getOidcConfig } from "./oidc.js";
|
||||
import { upsertUserFromLogin } from "./users.js";
|
||||
import { extractGroups } from "./groups.js";
|
||||
import { getSettings } from "../services/settingsStore.js";
|
||||
import { env } from "../env.js";
|
||||
import { db } from "../db/client.js";
|
||||
import { users } from "../db/schema.js";
|
||||
@@ -110,15 +112,35 @@ authRouter.get("/callback", signInLimiter, async (req, res, next) => {
|
||||
|
||||
let email: string | undefined = typeof claims.email === "string" ? claims.email : undefined;
|
||||
let name: string | undefined = typeof claims.name === "string" ? claims.name : undefined;
|
||||
const claimSources: Record<string, unknown>[] = [claims as Record<string, unknown>];
|
||||
try {
|
||||
const userinfo = await client.fetchUserInfo(config, tokens.access_token, claims.sub);
|
||||
email = userinfo.email ?? email;
|
||||
name = userinfo.name ?? userinfo.preferred_username ?? name;
|
||||
claimSources.push(userinfo as Record<string, unknown>);
|
||||
} catch {
|
||||
// fall back to ID token claims already captured above
|
||||
}
|
||||
|
||||
const { user, created } = await upsertUserFromLogin({ sub: claims.sub, email, name });
|
||||
// The groups, from wherever the provider put them. Read (and kept for the Users page) whether or not roles follow them, so the
|
||||
// names can be seen before they are relied on.
|
||||
const { signIn } = await getSettings();
|
||||
const groups = extractGroups(claimSources, signIn.groupsClaim);
|
||||
if (signIn.roleFromGroups && groups === null) {
|
||||
console.warn(`[auth] roles follow groups, but the sign-in carried no "${signIn.groupsClaim}" claim — roles left as they are`);
|
||||
}
|
||||
|
||||
const { user, created, roleChange } = await upsertUserFromLogin({ sub: claims.sub, email, name, groups });
|
||||
if (roleChange) {
|
||||
await recordAudit({
|
||||
actor: user,
|
||||
category: "user",
|
||||
action: roleChange.blocked ? "role_from_groups_blocked" : "role_from_groups",
|
||||
targetType: "user",
|
||||
targetId: user.id,
|
||||
detail: { from: roleChange.from, to: roleChange.to, groups: roleChange.groups, ...(roleChange.blocked ? { reason: "the only administrator" } : {}) },
|
||||
});
|
||||
}
|
||||
|
||||
// Access to the app is itself something worth being able to look back on: who got an account (and the very first
|
||||
// one becomes admin), and every sign-in with where it came from.
|
||||
|
||||
@@ -1,35 +1,72 @@
|
||||
import { eq } from "drizzle-orm";
|
||||
import { and, eq, ne } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { users } from "../db/schema.js";
|
||||
import { users, type UserRole } from "../db/schema.js";
|
||||
import { getSettings } from "../services/settingsStore.js";
|
||||
import { roleFromGroups } from "./groups.js";
|
||||
|
||||
export interface RoleChange {
|
||||
from: UserRole;
|
||||
to: UserRole;
|
||||
groups: string[];
|
||||
/** The group rules would have taken the last administrator's rights away, so the role was left alone. */
|
||||
blocked?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Called on every successful OIDC login. The very first user ever to sign in
|
||||
* becomes admin; everyone after defaults to viewer until an admin promotes
|
||||
* them from the Users page.
|
||||
* them from the Users page — or, when roles follow groups (Settings → Sign-in),
|
||||
* until the groups the identity provider reports decide it.
|
||||
*
|
||||
* `groups` is what the provider said, or null/undefined when it said nothing about groups at all. That is not "in no groups": with no
|
||||
* information the role is left exactly as it is, so a wrong claim name can't quietly demote everyone. And the last administrator is
|
||||
* never demoted by a group rule, whatever the groups say.
|
||||
*/
|
||||
export async function upsertUserFromLogin(params: {
|
||||
sub: string;
|
||||
email?: string;
|
||||
name?: string;
|
||||
}): Promise<{ user: typeof users.$inferSelect; created: boolean }> {
|
||||
groups?: string[] | null;
|
||||
}): Promise<{ user: typeof users.$inferSelect; created: boolean; roleChange?: RoleChange }> {
|
||||
const [existing] = await db.select().from(users).where(eq(users.oidcSub, params.sub)).limit(1);
|
||||
const now = new Date().toISOString();
|
||||
const { signIn } = await getSettings();
|
||||
const mapping = signIn.roleFromGroups && params.groups !== null && params.groups !== undefined;
|
||||
const groupsJson = params.groups ? JSON.stringify(params.groups) : undefined;
|
||||
|
||||
if (existing) {
|
||||
let role = existing.role;
|
||||
let roleChange: RoleChange | undefined;
|
||||
if (mapping) {
|
||||
const wanted = roleFromGroups(signIn, params.groups!, existing.role);
|
||||
if (wanted !== existing.role) {
|
||||
const demotesAdmin = existing.role === "admin" && wanted !== "admin";
|
||||
const otherAdmin = demotesAdmin ? await db.select({ id: users.id }).from(users).where(and(eq(users.role, "admin"), ne(users.id, existing.id))).limit(1) : [];
|
||||
if (demotesAdmin && otherAdmin.length === 0) {
|
||||
roleChange = { from: existing.role, to: wanted, groups: params.groups!, blocked: true };
|
||||
} else {
|
||||
role = wanted;
|
||||
roleChange = { from: existing.role, to: wanted, groups: params.groups! };
|
||||
}
|
||||
}
|
||||
}
|
||||
const [updated] = await db
|
||||
.update(users)
|
||||
.set({
|
||||
email: params.email ?? existing.email,
|
||||
name: params.name ?? existing.name,
|
||||
lastLoginAt: now,
|
||||
role,
|
||||
...(groupsJson !== undefined ? { groups: groupsJson } : {}),
|
||||
})
|
||||
.where(eq(users.id, existing.id))
|
||||
.returning();
|
||||
return { user: updated, created: false };
|
||||
return { user: updated, created: false, roleChange };
|
||||
}
|
||||
|
||||
const anyUser = await db.select({ id: users.id }).from(users).limit(1);
|
||||
const role = anyUser.length === 0 ? ("admin" as const) : ("viewer" as const);
|
||||
// The first person ever is the administrator, so there is always one; after that the groups decide, when they are used, else viewer.
|
||||
const role: UserRole = anyUser.length === 0 ? "admin" : mapping ? roleFromGroups({ ...signIn, otherwise: "viewer" }, params.groups!, "viewer") : "viewer";
|
||||
|
||||
const [created] = await db
|
||||
.insert(users)
|
||||
@@ -39,6 +76,7 @@ export async function upsertUserFromLogin(params: {
|
||||
name: params.name,
|
||||
role,
|
||||
lastLoginAt: now,
|
||||
...(groupsJson !== undefined ? { groups: groupsJson } : {}),
|
||||
})
|
||||
.returning();
|
||||
return { user: created, created: true };
|
||||
|
||||
@@ -20,6 +20,9 @@ export const users = sqliteTable("users", {
|
||||
favorites: text("favorites"),
|
||||
// When this person last pressed "Got it" on the dashboard summary of what changed; the next summary starts from here (ISO).
|
||||
lastLookedAt: text("last_looked_at"),
|
||||
// The groups the identity provider said this person was in at their last sign-in, as a JSON array — shown on the Users page so the group
|
||||
// names can be matched to roles (Settings → Sign-in). Null until they have signed in with groups available.
|
||||
groups: text("groups"),
|
||||
// How this person arranged the dashboard: a JSON object with the cards they hid ("hidden": ["Gitea"]) and the order of the others
|
||||
// ("order": {"DNS": 0, "Secrets": 1}). Follows them to any browser; null is the default layout.
|
||||
dashboardLayout: text("dashboard_layout"),
|
||||
@@ -539,3 +542,85 @@ export const maintenanceSchedules = sqliteTable("maintenance_schedules", {
|
||||
.notNull()
|
||||
.default(sql`(current_timestamp)`),
|
||||
});
|
||||
|
||||
// ─── Runbooks: what to do when something is wrong ───────────────────────────
|
||||
|
||||
// Notes attached to a server, integration or DNS provider — "if this is down, check the UPS first" — shown on its page and next to alerts
|
||||
// about it. A runbook can be for one kind of alert only (`category`, an alert category such as "disk"), or for everything about the thing.
|
||||
export const runbooks = sqliteTable(
|
||||
"runbooks",
|
||||
{
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
targetType: text("target_type").$type<MaintenanceTargetType>().notNull(),
|
||||
targetId: integer("target_id").notNull(), // polymorphic, like maintenance_windows — no FK; one whose target was deleted is simply not shown
|
||||
title: text("title").notNull(),
|
||||
body: text("body").notNull(),
|
||||
category: text("category"), // null = any alert about the target
|
||||
updatedBy: text("updated_by"),
|
||||
createdAt: text("created_at").notNull(), // ISO
|
||||
updatedAt: text("updated_at").notNull(), // ISO
|
||||
},
|
||||
(t) => [index("runbooks_target_idx").on(t.targetType, t.targetId)],
|
||||
);
|
||||
|
||||
// ─── Inventory: the hardware behind it all ──────────────────────────────────
|
||||
|
||||
export const assetKinds = ["server", "storage", "network", "power", "client", "other"] as const;
|
||||
export type AssetKind = (typeof assetKinds)[number];
|
||||
|
||||
// A piece of hardware: what it is, when it was bought, how long it is covered, where it sits and what it draws. Optionally the server it is.
|
||||
export const assets = sqliteTable("assets", {
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
name: text("name").notNull(),
|
||||
kind: text("kind").$type<AssetKind>().notNull().default("other"),
|
||||
manufacturer: text("manufacturer"),
|
||||
model: text("model"),
|
||||
serial: text("serial"),
|
||||
purchaseDate: text("purchase_date"), // YYYY-MM-DD
|
||||
purchasePrice: real("purchase_price"), // in the currency set under Settings → Inventory
|
||||
warrantyUntil: text("warranty_until"), // YYYY-MM-DD
|
||||
location: text("location"), // room, rack, shelf
|
||||
rackUnit: text("rack_unit"), // e.g. "U12" or "U12-13"
|
||||
powerWatts: integer("power_watts"), // typical draw, for the running cost
|
||||
serverId: integer("server_id"), // the server this is, if it is one — a plain reference (no FK); one whose server is gone shows as unlinked
|
||||
notes: text("notes"),
|
||||
createdAt: text("created_at").notNull(), // ISO
|
||||
updatedAt: text("updated_at").notNull(), // ISO
|
||||
});
|
||||
|
||||
// ─── Metrics tokens: read-only access to /metrics for Prometheus ────────────
|
||||
|
||||
// Only a hash of the secret is kept (like an agent token or a share link), so it can't be read back out of the database or a backup.
|
||||
export const metricsTokens = sqliteTable("metrics_tokens", {
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
label: text("label").notNull(), // e.g. "Prometheus"
|
||||
tokenHash: text("token_hash").notNull().unique(),
|
||||
tokenPrefix: text("token_prefix").notNull(),
|
||||
createdBy: text("created_by"),
|
||||
createdAt: text("created_at")
|
||||
.notNull()
|
||||
.default(sql`(current_timestamp)`),
|
||||
lastUsedAt: text("last_used_at"), // ISO
|
||||
});
|
||||
|
||||
// ─── Event webhooks: tell another system when something happens ─────────────
|
||||
|
||||
export const webhookEvents = ["alert.opened", "alert.resolved", "maintenance.started", "maintenance.ended"] as const;
|
||||
export type WebhookEvent = (typeof webhookEvents)[number];
|
||||
|
||||
// A URL that gets a signed JSON message for the events it subscribes to — for Home Assistant, n8n, a status board. Separate from the
|
||||
// chat-style webhook under Settings → Notifications, which carries the same text a person would read.
|
||||
export const eventWebhooks = sqliteTable("event_webhooks", {
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
label: text("label").notNull(),
|
||||
url: text("url").notNull(),
|
||||
secret: text("secret"), // encrypted at rest; signs each message (HMAC-SHA256 in X-HLM-Signature)
|
||||
events: text("events").notNull(), // JSON: WebhookEvent[]
|
||||
enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
|
||||
createdBy: text("created_by"),
|
||||
createdAt: text("created_at")
|
||||
.notNull()
|
||||
.default(sql`(current_timestamp)`),
|
||||
lastAttemptAt: text("last_attempt_at"), // ISO
|
||||
lastResult: text("last_result"), // "200", or what went wrong
|
||||
});
|
||||
@@ -11,6 +11,7 @@ import { initProxmoxBackupScheduler } from "./services/proxmoxBackupScheduler.js
|
||||
import { initPbsVerificationScheduler } from "./services/pbsVerificationScheduler.js";
|
||||
import { initQuietHoursScheduler } from "./services/quietHoursScheduler.js";
|
||||
import { initWeeklyDigestScheduler } from "./services/weeklyDigestScheduler.js";
|
||||
import { initMaintenanceEvents } from "./services/maintenanceEvents.js";
|
||||
import { initHealthScheduler } from "./services/healthScheduler.js";
|
||||
import { initAlertTicketScheduler } from "./services/alertTicketScheduler.js";
|
||||
import { initKumaMaintenanceScheduler } from "./services/kumaMaintenanceScheduler.js";
|
||||
@@ -42,6 +43,7 @@ await initProxmoxBackupScheduler();
|
||||
await initPbsVerificationScheduler();
|
||||
await initQuietHoursScheduler();
|
||||
await initWeeklyDigestScheduler();
|
||||
initMaintenanceEvents();
|
||||
await initHealthScheduler();
|
||||
await initAlertTicketScheduler();
|
||||
await initKumaMaintenanceScheduler();
|
||||
|
||||
@@ -43,6 +43,16 @@ export interface PbsFailedSnapshot {
|
||||
backupTime: number;
|
||||
}
|
||||
|
||||
/** The newest snapshot of one guest (or host) in a datastore — what "when was this last backed up" is answered from. */
|
||||
export interface PbsBackupGroup {
|
||||
backupType: string; // "vm" | "ct" | "host"
|
||||
backupId: string; // the guest's vmid, for vm and ct
|
||||
/** Unix seconds. */
|
||||
backupTime: number;
|
||||
/** What the last verify job found for this snapshot; null when it has not been checked. */
|
||||
verification: "ok" | "failed" | null;
|
||||
}
|
||||
|
||||
export interface PbsDatastore {
|
||||
name: string;
|
||||
comment: string | null;
|
||||
@@ -60,6 +70,8 @@ export interface PbsDatastore {
|
||||
latestSnapshotAt: number | null;
|
||||
/** Up to 20 of the most recent verification failures, newest first. */
|
||||
recentFailures: PbsFailedSnapshot[];
|
||||
/** The newest snapshot of every guest in the datastore. */
|
||||
groups: PbsBackupGroup[];
|
||||
}
|
||||
|
||||
export interface PbsNodeStatus {
|
||||
@@ -158,12 +170,20 @@ export function createPbsAdapter(config: PbsConfig): PbsAdapter {
|
||||
let unverifiedCount = 0;
|
||||
let latestSnapshotAt: number | null = null;
|
||||
const failures: PbsFailedSnapshot[] = [];
|
||||
const newest = new Map<string, PbsBackupGroup>();
|
||||
|
||||
for (const s of snapshots) {
|
||||
const backupTime = num(s?.["backup-time"]);
|
||||
if (backupTime !== null && (latestSnapshotAt === null || backupTime > latestSnapshotAt)) latestSnapshotAt = backupTime;
|
||||
|
||||
const state = str(s?.verification?.state)?.toLowerCase() ?? null;
|
||||
const groupType = str(s?.["backup-type"]);
|
||||
const groupId = str(s?.["backup-id"]);
|
||||
if (groupType && groupId && backupTime !== null) {
|
||||
const key = `${groupType}/${groupId}`;
|
||||
const seen = newest.get(key);
|
||||
if (!seen || backupTime > seen.backupTime) newest.set(key, { backupType: groupType, backupId: groupId, backupTime, verification: state === "ok" ? "ok" : state === "failed" ? "failed" : null });
|
||||
}
|
||||
if (state === "failed") {
|
||||
failedCount++;
|
||||
const backupType = str(s?.["backup-type"]);
|
||||
@@ -175,7 +195,7 @@ export function createPbsAdapter(config: PbsConfig): PbsAdapter {
|
||||
}
|
||||
|
||||
failures.sort((a, b) => b.backupTime - a.backupTime);
|
||||
return { snapshotCount: snapshots.length, failedCount, unverifiedCount, latestSnapshotAt, recentFailures: failures.slice(0, 20) };
|
||||
return { snapshotCount: snapshots.length, failedCount, unverifiedCount, latestSnapshotAt, recentFailures: failures.slice(0, 20), groups: [...newest.values()] };
|
||||
}
|
||||
|
||||
async function listDatastores(): Promise<PbsDatastore[]> {
|
||||
@@ -206,6 +226,7 @@ export function createPbsAdapter(config: PbsConfig): PbsAdapter {
|
||||
unverifiedCount: 0,
|
||||
latestSnapshotAt: null,
|
||||
recentFailures: [],
|
||||
groups: [],
|
||||
};
|
||||
}
|
||||
}),
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
import { Router } from "express";
|
||||
import { requireAuth } from "../auth/middleware.js";
|
||||
import { DEFAULT_STALE_HOURS, getBackupCoverage } from "../services/backupCoverage.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const backupsRouter = Router();
|
||||
|
||||
// Everyone signed in can see what is and isn't backed up, like the Proxmox and Backup Server pages it is drawn from.
|
||||
backupsRouter.use(requireAuth);
|
||||
|
||||
// `staleDays`: how old the newest backup may be before a guest counts as stale (default 2). `refresh=1` asks Proxmox again now.
|
||||
backupsRouter.get("/coverage", asyncHandler(async (req, res) => {
|
||||
const days = Number(req.query.staleDays);
|
||||
const staleHours = Number.isFinite(days) && days >= 1 && days <= 60 ? Math.round(days * 24) : DEFAULT_STALE_HOURS;
|
||||
res.json(await getBackupCoverage({ staleHours, refresh: req.query.refresh === "1" }));
|
||||
}));
|
||||
@@ -0,0 +1,97 @@
|
||||
import { Router } from "express";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { z } from "zod";
|
||||
import { db } from "../db/client.js";
|
||||
import { eventWebhooks, webhookEvents } from "../db/schema.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { deliver, parseEvents, sealSecret, type EventWebhookRow } from "../services/eventWebhooks.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const eventWebhooksRouter = Router();
|
||||
|
||||
// A URL that is sent a message whenever something happens: an administrator's to set up.
|
||||
eventWebhooksRouter.use(requireAuth, requireRole("admin"));
|
||||
|
||||
const MAX_WEBHOOKS = 20;
|
||||
|
||||
const url = z
|
||||
.string()
|
||||
.trim()
|
||||
.max(500)
|
||||
.refine((v) => {
|
||||
try {
|
||||
const u = new URL(v);
|
||||
return (u.protocol === "http:" || u.protocol === "https:") && !u.username && !u.password;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}, "Use an http:// or https:// address, without a user name or password in it");
|
||||
|
||||
const createSchema = z.object({
|
||||
label: z.string().trim().min(1).max(60),
|
||||
url,
|
||||
secret: z.string().max(200).optional(),
|
||||
events: z.array(z.enum(webhookEvents)).min(1).max(webhookEvents.length),
|
||||
});
|
||||
const patchSchema = z.object({ enabled: z.boolean() });
|
||||
|
||||
/** A webhook as the list shows it: never the secret, only whether there is one. */
|
||||
function present(row: EventWebhookRow) {
|
||||
return { id: row.id, label: row.label, url: row.url, hasSecret: !!row.secret, events: parseEvents(row.events), enabled: row.enabled, createdBy: row.createdBy, createdAt: row.createdAt, lastAttemptAt: row.lastAttemptAt, lastResult: row.lastResult };
|
||||
}
|
||||
|
||||
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
|
||||
|
||||
eventWebhooksRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const rows = await db.select().from(eventWebhooks).orderBy(eventWebhooks.label);
|
||||
res.json({ webhooks: rows.map(present), events: webhookEvents });
|
||||
}));
|
||||
|
||||
eventWebhooksRouter.post("/", asyncHandler(async (req, res) => {
|
||||
const parsed = createSchema.safeParse(req.body);
|
||||
if (!parsed.success) {
|
||||
return res.status(400).json({ error: "invalid_body", message: "Give it a name, an http(s) address and at least one event.", details: parsed.error.flatten() });
|
||||
}
|
||||
if ((await db.select({ id: eventWebhooks.id }).from(eventWebhooks)).length >= MAX_WEBHOOKS) {
|
||||
return res.status(400).json({ error: "too_many", message: `There are already ${MAX_WEBHOOKS} webhooks. Remove one you don't use first.` });
|
||||
}
|
||||
const actor = req.currentUser!;
|
||||
const [row] = await db
|
||||
.insert(eventWebhooks)
|
||||
.values({ label: parsed.data.label, url: parsed.data.url, secret: parsed.data.secret ? sealSecret(parsed.data.secret) : null, events: JSON.stringify([...new Set(parsed.data.events)]), createdBy: actor.name ?? actor.email ?? actor.oidcSub })
|
||||
.returning();
|
||||
// The address can carry a token of its own, so the audit log gets the name and the events, not the address.
|
||||
await recordAudit({ actor, category: "webhook", action: "create", targetType: "event_webhook", targetId: row.id, detail: { label: row.label, events: parseEvents(row.events), signed: !!row.secret } });
|
||||
res.status(201).json({ webhook: present(row) });
|
||||
}));
|
||||
|
||||
eventWebhooksRouter.patch("/:id", asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
const parsed = patchSchema.safeParse(req.body);
|
||||
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body" });
|
||||
const [row] = await db.update(eventWebhooks).set({ enabled: parsed.data.enabled }).where(eq(eventWebhooks.id, id)).returning();
|
||||
if (!row) return res.status(404).json({ error: "not_found" });
|
||||
await recordAudit({ actor: req.currentUser!, category: "webhook", action: parsed.data.enabled ? "enable" : "disable", targetType: "event_webhook", targetId: id, detail: { label: row.label } });
|
||||
res.json({ webhook: present(row) });
|
||||
}));
|
||||
|
||||
eventWebhooksRouter.delete("/:id", asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [row] = await db.select().from(eventWebhooks).where(eq(eventWebhooks.id, id)).limit(1);
|
||||
if (!row) return res.status(404).json({ error: "not_found" });
|
||||
await db.delete(eventWebhooks).where(eq(eventWebhooks.id, id));
|
||||
await recordAudit({ actor: req.currentUser!, category: "webhook", action: "delete", targetType: "event_webhook", targetId: id, detail: { label: row.label } });
|
||||
res.status(204).end();
|
||||
}));
|
||||
|
||||
// Sends one test message, signed like the real ones, and says how it went — whether or not the webhook is switched on or wants the event.
|
||||
eventWebhooksRouter.post("/:id/test", asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [row] = await db.select().from(eventWebhooks).where(eq(eventWebhooks.id, id)).limit(1);
|
||||
if (!row) return res.status(404).json({ error: "not_found" });
|
||||
const outcome = await deliver(row, "test", { message: "This is a test from Homelab Manager." });
|
||||
res.json(outcome);
|
||||
}));
|
||||
@@ -0,0 +1,120 @@
|
||||
import { Router } from "express";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { z } from "zod";
|
||||
import { db } from "../db/client.js";
|
||||
import { assetKinds, assets, servers } from "../db/schema.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { isRealDate } from "../services/inventory.js";
|
||||
import { getSettings, updateSettings } from "../services/settingsStore.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const inventoryRouter = Router();
|
||||
|
||||
inventoryRouter.use(requireAuth);
|
||||
|
||||
const text = (max: number) => z.string().trim().max(max).nullable().optional();
|
||||
const date = z.string().refine(isRealDate, "Use a date like 2026-10-17").nullable().optional();
|
||||
|
||||
const assetSchema = z.object({
|
||||
name: z.string().trim().min(1).max(120),
|
||||
kind: z.enum(assetKinds),
|
||||
manufacturer: text(100),
|
||||
model: text(100),
|
||||
serial: text(100),
|
||||
purchaseDate: date,
|
||||
purchasePrice: z.number().min(0).max(1_000_000_000).nullable().optional(),
|
||||
warrantyUntil: date,
|
||||
location: text(120),
|
||||
rackUnit: text(30),
|
||||
powerWatts: z.number().int().min(0).max(100_000).nullable().optional(),
|
||||
serverId: z.number().int().positive().nullable().optional(),
|
||||
notes: text(2000),
|
||||
});
|
||||
|
||||
/** A blank field is no value, not an empty string. */
|
||||
const clean = (v: string | null | undefined) => (v === undefined || v === null || v.trim() === "" ? null : v.trim());
|
||||
|
||||
function values(d: z.infer<typeof assetSchema>) {
|
||||
return {
|
||||
name: d.name,
|
||||
kind: d.kind,
|
||||
manufacturer: clean(d.manufacturer),
|
||||
model: clean(d.model),
|
||||
serial: clean(d.serial),
|
||||
purchaseDate: clean(d.purchaseDate),
|
||||
purchasePrice: d.purchasePrice ?? null,
|
||||
warrantyUntil: clean(d.warrantyUntil),
|
||||
location: clean(d.location),
|
||||
rackUnit: clean(d.rackUnit),
|
||||
powerWatts: d.powerWatts ?? null,
|
||||
serverId: d.serverId ?? null,
|
||||
notes: clean(d.notes),
|
||||
};
|
||||
}
|
||||
|
||||
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
|
||||
|
||||
// Everything signed-in people may see: the hardware, and the price of power the running cost is worked out with.
|
||||
inventoryRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const [rows, serverRows, settings] = await Promise.all([db.select().from(assets).orderBy(assets.name), db.select({ id: servers.id, name: servers.name }).from(servers), getSettings()]);
|
||||
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
|
||||
res.json({
|
||||
assets: rows.map((a) => ({ ...a, server: a.serverId !== null && serverName.has(a.serverId) ? { id: a.serverId, name: serverName.get(a.serverId)! } : null })),
|
||||
config: settings.inventory,
|
||||
servers: serverRows.sort((a, b) => a.name.localeCompare(b.name)),
|
||||
});
|
||||
}));
|
||||
|
||||
const configSchema = z.object({
|
||||
currency: z.string().trim().max(8),
|
||||
pricePerKwh: z.number().min(0).max(1000).nullable(),
|
||||
});
|
||||
|
||||
// The price of power, for the running cost. An administrator's to set.
|
||||
inventoryRouter.put("/config", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const parsed = configSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "A price per kWh (or none) and a short currency name.", details: parsed.error.flatten() });
|
||||
const before = (await getSettings()).inventory;
|
||||
const updated = await updateSettings({ inventory: parsed.data });
|
||||
await recordAudit({ actor: req.currentUser!, category: "inventory", action: "update_config", targetType: "settings", detail: { from: before, to: parsed.data } });
|
||||
res.json({ config: updated.inventory });
|
||||
}));
|
||||
|
||||
inventoryRouter.post("/", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const parsed = assetSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Give it a name and a kind; dates look like 2026-10-17.", details: parsed.error.flatten() });
|
||||
const v = values(parsed.data);
|
||||
if (v.serverId !== null && (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, v.serverId))).length === 0) {
|
||||
return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
|
||||
}
|
||||
const now = new Date().toISOString();
|
||||
const [row] = await db.insert(assets).values({ ...v, createdAt: now, updatedAt: now }).returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "inventory", action: "create", targetType: "asset", targetId: row.id, detail: { name: row.name, kind: row.kind } });
|
||||
res.status(201).json({ asset: row });
|
||||
}));
|
||||
|
||||
inventoryRouter.put("/:id", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
const parsed = assetSchema.safeParse(req.body);
|
||||
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", details: parsed.success ? undefined : parsed.error.flatten() });
|
||||
const [before] = await db.select().from(assets).where(eq(assets.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
const v = values(parsed.data);
|
||||
if (v.serverId !== null && (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, v.serverId))).length === 0) {
|
||||
return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
|
||||
}
|
||||
const [row] = await db.update(assets).set({ ...v, updatedAt: new Date().toISOString() }).where(eq(assets.id, id)).returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "inventory", action: "update", targetType: "asset", targetId: id, detail: { name: row.name } });
|
||||
res.json({ asset: row });
|
||||
}));
|
||||
|
||||
inventoryRouter.delete("/:id", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [before] = await db.select().from(assets).where(eq(assets.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
await db.delete(assets).where(eq(assets.id, id));
|
||||
await recordAudit({ actor: req.currentUser!, category: "inventory", action: "delete", targetType: "asset", targetId: id, detail: { name: before.name } });
|
||||
res.status(204).end();
|
||||
}));
|
||||
@@ -0,0 +1,63 @@
|
||||
import { Router } from "express";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { z } from "zod";
|
||||
import { db } from "../db/client.js";
|
||||
import { metricsTokens } from "../db/schema.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { buildMetrics } from "../services/metrics.js";
|
||||
import { createMetricsToken, publicMetricsToken, resolveMetricsToken } from "../services/metricsTokens.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
import { rateLimit } from "../utils/rateLimit.js";
|
||||
|
||||
/** Prometheus's scrape: no sign-in, a bearer token instead. The token goes in a header only — never the address, which ends up in logs. */
|
||||
export const metricsRouter = Router();
|
||||
const limiter = rateLimit({ windowMs: 60_000, max: 120, message: "Too many requests." });
|
||||
|
||||
metricsRouter.get("/", limiter, asyncHandler(async (req, res) => {
|
||||
const match = /^Bearer\s+(\S+)$/i.exec(req.get("authorization") ?? "");
|
||||
const token = match ? await resolveMetricsToken(match[1]) : null;
|
||||
if (!token) {
|
||||
// The same answer for no token, a wrong one and a removed one.
|
||||
res.setHeader("WWW-Authenticate", 'Bearer realm="metrics"');
|
||||
return res.status(401).type("text/plain").send("A metrics token is needed: send it as 'Authorization: Bearer <token>'.\n");
|
||||
}
|
||||
res.setHeader("Cache-Control", "no-store");
|
||||
// Written out as Prometheus documents it (Express would put the parameters in another order).
|
||||
res.setHeader("Content-Type", "text/plain; version=0.0.4; charset=utf-8");
|
||||
res.end(await buildMetrics());
|
||||
}));
|
||||
|
||||
/** Managing the tokens: an administrator's. */
|
||||
export const metricsTokensRouter = Router();
|
||||
metricsTokensRouter.use(requireAuth, requireRole("admin"));
|
||||
|
||||
const MAX_TOKENS = 10;
|
||||
|
||||
metricsTokensRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const rows = await db.select().from(metricsTokens).orderBy(metricsTokens.createdAt);
|
||||
res.json({ tokens: rows.map(publicMetricsToken) });
|
||||
}));
|
||||
|
||||
// The secret is in this answer and nowhere else, ever.
|
||||
metricsTokensRouter.post("/", asyncHandler(async (req, res) => {
|
||||
const parsed = z.object({ label: z.string().trim().min(1).max(60) }).safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Give the token a name, like Prometheus." });
|
||||
if ((await db.select({ id: metricsTokens.id }).from(metricsTokens)).length >= MAX_TOKENS) {
|
||||
return res.status(400).json({ error: "too_many", message: `There are already ${MAX_TOKENS} tokens. Remove one you don't use first.` });
|
||||
}
|
||||
const actor = req.currentUser!;
|
||||
const { row, token } = await createMetricsToken({ label: parsed.data.label, createdBy: actor.name ?? actor.email ?? actor.oidcSub });
|
||||
await recordAudit({ actor, category: "metrics_token", action: "create", targetType: "metrics_token", targetId: row.id, detail: { label: row.label } });
|
||||
res.status(201).json({ token: publicMetricsToken(row), secret: token });
|
||||
}));
|
||||
|
||||
metricsTokensRouter.delete("/:id", asyncHandler(async (req, res) => {
|
||||
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [row] = await db.select().from(metricsTokens).where(eq(metricsTokens.id, id)).limit(1);
|
||||
if (!row) return res.status(404).json({ error: "not_found" });
|
||||
await db.delete(metricsTokens).where(eq(metricsTokens.id, id));
|
||||
await recordAudit({ actor: req.currentUser!, category: "metrics_token", action: "delete", targetType: "metrics_token", targetId: id, detail: { label: row.label } });
|
||||
res.status(204).end();
|
||||
}));
|
||||
@@ -0,0 +1,98 @@
|
||||
import { Router } from "express";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { z } from "zod";
|
||||
import { db } from "../db/client.js";
|
||||
import { dnsProviders, integrations, maintenanceTargetTypes, runbooks, servers } from "../db/schema.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { describeTarget } from "../services/maintenance.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const runbooksRouter = Router();
|
||||
|
||||
// Anyone signed in can read what to do when something is wrong — that is the point of having it written down. Changing it is for operators.
|
||||
runbooksRouter.use(requireAuth);
|
||||
|
||||
/** More than anyone needs for one thing; each is another note to keep true. */
|
||||
const MAX_PER_TARGET = 20;
|
||||
export const ALERT_CATEGORIES = ["offline", "disk", "backup", "updates", "expiry", "automation", "integration", "monitoring", "tickets"] as const;
|
||||
|
||||
const bodySchema = z.object({
|
||||
title: z.string().trim().min(1).max(120),
|
||||
body: z.string().max(20_000),
|
||||
/** An alert category this is for, or null for any alert about the target. */
|
||||
category: z.enum(ALERT_CATEGORIES).nullable().optional(),
|
||||
});
|
||||
const createSchema = bodySchema.extend({ targetType: z.enum(maintenanceTargetTypes), targetId: z.number().int().positive() });
|
||||
|
||||
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
|
||||
|
||||
// What there is for one thing: the full text.
|
||||
runbooksRouter.get("/", asyncHandler(async (req, res) => {
|
||||
const targetType = z.enum(maintenanceTargetTypes).safeParse(req.query.targetType);
|
||||
const targetId = idOf(String(req.query.targetId ?? ""));
|
||||
if (!targetType.success || !Number.isInteger(targetId)) return res.status(400).json({ error: "invalid_target" });
|
||||
const rows = await db.select().from(runbooks).where(and(eq(runbooks.targetType, targetType.data), eq(runbooks.targetId, targetId))).orderBy(runbooks.title);
|
||||
res.json({ runbooks: rows });
|
||||
}));
|
||||
|
||||
// What there is, everywhere, by title only — so a page can show a "runbook" button exactly where one exists, without fetching the texts.
|
||||
// Notes for something that has since been removed are left out.
|
||||
runbooksRouter.get("/summary", asyncHandler(async (_req, res) => {
|
||||
const [rows, serverIds, integrationIds, providerIds] = await Promise.all([
|
||||
db.select({ id: runbooks.id, targetType: runbooks.targetType, targetId: runbooks.targetId, category: runbooks.category, title: runbooks.title }).from(runbooks),
|
||||
db.select({ id: servers.id }).from(servers),
|
||||
db.select({ id: integrations.id }).from(integrations),
|
||||
db.select({ id: dnsProviders.id }).from(dnsProviders),
|
||||
]);
|
||||
const exists = {
|
||||
server: new Set(serverIds.map((r) => r.id)),
|
||||
integration: new Set(integrationIds.map((r) => r.id)),
|
||||
dns_provider: new Set(providerIds.map((r) => r.id)),
|
||||
};
|
||||
res.json({ runbooks: rows.filter((r) => exists[r.targetType].has(r.targetId)) });
|
||||
}));
|
||||
|
||||
runbooksRouter.post("/", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const parsed = createSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Give it a title and some text, and say what it is for.", details: parsed.error.flatten() });
|
||||
const d = parsed.data;
|
||||
if (!(await describeTarget(d.targetType, d.targetId))) return res.status(400).json({ error: "unknown_target", message: "That server, integration or DNS provider doesn't exist." });
|
||||
const existing = await db.select({ id: runbooks.id }).from(runbooks).where(and(eq(runbooks.targetType, d.targetType), eq(runbooks.targetId, d.targetId)));
|
||||
if (existing.length >= MAX_PER_TARGET) return res.status(400).json({ error: "too_many", message: `There are already ${MAX_PER_TARGET} notes for this. Tidy those first.` });
|
||||
|
||||
const actor = req.currentUser!;
|
||||
const now = new Date().toISOString();
|
||||
const [row] = await db
|
||||
.insert(runbooks)
|
||||
.values({ targetType: d.targetType, targetId: d.targetId, title: d.title, body: d.body, category: d.category ?? null, updatedBy: actor.name ?? actor.email ?? actor.oidcSub, createdAt: now, updatedAt: now })
|
||||
.returning();
|
||||
await recordAudit({ actor, category: "runbook", action: "create", targetType: d.targetType, targetId: d.targetId, detail: { title: d.title, category: d.category ?? null } });
|
||||
res.status(201).json({ runbook: row });
|
||||
}));
|
||||
|
||||
runbooksRouter.put("/:id", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
const parsed = bodySchema.safeParse(req.body);
|
||||
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", details: parsed.success ? undefined : parsed.error.flatten() });
|
||||
const [before] = await db.select().from(runbooks).where(eq(runbooks.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
const actor = req.currentUser!;
|
||||
const [row] = await db
|
||||
.update(runbooks)
|
||||
.set({ title: parsed.data.title, body: parsed.data.body, category: parsed.data.category ?? null, updatedBy: actor.name ?? actor.email ?? actor.oidcSub, updatedAt: new Date().toISOString() })
|
||||
.where(eq(runbooks.id, id))
|
||||
.returning();
|
||||
await recordAudit({ actor, category: "runbook", action: "update", targetType: before.targetType, targetId: before.targetId, detail: { title: parsed.data.title } });
|
||||
res.json({ runbook: row });
|
||||
}));
|
||||
|
||||
runbooksRouter.delete("/:id", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [before] = await db.select().from(runbooks).where(eq(runbooks.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
await db.delete(runbooks).where(eq(runbooks.id, id));
|
||||
await recordAudit({ actor: req.currentUser!, category: "runbook", action: "delete", targetType: before.targetType, targetId: before.targetId, detail: { title: before.title } });
|
||||
res.status(204).end();
|
||||
}));
|
||||
@@ -102,6 +102,16 @@ const updateSchema = z.object({
|
||||
.object({ enabled: z.boolean(), start: z.string().regex(/^\d{2}:\d{2}$/), end: z.string().regex(/^\d{2}:\d{2}$/) })
|
||||
.partial()
|
||||
.optional(),
|
||||
signIn: z
|
||||
.object({
|
||||
roleFromGroups: z.boolean(),
|
||||
groupsClaim: z.string().regex(/^[A-Za-z0-9_:.\-/]{1,60}$/, "Use the claim's name, like groups"),
|
||||
adminGroups: z.array(z.string().trim().min(1).max(100)).max(20),
|
||||
operatorGroups: z.array(z.string().trim().min(1).max(100)).max(20),
|
||||
otherwise: z.enum(["viewer", "keep"]),
|
||||
})
|
||||
.partial()
|
||||
.optional(),
|
||||
digest: z
|
||||
.object({ enabled: z.boolean(), day: z.number().int().min(0).max(6), time: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/) })
|
||||
.partial()
|
||||
|
||||
@@ -6,14 +6,32 @@ import { users, userRoles } from "../db/schema.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
import { getSettings } from "../services/settingsStore.js";
|
||||
import { roleFromGroups } from "../auth/groups.js";
|
||||
|
||||
export const usersRouter = Router();
|
||||
|
||||
usersRouter.use(requireAuth, requireRole("admin"));
|
||||
|
||||
/** What is stored for a person's groups, as a list. Anything that isn't one is none rather than an error. */
|
||||
export function parseGroups(stored: string | null): string[] | null {
|
||||
if (!stored) return null;
|
||||
try {
|
||||
const v = JSON.parse(stored);
|
||||
return Array.isArray(v) ? v.filter((g): g is string => typeof g === "string") : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
usersRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const rows = await db.select().from(users).orderBy(users.createdAt);
|
||||
res.json({ users: rows });
|
||||
const { signIn } = await getSettings();
|
||||
res.json({
|
||||
users: rows.map((u) => ({ id: u.id, oidcSub: u.oidcSub, email: u.email, name: u.name, role: u.role, createdAt: u.createdAt, lastLoginAt: u.lastLoginAt, groups: parseGroups(u.groups) })),
|
||||
// When roles follow groups, a role set by hand would be undone at the next sign-in, so the page says so instead of offering it.
|
||||
rolesFollowGroups: signIn.roleFromGroups,
|
||||
});
|
||||
}));
|
||||
|
||||
const updateRoleSchema = z.object({
|
||||
@@ -21,6 +39,10 @@ const updateRoleSchema = z.object({
|
||||
});
|
||||
|
||||
usersRouter.patch("/:id/role", asyncHandler(async (req, res) => {
|
||||
const { signIn } = await getSettings();
|
||||
if (signIn.roleFromGroups) {
|
||||
return res.status(409).json({ error: "managed_by_groups", message: "Roles are set from sign-in groups (Settings → Sign-in), so a role set here would be undone at the next sign-in. Change the person's group in the identity provider, or switch the group rules off." });
|
||||
}
|
||||
const id = Number(req.params.id);
|
||||
const parsed = updateRoleSchema.safeParse(req.body);
|
||||
if (!parsed.success) {
|
||||
@@ -60,3 +82,26 @@ usersRouter.patch("/:id/role", asyncHandler(async (req, res) => {
|
||||
|
||||
res.json({ user: updated });
|
||||
}));
|
||||
|
||||
const previewSchema = z.object({
|
||||
groupsClaim: z.string().optional(),
|
||||
adminGroups: z.array(z.string().trim().min(1).max(100)).max(20).default([]),
|
||||
operatorGroups: z.array(z.string().trim().min(1).max(100)).max(20).default([]),
|
||||
otherwise: z.enum(["viewer", "keep"]).default("viewer"),
|
||||
});
|
||||
|
||||
// What each person would be, if these group rules were switched on now and they signed in again with the groups they had last time.
|
||||
// Nothing is saved or changed: it is for checking the names before relying on them.
|
||||
usersRouter.post("/preview-group-roles", asyncHandler(async (req, res) => {
|
||||
const parsed = previewSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", details: parsed.error.flatten() });
|
||||
const rows = await db.select().from(users).orderBy(users.name);
|
||||
const rules = { roleFromGroups: true, groupsClaim: parsed.data.groupsClaim ?? "groups", adminGroups: parsed.data.adminGroups, operatorGroups: parsed.data.operatorGroups, otherwise: parsed.data.otherwise };
|
||||
const people = rows.map((u) => {
|
||||
const groups = parseGroups(u.groups);
|
||||
return { id: u.id, name: u.name ?? u.email ?? u.oidcSub, role: u.role, groups, wouldBe: groups === null ? null : roleFromGroups(rules, groups, u.role) };
|
||||
});
|
||||
// Said plainly, because it is the one way to lock yourself out: nobody left with admin.
|
||||
const adminsAfter = people.filter((p) => (p.wouldBe ?? p.role) === "admin").length;
|
||||
res.json({ people, adminsAfter });
|
||||
}));
|
||||
@@ -0,0 +1,219 @@
|
||||
/**
|
||||
* Backup coverage: for every Proxmox guest, is it in a backup job, when was it last backed up, and did that go well — across Proxmox's
|
||||
* own backup tasks and Proxmox Backup Server. It answers "what isn't backed up?" in one place, instead of an alert per gap.
|
||||
*
|
||||
* Where the answer comes from, and what it can't say:
|
||||
* - "In a job" is worked out from the backup jobs (the same rule the backup alerts use).
|
||||
* - "Last backup" is the newest snapshot of that guest on a Backup Server, or the last successful Proxmox backup task that names that
|
||||
* guest. A task from an "all guests" job doesn't name its guests (Proxmox keeps those in the task's log), so for guests backed up to
|
||||
* something other than a Backup Server, a job that covers them with no way of seeing when it last ran reads as "unknown" — said so,
|
||||
* never guessed at.
|
||||
*/
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, servers } from "../db/schema.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createPbsAdapter, type PbsBackupGroup } from "../integrations/pbs/adapter.js";
|
||||
import { createProxmoxAdapter, guestsWithoutBackupCoverage, type ProxmoxBackupJob, type ProxmoxBackupTask, type ProxmoxGuest } from "../integrations/proxmox/adapter.js";
|
||||
|
||||
export type CoverageStatus = "ok" | "stale" | "failed" | "none" | "unknown";
|
||||
|
||||
export const DEFAULT_STALE_HOURS = 48;
|
||||
|
||||
export interface CoverageGuest {
|
||||
integrationId: number;
|
||||
integrationName: string;
|
||||
node: string;
|
||||
vmid: number;
|
||||
name: string;
|
||||
type: "qemu" | "lxc";
|
||||
/** The guest's power state, as Proxmox says. */
|
||||
power: string;
|
||||
inJob: boolean;
|
||||
/** ISO — the newest backup found, from either source. */
|
||||
lastBackupAt: string | null;
|
||||
lastSource: "pbs" | "task" | null;
|
||||
status: CoverageStatus;
|
||||
/** Why it is "failed", when it is. */
|
||||
failure: string | null;
|
||||
/** The server in this app that is this guest, if one is linked to it. */
|
||||
server: { id: number; name: string } | null;
|
||||
}
|
||||
|
||||
export interface ProxmoxSource {
|
||||
integrationId: number;
|
||||
integrationName: string;
|
||||
guests: ProxmoxGuest[];
|
||||
jobs: ProxmoxBackupJob[];
|
||||
tasks: ProxmoxBackupTask[];
|
||||
}
|
||||
|
||||
export interface PbsSource {
|
||||
integrationName: string;
|
||||
groups: PbsBackupGroup[];
|
||||
}
|
||||
|
||||
export interface SourceStatus {
|
||||
kind: "proxmox" | "pbs";
|
||||
name: string;
|
||||
ok: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface LinkedServer {
|
||||
id: number;
|
||||
name: string;
|
||||
proxmoxIntegrationId: number | null;
|
||||
proxmoxVmid: number | null;
|
||||
}
|
||||
|
||||
export interface CoverageReport {
|
||||
guests: CoverageGuest[];
|
||||
counts: Record<CoverageStatus, number> & { total: number };
|
||||
sources: SourceStatus[];
|
||||
staleHours: number;
|
||||
generatedAt: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure: the rows, from what the sources said. A guest that is in no job and has no backup at all is "none"; one with a failed latest
|
||||
* run or a snapshot that failed verification is "failed"; otherwise it is "ok" or "stale" by the age of its newest backup; and one in a job
|
||||
* with nothing to say when it last ran is "unknown".
|
||||
*/
|
||||
export function buildCoverage(input: { proxmox: ProxmoxSource[]; pbs: PbsSource[]; servers: LinkedServer[]; now: Date; staleHours?: number }): CoverageGuest[] {
|
||||
const staleMs = (input.staleHours ?? DEFAULT_STALE_HOURS) * 3600_000;
|
||||
const nowMs = input.now.getTime();
|
||||
|
||||
// The newest snapshot of each guest, across every Backup Server (guests are told apart there by "vm/100" or "ct/101").
|
||||
const snapshots = new Map<string, PbsBackupGroup>();
|
||||
for (const src of input.pbs) {
|
||||
for (const g of src.groups) {
|
||||
const key = `${g.backupType}/${g.backupId}`;
|
||||
const seen = snapshots.get(key);
|
||||
if (!seen || g.backupTime > seen.backupTime) snapshots.set(key, g);
|
||||
}
|
||||
}
|
||||
|
||||
const out: CoverageGuest[] = [];
|
||||
for (const src of input.proxmox) {
|
||||
const uncovered = new Set(guestsWithoutBackupCoverage(src.guests, src.jobs).map((g) => g.vmid));
|
||||
for (const g of src.guests) {
|
||||
const forGuest = src.tasks.filter((t) => t.guestId === String(g.vmid)).sort((a, b) => b.startTime.localeCompare(a.startTime));
|
||||
const lastOkTask = forGuest.find((t) => t.ok);
|
||||
const latestTask = forGuest[0];
|
||||
const snap = snapshots.get(`${g.type === "qemu" ? "vm" : "ct"}/${g.vmid}`);
|
||||
|
||||
const candidates: { at: number; source: "pbs" | "task" }[] = [];
|
||||
if (snap) candidates.push({ at: snap.backupTime * 1000, source: "pbs" });
|
||||
if (lastOkTask) candidates.push({ at: Date.parse(lastOkTask.endTime ?? lastOkTask.startTime), source: "task" });
|
||||
const newest = candidates.sort((a, b) => b.at - a.at)[0];
|
||||
|
||||
let failure: string | null = null;
|
||||
if (snap?.verification === "failed" && newest?.source === "pbs") failure = "the newest snapshot failed verification";
|
||||
else if (latestTask && !latestTask.ok && latestTask.status !== "running" && (!newest || Date.parse(latestTask.startTime) > newest.at)) failure = `the latest backup didn't succeed (${latestTask.status})`;
|
||||
|
||||
const inJob = !uncovered.has(g.vmid);
|
||||
let status: CoverageStatus;
|
||||
if (failure) status = "failed";
|
||||
else if (newest) status = nowMs - newest.at > staleMs ? "stale" : "ok";
|
||||
else status = inJob ? "unknown" : "none";
|
||||
|
||||
const linked = input.servers.find((s) => s.proxmoxIntegrationId === src.integrationId && s.proxmoxVmid === g.vmid);
|
||||
out.push({
|
||||
integrationId: src.integrationId,
|
||||
integrationName: src.integrationName,
|
||||
node: g.node,
|
||||
vmid: g.vmid,
|
||||
name: g.name,
|
||||
type: g.type,
|
||||
power: g.status,
|
||||
inJob,
|
||||
lastBackupAt: newest ? new Date(newest.at).toISOString() : null,
|
||||
lastSource: newest ? newest.source : null,
|
||||
status,
|
||||
failure,
|
||||
server: linked ? { id: linked.id, name: linked.name } : null,
|
||||
});
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function summarize(guests: CoverageGuest[]): CoverageReport["counts"] {
|
||||
const counts = { ok: 0, stale: 0, failed: 0, none: 0, unknown: 0, total: guests.length };
|
||||
for (const g of guests) counts[g.status]++;
|
||||
return counts;
|
||||
}
|
||||
|
||||
// ─── Reading the sources ────────────────────────────────────────────────────
|
||||
|
||||
interface Loaded {
|
||||
proxmox: ProxmoxSource[];
|
||||
pbs: PbsSource[];
|
||||
sources: SourceStatus[];
|
||||
}
|
||||
|
||||
/** How long what was read from Proxmox and Backup Server is reused: a page refreshed a few times asks them once. */
|
||||
const CACHE_MS = 60_000;
|
||||
let cache: { at: number; loaded: Loaded } | null = null;
|
||||
let loaderOverride: (() => Promise<Loaded>) | null = null;
|
||||
|
||||
/** For tests: read from this instead of the real integrations. */
|
||||
export function setCoverageLoaderForTests(loader: (() => Promise<Loaded>) | null): void {
|
||||
loaderOverride = loader;
|
||||
cache = null;
|
||||
}
|
||||
|
||||
async function loadSources(): Promise<Loaded> {
|
||||
if (loaderOverride) return loaderOverride();
|
||||
const loaded: Loaded = { proxmox: [], pbs: [], sources: [] };
|
||||
|
||||
const px = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "proxmox"), eq(integrations.enabled, true)));
|
||||
await Promise.all(
|
||||
px.map(async (row) => {
|
||||
try {
|
||||
const cfg = await loadIntegrationConfig(row.id);
|
||||
if (!cfg) return;
|
||||
const adapter = createProxmoxAdapter(cfg.config as any);
|
||||
const [guests, jobs, tasks] = await Promise.all([adapter.listGuests(), adapter.listBackupJobs(), adapter.listRecentBackupTasks(50)]);
|
||||
loaded.proxmox.push({ integrationId: row.id, integrationName: row.name, guests, jobs, tasks });
|
||||
loaded.sources.push({ kind: "proxmox", name: row.name, ok: true });
|
||||
} catch (err) {
|
||||
loaded.sources.push({ kind: "proxmox", name: row.name, ok: false, error: err instanceof Error ? err.message : String(err) });
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
const pbs = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "pbs"), eq(integrations.enabled, true)));
|
||||
await Promise.all(
|
||||
pbs.map(async (row) => {
|
||||
try {
|
||||
const cfg = await loadIntegrationConfig(row.id);
|
||||
if (!cfg) return;
|
||||
const datastores = await createPbsAdapter(cfg.config as any).listDatastores();
|
||||
const readable = datastores.filter((d) => d.error === null);
|
||||
loaded.pbs.push({ integrationName: row.name, groups: readable.flatMap((d) => d.groups) });
|
||||
const unreadable = datastores.filter((d) => d.error !== null);
|
||||
loaded.sources.push(unreadable.length > 0 ? { kind: "pbs", name: row.name, ok: false, error: `couldn't read ${unreadable.map((d) => d.name).join(", ")}` } : { kind: "pbs", name: row.name, ok: true });
|
||||
} catch (err) {
|
||||
loaded.sources.push({ kind: "pbs", name: row.name, ok: false, error: err instanceof Error ? err.message : String(err) });
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
loaded.sources.sort((a, b) => a.name.localeCompare(b.name));
|
||||
return loaded;
|
||||
}
|
||||
|
||||
export async function getBackupCoverage(options: { staleHours?: number; refresh?: boolean; now?: Date } = {}): Promise<CoverageReport> {
|
||||
const now = options.now ?? new Date();
|
||||
const staleHours = options.staleHours ?? DEFAULT_STALE_HOURS;
|
||||
if (!cache || options.refresh || now.getTime() - cache.at > CACHE_MS || loaderOverride) {
|
||||
cache = { at: now.getTime(), loaded: await loadSources() };
|
||||
}
|
||||
const serverRows = await db.select({ id: servers.id, name: servers.name, proxmoxIntegrationId: servers.proxmoxIntegrationId, proxmoxVmid: servers.proxmoxVmid }).from(servers);
|
||||
const guests = buildCoverage({ proxmox: cache.loaded.proxmox, pbs: cache.loaded.pbs, servers: serverRows, now, staleHours }).sort(
|
||||
(a, b) => a.name.localeCompare(b.name) || a.vmid - b.vmid,
|
||||
);
|
||||
return { guests, counts: summarize(guests), sources: cache.loaded.sources, staleHours, generatedAt: now.toISOString() };
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
/**
|
||||
* Event webhooks: tell another system when something happens. Each webhook is a URL with the events it wants; when one happens it is
|
||||
* sent a JSON message, signed with the webhook's secret so the receiver can tell it came from here. This is separate from the chat-style
|
||||
* webhook under Settings → Notifications, which carries the same sentence a person would read.
|
||||
*
|
||||
* alert.opened / alert.resolved a problem began or ended — on the same terms as a notification (a problem under maintenance,
|
||||
* or folded under another, isn't announced until it stands alone)
|
||||
* maintenance.started / .ended a maintenance window began or finished, however it was made
|
||||
*
|
||||
* Delivery is best-effort and never holds anything up: one try, one more if that fails, then the result is noted on the webhook (shown in
|
||||
* Settings) and forgotten.
|
||||
*/
|
||||
import { createHmac, randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { eventWebhooks, webhookEvents, type WebhookEvent } from "../db/schema.js";
|
||||
import { decryptSecret, encryptSecret } from "../crypto.js";
|
||||
import { env } from "../env.js";
|
||||
|
||||
export type EventWebhookRow = typeof eventWebhooks.$inferSelect;
|
||||
|
||||
const ENCRYPTED_PREFIX = "enc:v1:";
|
||||
const TIMEOUT_MS = 5000;
|
||||
const RETRY_AFTER_MS = 2000;
|
||||
|
||||
/** The stored form of a secret: encrypted when an encryption key is set, as the notification channels' secrets are. */
|
||||
export function sealSecret(plain: string): string {
|
||||
return env.credentialsEncryptionEnabled ? ENCRYPTED_PREFIX + encryptSecret(plain) : plain;
|
||||
}
|
||||
|
||||
function openSecret(stored: string | null): string | null {
|
||||
if (!stored) return null;
|
||||
if (!stored.startsWith(ENCRYPTED_PREFIX)) return stored;
|
||||
try {
|
||||
return decryptSecret(stored.slice(ENCRYPTED_PREFIX.length));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function parseEvents(stored: string): WebhookEvent[] {
|
||||
try {
|
||||
const v = JSON.parse(stored);
|
||||
return Array.isArray(v) ? v.filter((e): e is WebhookEvent => (webhookEvents as readonly string[]).includes(e)) : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/** The signature a receiver should compute over the exact body it got: HMAC-SHA256 with the secret, as hex, prefixed "sha256=". */
|
||||
export function signBody(secret: string, body: string): string {
|
||||
return `sha256=${createHmac("sha256", secret).update(body).digest("hex")}`;
|
||||
}
|
||||
|
||||
export interface Delivery {
|
||||
ok: boolean;
|
||||
/** "200", or what went wrong. */
|
||||
result: string;
|
||||
}
|
||||
|
||||
async function post(row: EventWebhookRow, event: string, body: string, deliveryId: string): Promise<Delivery> {
|
||||
const secret = openSecret(row.secret);
|
||||
const headers: Record<string, string> = {
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "HomelabManager-Webhook",
|
||||
"X-HLM-Event": event,
|
||||
"X-HLM-Delivery": deliveryId,
|
||||
};
|
||||
if (secret) headers["X-HLM-Signature"] = signBody(secret, body);
|
||||
try {
|
||||
const res = await fetch(row.url, { method: "POST", headers, body, signal: AbortSignal.timeout(TIMEOUT_MS), redirect: "manual" });
|
||||
return { ok: res.status >= 200 && res.status < 300, result: String(res.status) };
|
||||
} catch (err) {
|
||||
const name = err instanceof Error ? err.name : "";
|
||||
return { ok: false, result: name === "TimeoutError" ? "timed out" : err instanceof Error ? err.message.slice(0, 120) : "failed" };
|
||||
}
|
||||
}
|
||||
|
||||
/** One message to one webhook, retried once if it didn't go through. The outcome is written on the webhook. */
|
||||
export async function deliver(row: EventWebhookRow, event: string, data: unknown, options: { retryAfterMs?: number } = {}): Promise<Delivery> {
|
||||
const deliveryId = randomUUID();
|
||||
const body = JSON.stringify({ event, id: deliveryId, createdAt: new Date().toISOString(), source: "homelab-manager", data });
|
||||
let outcome = await post(row, event, body, deliveryId);
|
||||
if (!outcome.ok) {
|
||||
await new Promise((r) => setTimeout(r, options.retryAfterMs ?? RETRY_AFTER_MS));
|
||||
outcome = await post(row, event, body, deliveryId);
|
||||
}
|
||||
try {
|
||||
await db.update(eventWebhooks).set({ lastAttemptAt: new Date().toISOString(), lastResult: outcome.result }).where(eq(eventWebhooks.id, row.id));
|
||||
} catch (err) {
|
||||
console.error("[webhooks] couldn't note the result:", err);
|
||||
}
|
||||
return outcome;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sends an event to every enabled webhook that wants it. Returns at once — delivery carries on in the background and can never delay or
|
||||
* fail what caused the event.
|
||||
*/
|
||||
export function emitEvent(event: WebhookEvent, data: unknown): void {
|
||||
void (async () => {
|
||||
const rows = await db.select().from(eventWebhooks).where(eq(eventWebhooks.enabled, true));
|
||||
await Promise.all(rows.filter((r) => parseEvents(r.events).includes(event)).map((r) => deliver(r, event, data)));
|
||||
})().catch((err) => console.error("[webhooks] couldn't send an event:", err));
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
/**
|
||||
* The small calculations around the hardware inventory: how long a warranty has left, and what a machine's power draw costs to run.
|
||||
* Kept apart from the routes so the rules can be tested exactly.
|
||||
*/
|
||||
|
||||
export type WarrantyStatus = "none" | "ok" | "expiring" | "expired";
|
||||
|
||||
/** A warranty that ends within this many days is "expiring". */
|
||||
export const WARRANTY_WARN_DAYS = 90;
|
||||
|
||||
/** Whether a `YYYY-MM-DD` is a real calendar date. */
|
||||
export function isRealDate(value: string): boolean {
|
||||
if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) return false;
|
||||
const t = Date.parse(`${value}T00:00:00Z`);
|
||||
return Number.isFinite(t) && new Date(t).toISOString().slice(0, 10) === value;
|
||||
}
|
||||
|
||||
/** Days from today (UTC) to the end of a warranty; negative once it has ended. Null with no warranty date. */
|
||||
export function warrantyDaysLeft(warrantyUntil: string | null, now: Date = new Date()): number | null {
|
||||
if (!warrantyUntil || !isRealDate(warrantyUntil)) return null;
|
||||
const end = Date.parse(`${warrantyUntil}T00:00:00Z`);
|
||||
const today = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate());
|
||||
return Math.round((end - today) / 86_400_000);
|
||||
}
|
||||
|
||||
export function warrantyStatus(warrantyUntil: string | null, now: Date = new Date(), warnDays: number = WARRANTY_WARN_DAYS): WarrantyStatus {
|
||||
const days = warrantyDaysLeft(warrantyUntil, now);
|
||||
if (days === null) return "none";
|
||||
if (days < 0) return "expired";
|
||||
return days <= warnDays ? "expiring" : "ok";
|
||||
}
|
||||
|
||||
/** Kilowatt-hours a steady draw uses in a year. */
|
||||
export function yearlyKwh(watts: number): number {
|
||||
return Math.round((watts / 1000) * 24 * 365 * 100) / 100;
|
||||
}
|
||||
|
||||
/** What a year of a steady draw costs at a price per kWh, rounded to whole units of currency. Null when there is no draw or no price. */
|
||||
export function yearlyCost(watts: number | null, pricePerKwh: number | null): number | null {
|
||||
if (watts === null || watts <= 0 || pricePerKwh === null || !(pricePerKwh > 0)) return null;
|
||||
return Math.round(yearlyKwh(watts) * pricePerKwh);
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
/**
|
||||
* Turns maintenance windows beginning and ending into events, whatever made the window — by hand, in bulk, from a recurring schedule or
|
||||
* mirrored from Uptime Kuma, and whether it ended on its own or was ended early. It simply watches which windows are running: one that
|
||||
* wasn't a moment ago is "started", one that was and isn't now is "ended". After a restart it starts from what is running then, so nothing
|
||||
* is announced twice or announced for a window that began before.
|
||||
*/
|
||||
import { listActiveWindows, describeTarget, type ActiveWindow } from "./maintenance.js";
|
||||
import { emitEvent } from "./eventWebhooks.js";
|
||||
|
||||
const INTERVAL_MS = 30_000;
|
||||
|
||||
let known: Map<number, ActiveWindow> | null = null;
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
async function describe(w: ActiveWindow) {
|
||||
const target = await describeTarget(w.targetType, w.targetId);
|
||||
return { id: w.id, targetType: w.targetType, targetId: w.targetId, targetName: target?.name ?? null, reason: w.reason, startedAt: w.startedAt, endsAt: w.endsAt, source: w.source };
|
||||
}
|
||||
|
||||
/** One look. Returns what it announced, for the tests. */
|
||||
export async function checkMaintenanceEvents(now: Date = new Date()): Promise<{ started: number[]; ended: number[] }> {
|
||||
const active = new Map((await listActiveWindows(now)).map((w) => [w.id, w]));
|
||||
const started: number[] = [];
|
||||
const ended: number[] = [];
|
||||
if (known !== null) {
|
||||
for (const [id, w] of active) {
|
||||
if (!known.has(id)) {
|
||||
started.push(id);
|
||||
emitEvent("maintenance.started", { window: await describe(w) });
|
||||
}
|
||||
}
|
||||
for (const [id, w] of known) {
|
||||
if (!active.has(id)) {
|
||||
ended.push(id);
|
||||
// The window as it was last seen (one ended early has had its end brought forward since), and when it was noticed to be over.
|
||||
emitEvent("maintenance.ended", { window: { ...(await describe(w)), endedAt: now.toISOString() } });
|
||||
}
|
||||
}
|
||||
}
|
||||
known = active;
|
||||
return { started, ended };
|
||||
}
|
||||
|
||||
/** For tests: forget what was running, so the next look is a first look. */
|
||||
export function resetMaintenanceEventsForTests(): void {
|
||||
known = null;
|
||||
}
|
||||
|
||||
export function initMaintenanceEvents(): void {
|
||||
if (timer) return;
|
||||
void checkMaintenanceEvents().catch((err) => console.error("[maintenanceEvents]", err));
|
||||
timer = setInterval(() => void checkMaintenanceEvents().catch((err) => console.error("[maintenanceEvents]", err)), INTERVAL_MS);
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
/**
|
||||
* What /metrics says, in Prometheus's text format: servers (reporting or not, load, memory, disks, updates, agent version), alerts by
|
||||
* severity, integrations, maintenance, things that expire. It reads what the app already knows — nothing here calls out to a server or an
|
||||
* integration on its own, so a scrape every fifteen seconds costs a few database reads (the alerts are the app's own minute-old result).
|
||||
*/
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, integrations, maintenanceWindows, secrets, servers } from "../db/schema.js";
|
||||
import { env } from "../env.js";
|
||||
import { getAlerts } from "./alerts.js";
|
||||
import { isAgentOutdated, latestAgentVersions } from "./agentVersions.js";
|
||||
import { getSourceStatus } from "./integrationHealthMonitor.js";
|
||||
import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
import { computeSecretStatus } from "./secretStatus.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { getUptime } from "./uptimeHistory.js";
|
||||
import { and, gt, lte } from "drizzle-orm";
|
||||
|
||||
/** A label value as Prometheus wants it: backslash, double quote and newline escaped. */
|
||||
export function escapeLabel(value: string): string {
|
||||
return value.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\n/g, "\\n");
|
||||
}
|
||||
|
||||
/** A metric name's HELP text may not contain a raw newline either. */
|
||||
const help = (text: string) => text.replace(/\\/g, "\\\\").replace(/\n/g, "\\n");
|
||||
|
||||
type Labels = Record<string, string>;
|
||||
|
||||
class Exposition {
|
||||
private lines: string[] = [];
|
||||
private declared = new Set<string>();
|
||||
|
||||
/** One sample. The HELP and TYPE lines for a metric are written the first time it appears. */
|
||||
add(name: string, type: "gauge" | "counter", description: string, value: number, labels: Labels = {}): void {
|
||||
if (!Number.isFinite(value)) return;
|
||||
if (!this.declared.has(name)) {
|
||||
this.declared.add(name);
|
||||
this.lines.push(`# HELP ${name} ${help(description)}`, `# TYPE ${name} ${type}`);
|
||||
}
|
||||
const parts = Object.entries(labels).map(([k, v]) => `${k}="${escapeLabel(v)}"`);
|
||||
this.lines.push(`${name}${parts.length ? `{${parts.join(",")}}` : ""} ${Number.isInteger(value) ? value : Number(value.toFixed(4))}`);
|
||||
}
|
||||
|
||||
toString(): string {
|
||||
return `${this.lines.join("\n")}\n`;
|
||||
}
|
||||
}
|
||||
|
||||
export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
const started = Date.now();
|
||||
const out = new Exposition();
|
||||
const settings = await getSettings();
|
||||
const nowSec = Math.floor(now.getTime() / 1000);
|
||||
|
||||
out.add("homelab_info", "gauge", "Always 1; the labels say which installation this is.", 1, { environment: env.environment?.name || "" });
|
||||
|
||||
// ── Servers ──
|
||||
const latest = latestAgentVersions();
|
||||
const serverRows = await db.select().from(servers);
|
||||
const offlineAfterMs = settings.healthChecks.serverOfflineMinutes * 60_000;
|
||||
for (const s of serverRows) {
|
||||
if (!s.lastSeenAt) continue; // never reported: nothing to say about it
|
||||
const server = s.name;
|
||||
const seen = Date.parse(s.lastSeenAt);
|
||||
out.add("homelab_server_up", "gauge", "1 if the server's agent has reported within the offline threshold, else 0.", now.getTime() - seen <= offlineAfterMs ? 1 : 0, { server });
|
||||
out.add("homelab_server_last_seen_timestamp_seconds", "gauge", "When the server's agent last reported, as a Unix time.", Math.floor(seen / 1000), { server });
|
||||
if (s.cpuLoadPercent !== null) out.add("homelab_server_cpu_load_percent", "gauge", "CPU load as the agent reports it.", s.cpuLoadPercent, { server });
|
||||
if (s.memTotalBytes && s.memUsedBytes !== null) out.add("homelab_server_memory_used_percent", "gauge", "Memory in use.", (s.memUsedBytes / s.memTotalBytes) * 100, { server });
|
||||
let disks: { mount: string; sizeBytes: number; usedBytes: number }[] = [];
|
||||
try {
|
||||
disks = s.disks ? JSON.parse(s.disks) : [];
|
||||
} catch {
|
||||
// a garbled report shouldn't blank the whole scrape
|
||||
}
|
||||
for (const d of disks) {
|
||||
if (d.sizeBytes > 0) out.add("homelab_server_disk_used_percent", "gauge", "How full a disk is.", (d.usedBytes / d.sizeBytes) * 100, { server, mount: d.mount });
|
||||
}
|
||||
const updates = parsePendingUpdates(s.pendingUpdates);
|
||||
if (updates) {
|
||||
out.add("homelab_server_pending_updates", "gauge", "Updates the agent says are waiting to be installed.", updates.count, { server });
|
||||
if (updates.security !== null) out.add("homelab_server_pending_security_updates", "gauge", "Of those, the ones that are security fixes.", updates.security, { server });
|
||||
out.add("homelab_server_reboot_required", "gauge", "1 if a restart is due.", updates.rebootRequired ? 1 : 0, { server });
|
||||
}
|
||||
out.add("homelab_server_agent_outdated", "gauge", "1 if the server's agent is older than the one this installation would install now.", isAgentOutdated(s, latest) ? 1 : 0, { server });
|
||||
}
|
||||
try {
|
||||
const uptime = await getUptime(now, 30);
|
||||
for (const s of serverRows) {
|
||||
const series = uptime.servers[s.id];
|
||||
if (series) out.add("homelab_server_uptime_ratio_30d", "gauge", "Share of the last 30 days a server was up, 0 to 1.", series.percent / 100, { server: s.name });
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[metrics] couldn't read the uptime:", err);
|
||||
}
|
||||
|
||||
// ── Alerts ──
|
||||
try {
|
||||
const report = await getAlerts(false);
|
||||
for (const severity of ["critical", "warning", "info"] as const) out.add("homelab_alerts", "gauge", "Active alerts by severity (not counting silenced ones).", report.counts[severity], { severity });
|
||||
out.add("homelab_alerts_silenced", "gauge", "Alerts held back by a maintenance window.", report.counts.silenced);
|
||||
out.add("homelab_alerts_grouped", "gauge", "Alerts folded under a server that is down.", report.counts.grouped);
|
||||
} catch (err) {
|
||||
console.error("[metrics] couldn't read the alerts:", err);
|
||||
}
|
||||
|
||||
// ── Integrations ──
|
||||
for (const i of await db.select({ name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations)) {
|
||||
if (!i.enabled) continue;
|
||||
const status = getSourceStatus(i.type).status;
|
||||
if (status === "unknown") continue; // nothing has called it yet: not up, not down
|
||||
out.add("homelab_integration_up", "gauge", "1 if the integration's latest calls are working, 0 if they are failing.", status === "ok" ? 1 : 0, { integration: i.name, type: i.type });
|
||||
}
|
||||
|
||||
// ── Maintenance ──
|
||||
const iso = now.toISOString();
|
||||
const active = await db.select({ id: maintenanceWindows.id }).from(maintenanceWindows).where(and(lte(maintenanceWindows.startedAt, iso), gt(maintenanceWindows.endsAt, iso)));
|
||||
out.add("homelab_maintenance_active_windows", "gauge", "Maintenance windows running now.", active.length);
|
||||
|
||||
// ── Things that expire ──
|
||||
for (const s of await db.select().from(secrets)) {
|
||||
const status = computeSecretStatus(s.expiryDate, s.warnDays, now);
|
||||
if (status.daysLeft !== null) out.add("homelab_secret_expiry_days", "gauge", "Days until a secret expires (negative once it has).", status.daysLeft, { name: s.name });
|
||||
}
|
||||
|
||||
// ── Inventory ──
|
||||
const power = (await db.select({ w: assets.powerWatts }).from(assets)).reduce((n, a) => n + (a.w ?? 0), 0);
|
||||
out.add("homelab_inventory_power_watts", "gauge", "Typical power draw of everything in the inventory, in watts.", power);
|
||||
|
||||
out.add("homelab_scrape_duration_seconds", "gauge", "How long building this answer took.", (Date.now() - started) / 1000);
|
||||
out.add("homelab_scrape_timestamp_seconds", "gauge", "When this answer was built, as a Unix time.", nowSec);
|
||||
return out.toString();
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
/**
|
||||
* Metrics tokens: read-only access to /metrics for Prometheus. Like an agent token or a share link, the secret is shown once, when it is
|
||||
* made, and only its SHA-256 is kept — so it can't be read back out of the database or a backup.
|
||||
*/
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { metricsTokens } from "../db/schema.js";
|
||||
|
||||
export type MetricsTokenRow = typeof metricsTokens.$inferSelect;
|
||||
|
||||
const hash = (token: string) => createHash("sha256").update(token).digest("hex");
|
||||
|
||||
/** Use is not recorded more than once in this time, so a scrape every fifteen seconds doesn't write the database each time. */
|
||||
const USED_WRITE_GAP_MS = 60_000;
|
||||
|
||||
export async function createMetricsToken(params: { label: string; createdBy: string }): Promise<{ row: MetricsTokenRow; token: string }> {
|
||||
const token = `hlm_m_${randomBytes(24).toString("base64url")}`;
|
||||
const [row] = await db.insert(metricsTokens).values({ label: params.label, tokenHash: hash(token), tokenPrefix: token.slice(0, 12), createdBy: params.createdBy }).returning();
|
||||
return { row, token };
|
||||
}
|
||||
|
||||
/** The token a secret belongs to, if there is one; notes that it was used. */
|
||||
export async function resolveMetricsToken(token: string, now: Date = new Date()): Promise<MetricsTokenRow | null> {
|
||||
if (typeof token !== "string" || token.length < 20 || token.length > 128) return null;
|
||||
const [row] = await db.select().from(metricsTokens).where(eq(metricsTokens.tokenHash, hash(token))).limit(1);
|
||||
if (!row) return null;
|
||||
if (!row.lastUsedAt || now.getTime() - Date.parse(row.lastUsedAt) >= USED_WRITE_GAP_MS) {
|
||||
await db.update(metricsTokens).set({ lastUsedAt: now.toISOString() }).where(eq(metricsTokens.id, row.id));
|
||||
}
|
||||
return row;
|
||||
}
|
||||
|
||||
/** A token as the admin list shows it: everything but the hash. */
|
||||
export function publicMetricsToken(row: MetricsTokenRow) {
|
||||
return { id: row.id, label: row.label, tokenPrefix: row.tokenPrefix, createdBy: row.createdBy, createdAt: row.createdAt, lastUsedAt: row.lastUsedAt };
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import nodemailer from "nodemailer";
|
||||
import { getSettings, type NotificationEvents } from "./settingsStore.js";
|
||||
import { enqueueNotification, listQueuedNotifications, clearQueuedNotifications } from "./notificationQueue.js";
|
||||
import { emitEvent } from "./eventWebhooks.js";
|
||||
|
||||
// ─── Channel senders (best-effort — failures are logged, never thrown) ──────
|
||||
|
||||
@@ -309,7 +310,9 @@ export async function notifyPbsVerificationFailed(
|
||||
);
|
||||
}
|
||||
|
||||
export async function notifyHealthIssues(issues: { message: string }[]): Promise<void> {
|
||||
export async function notifyHealthIssues(issues: { message: string; key?: string; source?: string; severity?: string }[]): Promise<void> {
|
||||
// The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told.
|
||||
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
|
||||
if (issues.length === 0) return;
|
||||
if (!(await eventEnabled("healthAlerts"))) return;
|
||||
await notify(
|
||||
@@ -318,7 +321,8 @@ export async function notifyHealthIssues(issues: { message: string }[]): Promise
|
||||
);
|
||||
}
|
||||
|
||||
export async function notifyHealthRecovered(resolved: { message: string }[]): Promise<void> {
|
||||
export async function notifyHealthRecovered(resolved: { message: string; key?: string }[]): Promise<void> {
|
||||
for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } });
|
||||
if (resolved.length === 0) return;
|
||||
if (!(await eventEnabled("healthAlerts"))) return;
|
||||
await notify(
|
||||
@@ -327,7 +331,9 @@ export async function notifyHealthRecovered(resolved: { message: string }[]): Pr
|
||||
);
|
||||
}
|
||||
|
||||
export async function notifyAutomationFailed(issues: { message: string }[]): Promise<void> {
|
||||
export async function notifyAutomationFailed(issues: { message: string; key?: string; source?: string; severity?: string }[]): Promise<void> {
|
||||
// The event webhooks hear of it whatever this notification is switched on for: they are a separate way of being told.
|
||||
for (const i of issues) emitEvent("alert.opened", { alert: { key: i.key ?? null, source: i.source ?? null, severity: i.severity ?? "warning", message: i.message } });
|
||||
if (issues.length === 0) return;
|
||||
if (!(await eventEnabled("automationAlerts"))) return;
|
||||
await notify(
|
||||
@@ -336,7 +342,8 @@ export async function notifyAutomationFailed(issues: { message: string }[]): Pro
|
||||
);
|
||||
}
|
||||
|
||||
export async function notifyAutomationRecovered(resolved: { message: string }[]): Promise<void> {
|
||||
export async function notifyAutomationRecovered(resolved: { message: string; key?: string }[]): Promise<void> {
|
||||
for (const r of resolved) emitEvent("alert.resolved", { alert: { key: r.key ?? null, message: r.message } });
|
||||
if (resolved.length === 0) return;
|
||||
if (!(await eventEnabled("automationAlerts"))) return;
|
||||
await notify(
|
||||
@@ -409,6 +416,7 @@ export function sourceLabel(source: string): string {
|
||||
}
|
||||
|
||||
export async function notifyIntegrationDown(source: string, consecutiveFailures: number): Promise<void> {
|
||||
emitEvent("alert.opened", { alert: { key: `integration:${source}`, source, severity: "warning", message: `${sourceLabel(source)} has failed its last ${consecutiveFailures} calls in a row.` } });
|
||||
if (!(await eventEnabled("integrationFailureAlerts"))) return;
|
||||
await notify(
|
||||
"Homelab Manager — Integration Down",
|
||||
@@ -417,6 +425,7 @@ export async function notifyIntegrationDown(source: string, consecutiveFailures:
|
||||
}
|
||||
|
||||
export async function notifyIntegrationRecovered(source: string): Promise<void> {
|
||||
emitEvent("alert.resolved", { alert: { key: `integration:${source}`, message: `${sourceLabel(source)} is answering again.` } });
|
||||
if (!(await eventEnabled("integrationFailureAlerts"))) return;
|
||||
await notify("Homelab Manager — Integration Recovered", `${sourceLabel(source)} succeeded again after failing.`);
|
||||
}
|
||||
@@ -108,6 +108,24 @@ export interface DigestSettings {
|
||||
time: string;
|
||||
}
|
||||
|
||||
/** How roles are decided at sign-in: by hand on the Users page (the default), or from the groups the identity provider reports. */
|
||||
export interface SignInSettings {
|
||||
roleFromGroups: boolean;
|
||||
/** The claim that carries the groups; "groups" in Authentik. */
|
||||
groupsClaim: string;
|
||||
adminGroups: string[];
|
||||
operatorGroups: string[];
|
||||
/** Someone in none of the named groups gets viewer, or keeps what they have. */
|
||||
otherwise: "viewer" | "keep";
|
||||
}
|
||||
|
||||
/** What the inventory needs to work out a running cost. Without a price per kWh it shows draw but no cost. */
|
||||
export interface InventorySettings {
|
||||
/** A short name shown after amounts ("SEK", "kr", "$"); may be blank. */
|
||||
currency: string;
|
||||
pricePerKwh: number | null;
|
||||
}
|
||||
|
||||
export interface ConsistencySettings {
|
||||
/** CIDR ranges and single addresses the consistency report ignores entirely — e.g. Docker networks that repeat on many hosts. */
|
||||
excludedRanges: string[];
|
||||
@@ -160,6 +178,8 @@ export interface AppSettings {
|
||||
quietHours: QuietHoursSettings;
|
||||
healthChecks: HealthCheckSettings;
|
||||
digest: DigestSettings;
|
||||
signIn: SignInSettings;
|
||||
inventory: InventorySettings;
|
||||
consistency: ConsistencySettings;
|
||||
nameGenerator: NameGeneratorSettings;
|
||||
alertTickets: AlertTicketSettings;
|
||||
@@ -193,6 +213,8 @@ const DEFAULTS: AppSettings = {
|
||||
logRetention: { enabled: false, retentionDays: 90, intervalHours: 24 },
|
||||
quietHours: { enabled: false, start: "22:00", end: "07:00" },
|
||||
digest: { enabled: false, day: 1, time: "08:00" },
|
||||
inventory: { currency: "", pricePerKwh: null },
|
||||
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
|
||||
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14 },
|
||||
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
|
||||
// Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN.
|
||||
|
||||
@@ -5,9 +5,11 @@
|
||||
* into a few lines per topic and leaves out any topic with nothing to say. Plain text, so it reads in a phone notification and an email alike.
|
||||
*/
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, secrets, servers } from "../db/schema.js";
|
||||
import { assets, integrations, secrets, servers } from "../db/schema.js";
|
||||
import { warrantyDaysLeft } from "./inventory.js";
|
||||
import { env } from "../env.js";
|
||||
import { getAlerts } from "./alerts.js";
|
||||
import { getBackupCoverage } from "./backupCoverage.js";
|
||||
import { getChangesSince } from "./changesSince.js";
|
||||
import { collectDomainAlerts } from "./domainMonitor.js";
|
||||
import { describeTarget, listUpcomingWindows } from "./maintenance.js";
|
||||
@@ -157,6 +159,13 @@ export async function buildDigest(now: Date = new Date()): Promise<Digest> {
|
||||
for (const d of (await collectDomainAlerts(now)).expiring) {
|
||||
expiring.push({ daysLeft: d.daysLeft, text: d.daysLeft < 0 ? `Domain ${d.name} expired ${plural(Math.abs(d.daysLeft), "day")} ago` : `Domain ${d.name} expires in ${plural(d.daysLeft, "day")} (${d.expiresAt})` });
|
||||
}
|
||||
// A warranty about to end is worth knowing while there is still time to claim on it; one that ended long ago is not news.
|
||||
for (const a of await db.select({ name: assets.name, warrantyUntil: assets.warrantyUntil }).from(assets)) {
|
||||
const days = warrantyDaysLeft(a.warrantyUntil, now);
|
||||
if (days !== null && days >= -30 && days <= 60) {
|
||||
expiring.push({ daysLeft: days, text: days < 0 ? `Warranty on ${a.name} ended ${plural(Math.abs(days), "day")} ago` : `Warranty on ${a.name} ends in ${plural(days, "day")} (${a.warrantyUntil})` });
|
||||
}
|
||||
}
|
||||
if (expiring.length > 0) {
|
||||
expiring.sort((a, b) => a.daysLeft - b.daysLeft);
|
||||
sections.push({ heading: "Expiring", lines: listWithMore(expiring.map((e) => `- ${e.text}`)), attention: true });
|
||||
@@ -185,6 +194,22 @@ export async function buildDigest(now: Date = new Date()): Promise<Digest> {
|
||||
console.error("[digest] couldn't read the pending updates:", err);
|
||||
}
|
||||
|
||||
// ── Backups ──
|
||||
try {
|
||||
const coverage = await getBackupCoverage({ now });
|
||||
const order = { failed: 0, none: 1, stale: 2 } as const;
|
||||
const bad = coverage.guests.filter((g): g is typeof g & { status: keyof typeof order } => g.status === "failed" || g.status === "none" || g.status === "stale").sort((a, b) => order[a.status] - order[b.status] || a.name.localeCompare(b.name));
|
||||
if (bad.length > 0) {
|
||||
const what = (g: (typeof bad)[number]) =>
|
||||
g.status === "failed" ? `${g.failure}` : g.status === "none" ? "no backup, and not in any backup job" : `last backup ${spanWords((now.getTime() - Date.parse(g.lastBackupAt!)) / 60_000)} ago`;
|
||||
const lines = [`${coverage.counts.ok} of ${plural(coverage.counts.total, "guest")} ${coverage.counts.ok === 1 ? "has" : "have"} a recent backup.`];
|
||||
lines.push(...listWithMore(bad.map((g) => `- ${g.name} (${g.type === "qemu" ? "VM" : "container"} ${g.vmid}): ${what(g)}`)));
|
||||
sections.push({ heading: "Backups", lines, attention: true });
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[digest] couldn't read the backup coverage:", err);
|
||||
}
|
||||
|
||||
// ── Planned ──
|
||||
try {
|
||||
const horizon = now.getTime() + DIGEST_DAYS * DAY_MS;
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
/**
|
||||
* Backup coverage: what the rows say for each kind of guest, and the route and the digest around them.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-coverage-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { buildCoverage, summarize, setCoverageLoaderForTests } = await import("../src/services/backupCoverage.js");
|
||||
const { backupsRouter } = await import("../src/routes/backups.js");
|
||||
const { buildDigest } = await import("../src/services/weeklyDigest.js");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const NOW = new Date("2026-10-17T12:00:00Z");
|
||||
const HOUR = 3600_000;
|
||||
const guest = (vmid: number, name: string, type: "qemu" | "lxc" = "qemu", node = "pve1") => ({ vmid, name, node, type, status: "running", cpu: null, maxmem: null, mem: null, uptime: null });
|
||||
const allJob = { id: "j1", enabled: true, schedule: "02:00", storage: "pbs", node: null, allGuests: true, vmids: null, exclude: null };
|
||||
const task = (guestId: string | null, ago: number, ok = true, status = ok ? "OK" : "job errors") => ({
|
||||
node: "pve1",
|
||||
upid: `upid-${guestId}-${ago}`,
|
||||
guestId,
|
||||
status,
|
||||
ok,
|
||||
startTime: new Date(NOW.getTime() - ago * HOUR).toISOString(),
|
||||
endTime: new Date(NOW.getTime() - ago * HOUR + 600_000).toISOString(),
|
||||
});
|
||||
const snap = (type: string, id: string, ago: number, verification: "ok" | "failed" | null = "ok") => ({ backupType: type, backupId: id, backupTime: Math.floor((NOW.getTime() - ago * HOUR) / 1000), verification });
|
||||
|
||||
const px = (guests: ReturnType<typeof guest>[], jobs: (typeof allJob)[], tasks: ReturnType<typeof task>[]) => [{ integrationId: 1, integrationName: "Felicia", guests, jobs, tasks }];
|
||||
|
||||
describe("what each guest reads as", () => {
|
||||
test("backed up recently to a Backup Server: ok; a while ago: stale", () => {
|
||||
const rows = buildCoverage({ proxmox: px([guest(100, "web"), guest(101, "db", "lxc")], [allJob], []), pbs: [{ integrationName: "PBS", groups: [snap("vm", "100", 5), snap("ct", "101", 100)] }], servers: [], now: NOW });
|
||||
const by = Object.fromEntries(rows.map((r) => [r.name, r]));
|
||||
assert.equal(by.web.status, "ok");
|
||||
assert.equal(by.web.lastSource, "pbs");
|
||||
assert.equal(by.db.status, "stale", "100 hours is more than the 48 by default");
|
||||
assert.equal(by.web.lastBackupAt, new Date(NOW.getTime() - 5 * HOUR).toISOString());
|
||||
});
|
||||
|
||||
test("a task that names the guest counts, and the newer of the two sources wins", () => {
|
||||
const rows = buildCoverage({ proxmox: px([guest(100, "web")], [allJob], [task("100", 30), task("100", 3, false)]), pbs: [{ integrationName: "PBS", groups: [snap("vm", "100", 10)] }], servers: [], now: NOW });
|
||||
assert.equal(rows[0].lastSource, "pbs", "the snapshot (10 h) is newer than the successful task (30 h)");
|
||||
assert.equal(rows[0].status, "failed", "but the latest run, 3 h ago, failed");
|
||||
assert.match(rows[0].failure!, /job errors/);
|
||||
});
|
||||
|
||||
test("a failed run followed by a good backup is not a failure any more", () => {
|
||||
const rows = buildCoverage({ proxmox: px([guest(100, "web")], [allJob], [task("100", 8, false), task("100", 2, true)]), pbs: [], servers: [], now: NOW });
|
||||
assert.equal(rows[0].status, "ok");
|
||||
assert.equal(rows[0].lastSource, "task");
|
||||
});
|
||||
|
||||
test("a snapshot that failed verification is a failure", () => {
|
||||
const rows = buildCoverage({ proxmox: px([guest(100, "web")], [allJob], []), pbs: [{ integrationName: "PBS", groups: [snap("vm", "100", 2, "failed")] }], servers: [], now: NOW });
|
||||
assert.equal(rows[0].status, "failed");
|
||||
assert.match(rows[0].failure!, /verification/);
|
||||
});
|
||||
|
||||
test("in a job with nothing to say when it last ran is unknown; in no job and no backup is none", () => {
|
||||
const only101 = { ...allJob, allGuests: false, vmids: "100" };
|
||||
const rows = buildCoverage({ proxmox: px([guest(100, "covered"), guest(101, "forgotten")], [only101], []), pbs: [], servers: [], now: NOW });
|
||||
const by = Object.fromEntries(rows.map((r) => [r.name, r]));
|
||||
assert.equal(by.covered.status, "unknown");
|
||||
assert.equal(by.covered.inJob, true);
|
||||
assert.equal(by.forgotten.status, "none");
|
||||
assert.equal(by.forgotten.inJob, false);
|
||||
});
|
||||
|
||||
test("a guest in no job that still has an old backup is judged by its age", () => {
|
||||
const rows = buildCoverage({ proxmox: px([guest(101, "manual")], [], []), pbs: [{ integrationName: "PBS", groups: [snap("vm", "101", 6)] }], servers: [], now: NOW });
|
||||
assert.equal(rows[0].status, "ok");
|
||||
assert.equal(rows[0].inJob, false);
|
||||
});
|
||||
|
||||
test("how long counts as stale is up to the caller", () => {
|
||||
const input = { proxmox: px([guest(100, "web")], [allJob], []), pbs: [{ integrationName: "PBS", groups: [snap("vm", "100", 100)] }], servers: [], now: NOW };
|
||||
assert.equal(buildCoverage({ ...input, staleHours: 24 * 3 })[0].status, "stale");
|
||||
assert.equal(buildCoverage({ ...input, staleHours: 24 * 7 })[0].status, "ok");
|
||||
});
|
||||
|
||||
test("a VM and a container with the same number aren't mixed up, and a linked server is named", () => {
|
||||
const rows = buildCoverage({
|
||||
proxmox: px([guest(100, "vm100", "qemu"), guest(100, "ct100", "lxc", "pve2")], [allJob], []),
|
||||
pbs: [{ integrationName: "PBS", groups: [snap("ct", "100", 1)] }],
|
||||
servers: [{ id: 7, name: "elsa", proxmoxIntegrationId: 1, proxmoxVmid: 100 }],
|
||||
now: NOW,
|
||||
});
|
||||
const by = Object.fromEntries(rows.map((r) => [r.name, r]));
|
||||
assert.equal(by.ct100.status, "ok");
|
||||
assert.equal(by.vm100.status, "unknown", "only the container has a snapshot");
|
||||
assert.deepEqual(by.vm100.server, { id: 7, name: "elsa" });
|
||||
});
|
||||
|
||||
test("the counts add up", () => {
|
||||
const rows = buildCoverage({ proxmox: px([guest(100, "a"), guest(101, "b")], [{ ...allJob, allGuests: false, vmids: "100" }], []), pbs: [{ integrationName: "PBS", groups: [snap("vm", "100", 1)] }], servers: [], now: NOW });
|
||||
assert.deepEqual(summarize(rows), { ok: 1, stale: 0, failed: 0, none: 1, unknown: 0, total: 2 });
|
||||
});
|
||||
});
|
||||
|
||||
describe("the route and the digest", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values({ oidcSub: "viewer", name: "Vic", role: "viewer" });
|
||||
await db.insert(servers).values({ name: "elsa", apiTokenHash: "a", apiTokenPrefix: "a", lastSeenAt: new Date().toISOString() });
|
||||
setCoverageLoaderForTests(async () => ({
|
||||
proxmox: px([guest(100, "web"), guest(101, "lonely")], [{ ...allJob, allGuests: false, vmids: "100" }], []).map((p) => ({ ...p })),
|
||||
pbs: [{ integrationName: "PBS", groups: [snap("vm", "100", 100)] }],
|
||||
sources: [
|
||||
{ kind: "proxmox" as const, name: "Felicia", ok: true },
|
||||
{ kind: "pbs" as const, name: "PBS", ok: false, error: "couldn't read tank" },
|
||||
],
|
||||
}));
|
||||
const a = express();
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/backups", backupsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/backups`;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
setCoverageLoaderForTests(null);
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
test("anyone signed in gets the rows, the counts, and which sources couldn't be read", async () => {
|
||||
const body = await (await fetch(`${base}/coverage`)).json();
|
||||
assert.equal(body.staleHours, 48);
|
||||
assert.deepEqual(body.guests.map((g: { name: string }) => g.name), ["lonely", "web"]);
|
||||
assert.equal(body.counts.total, 2);
|
||||
assert.equal(body.sources.find((s: { name: string }) => s.name === "PBS").ok, false);
|
||||
});
|
||||
|
||||
test("the stale threshold is set in days, and nonsense falls back to the default", async () => {
|
||||
// web's newest snapshot is 100 hours old: stale at 2 days, fine at 7.
|
||||
assert.equal((await (await fetch(`${base}/coverage?staleDays=7`)).json()).guests.find((g: { name: string }) => g.name === "web").status, "ok");
|
||||
assert.equal((await (await fetch(`${base}/coverage?staleDays=2`)).json()).guests.find((g: { name: string }) => g.name === "web").status, "stale");
|
||||
assert.equal((await (await fetch(`${base}/coverage?staleDays=banana`)).json()).staleHours, 48);
|
||||
assert.equal((await (await fetch(`${base}/coverage?staleDays=9999`)).json()).staleHours, 48);
|
||||
});
|
||||
|
||||
test("the weekly digest lists what isn't backed up, worst first", async () => {
|
||||
const digest = await buildDigest(new Date());
|
||||
assert.match(digest.message, /BACKUPS\n0 of 2 guests have a recent backup\.\n- lonely \(VM 101\): no backup, and not in any backup job\n- web \(VM 100\): last backup \d+ d( \d+ h)? ago/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,183 @@
|
||||
/**
|
||||
* The hardware inventory: warranty and running-cost arithmetic, and the routes.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-inventory-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, auditLog } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { inventoryRouter } = await import("../src/routes/inventory.js");
|
||||
const { isRealDate, warrantyDaysLeft, warrantyStatus, yearlyKwh, yearlyCost } = await import("../src/services/inventory.js");
|
||||
const { buildDigest } = await import("../src/services/weeklyDigest.js");
|
||||
const { getSettings } = await import("../src/services/settingsStore.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const NOW = new Date("2026-10-17T12:30:00Z");
|
||||
|
||||
describe("dates and warranties", () => {
|
||||
test("only real calendar dates", () => {
|
||||
assert.equal(isRealDate("2026-10-17"), true);
|
||||
assert.equal(isRealDate("2024-02-29"), true);
|
||||
for (const bad of ["2026-02-30", "2026-13-01", "17/10/2026", "2026-1-5", "", "yesterday"]) assert.equal(isRealDate(bad), false, bad);
|
||||
});
|
||||
|
||||
test("days left count whole days from today, and go negative once it has ended", () => {
|
||||
assert.equal(warrantyDaysLeft("2026-10-17", NOW), 0, "ends today: still covered");
|
||||
assert.equal(warrantyDaysLeft("2026-10-27", NOW), 10);
|
||||
assert.equal(warrantyDaysLeft("2026-10-10", NOW), -7);
|
||||
assert.equal(warrantyDaysLeft(null, NOW), null);
|
||||
assert.equal(warrantyDaysLeft("garbage", NOW), null);
|
||||
});
|
||||
|
||||
test("status: none, ok, expiring within 90 days, expired", () => {
|
||||
assert.equal(warrantyStatus(null, NOW), "none");
|
||||
assert.equal(warrantyStatus("2028-01-01", NOW), "ok");
|
||||
assert.equal(warrantyStatus("2027-01-10", NOW), "expiring", "86 days");
|
||||
assert.equal(warrantyStatus("2027-01-20", NOW), "ok", "95 days");
|
||||
assert.equal(warrantyStatus("2026-10-17", NOW), "expiring");
|
||||
assert.equal(warrantyStatus("2026-10-16", NOW), "expired");
|
||||
});
|
||||
});
|
||||
|
||||
describe("running cost", () => {
|
||||
test("a steady draw over a year, at a price per kWh", () => {
|
||||
assert.equal(yearlyKwh(100), 876);
|
||||
assert.equal(yearlyCost(100, 2), 1752);
|
||||
assert.equal(yearlyCost(45, 1.5), Math.round(0.045 * 8760 * 1.5));
|
||||
});
|
||||
test("no draw or no price is no cost, not zero", () => {
|
||||
assert.equal(yearlyCost(null, 2), null);
|
||||
assert.equal(yearlyCost(0, 2), null);
|
||||
assert.equal(yearlyCost(100, null), null);
|
||||
assert.equal(yearlyCost(100, 0), null);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the inventory routes", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "operator";
|
||||
let elsa = 0;
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
||||
{ oidcSub: "operator", name: "Olga", role: "operator" },
|
||||
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
||||
]);
|
||||
elsa = (await db.insert(servers).values({ name: "elsa", apiTokenHash: "a", apiTokenPrefix: "a" }).returning())[0].id;
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/inventory", inventoryRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/inventory`;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const send = (method: string, path: string, body?: unknown) =>
|
||||
fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
|
||||
let nasId = 0;
|
||||
|
||||
test("an operator adds hardware, and blanks mean no value", async () => {
|
||||
const res = await send("POST", "/", { name: "Synology DS920+", kind: "storage", manufacturer: "Synology", model: "DS920+", serial: " ", purchaseDate: "2024-03-01", purchasePrice: 5200, warrantyUntil: "2027-03-01", location: "Rack, U12", rackUnit: "U12", powerWatts: 45, notes: "" });
|
||||
assert.equal(res.status, 201);
|
||||
const row = (await res.json()).asset;
|
||||
nasId = row.id;
|
||||
assert.equal(row.serial, null);
|
||||
assert.equal(row.notes, null);
|
||||
assert.equal(row.powerWatts, 45);
|
||||
assert.equal((await send("POST", "/", { name: "elsa's box", kind: "server", serverId: elsa, powerWatts: 120, warrantyUntil: new Date(Date.now() + 20 * 86_400_000).toISOString().slice(0, 10) })).status, 201);
|
||||
});
|
||||
|
||||
test("everyone signed in can read the list, with the linked server's name and the price of power", async () => {
|
||||
who = "viewer";
|
||||
const body = await (await send("GET", "/")).json();
|
||||
assert.equal(body.assets.length, 2);
|
||||
const box = body.assets.find((a: { name: string }) => a.name === "elsa's box");
|
||||
assert.deepEqual(box.server, { id: elsa, name: "elsa" });
|
||||
assert.equal(body.assets.find((a: { name: string }) => a.name.startsWith("Synology")).server, null);
|
||||
assert.deepEqual(body.config, { currency: "", pricePerKwh: null });
|
||||
assert.deepEqual(body.servers, [{ id: elsa, name: "elsa" }]);
|
||||
who = "operator";
|
||||
});
|
||||
|
||||
test("only operators change it, and only an administrator sets the price of power", async () => {
|
||||
who = "viewer";
|
||||
assert.equal((await send("POST", "/", { name: "x", kind: "other" })).status, 403);
|
||||
assert.equal((await send("PUT", `/${nasId}`, { name: "x", kind: "other" })).status, 403);
|
||||
assert.equal((await send("DELETE", `/${nasId}`)).status, 403);
|
||||
who = "operator";
|
||||
assert.equal((await send("PUT", "/config", { currency: "SEK", pricePerKwh: 1.8 })).status, 403);
|
||||
who = "admin";
|
||||
const ok = await send("PUT", "/config", { currency: "SEK", pricePerKwh: 1.8 });
|
||||
assert.equal(ok.status, 200);
|
||||
assert.deepEqual((await getSettings()).inventory, { currency: "SEK", pricePerKwh: 1.8 });
|
||||
assert.equal((await send("PUT", "/config", { currency: "SEK", pricePerKwh: -1 })).status, 400);
|
||||
who = "operator";
|
||||
});
|
||||
|
||||
test("a bad entry is refused: no name, made-up kind, impossible date, negative price, unknown server", async () => {
|
||||
for (const bad of [
|
||||
{ name: "", kind: "other" },
|
||||
{ name: "x", kind: "toaster" },
|
||||
{ name: "x", kind: "other", purchaseDate: "2026-02-30" },
|
||||
{ name: "x", kind: "other", warrantyUntil: "next year" },
|
||||
{ name: "x", kind: "other", purchasePrice: -5 },
|
||||
{ name: "x", kind: "other", powerWatts: 1.5 },
|
||||
{ name: "x", kind: "other", serverId: 99999 },
|
||||
]) {
|
||||
assert.equal((await send("POST", "/", bad)).status, 400, JSON.stringify(bad));
|
||||
}
|
||||
});
|
||||
|
||||
test("editing and removing work, and are in the audit log", async () => {
|
||||
const put = await send("PUT", `/${nasId}`, { name: "Synology DS920+ (hall)", kind: "storage", powerWatts: 50 });
|
||||
assert.equal(put.status, 200);
|
||||
const row = (await put.json()).asset;
|
||||
assert.equal(row.name, "Synology DS920+ (hall)");
|
||||
assert.equal(row.manufacturer, null, "a field left out is cleared, not kept: this replaces the entry");
|
||||
assert.equal((await send("PUT", "/99999", { name: "x", kind: "other" })).status, 404);
|
||||
assert.equal((await send("DELETE", `/${nasId}`)).status, 204);
|
||||
assert.equal((await send("DELETE", `/${nasId}`)).status, 404);
|
||||
const actions = (await db.select().from(auditLog)).filter((e) => e.category === "inventory").map((e) => e.action);
|
||||
for (const a of ["create", "update", "delete", "update_config"]) assert.ok(actions.includes(a), a);
|
||||
});
|
||||
|
||||
test("a warranty about to end is in the weekly digest, with a date", async () => {
|
||||
await db.update(servers).set({ lastSeenAt: new Date().toISOString() }).where(eq(servers.id, elsa));
|
||||
const digest = await buildDigest(new Date());
|
||||
assert.match(digest.message, /EXPIRING\n- Warranty on elsa's box ends in (19|20|21) days \(\d{4}-\d{2}-\d{2}\)/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,322 @@
|
||||
/**
|
||||
* The Prometheus endpoint and its tokens, and the signed event webhooks.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import http from "node:http";
|
||||
import { createHmac } from "node:crypto";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-metrics-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, secrets, maintenanceWindows, metricsTokens, eventWebhooks, auditLog } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { metricsRouter, metricsTokensRouter } = await import("../src/routes/metrics.js");
|
||||
const { eventWebhooksRouter } = await import("../src/routes/eventWebhooks.js");
|
||||
const { escapeLabel } = await import("../src/services/metrics.js");
|
||||
const { deliver, emitEvent, signBody } = await import("../src/services/eventWebhooks.js");
|
||||
const { checkMaintenanceEvents, resetMaintenanceEventsForTests } = await import("../src/services/maintenanceEvents.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
describe("escaping", () => {
|
||||
test("a label value keeps its quotes, backslashes and new lines from breaking the line", () => {
|
||||
assert.equal(escapeLabel('say "hi"'), 'say \\"hi\\"');
|
||||
assert.equal(escapeLabel("C:\\temp"), "C:\\\\temp");
|
||||
assert.equal(escapeLabel("a\nb"), "a\\nb");
|
||||
});
|
||||
});
|
||||
|
||||
describe("metrics and webhooks", () => {
|
||||
let app: http.Server;
|
||||
let base = "";
|
||||
let who = "admin";
|
||||
let receiver: http.Server;
|
||||
const received: { headers: http.IncomingHttpHeaders; body: string }[] = [];
|
||||
let receiverStatus: number[] = []; // statuses to answer with, in turn; 200 once it runs out
|
||||
let receiverUrl = "";
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
||||
{ oidcSub: "operator", name: "Olga", role: "operator" },
|
||||
]);
|
||||
const now = new Date();
|
||||
await db.insert(servers).values([
|
||||
{
|
||||
name: 'elsa "main"',
|
||||
apiTokenHash: "a",
|
||||
apiTokenPrefix: "a",
|
||||
lastSeenAt: now.toISOString(),
|
||||
cpuLoadPercent: 12.5,
|
||||
memTotalBytes: 1000,
|
||||
memUsedBytes: 250,
|
||||
disks: JSON.stringify([{ mount: "C:\\", sizeBytes: 100, usedBytes: 40 }]),
|
||||
pendingUpdates: JSON.stringify({ manager: "apt", count: 7, security: 2, rebootRequired: true, reportedAt: now.toISOString() }),
|
||||
agentVersion: 1,
|
||||
},
|
||||
{ name: "anna", apiTokenHash: "b", apiTokenPrefix: "b", lastSeenAt: new Date(now.getTime() - 5 * 3600_000).toISOString() },
|
||||
{ name: "never", apiTokenHash: "c", apiTokenPrefix: "c" },
|
||||
]);
|
||||
await db.insert(secrets).values({ name: "VPN key", type: "api_token", expiryDate: new Date(now.getTime() + 9 * 86_400_000).toISOString().slice(0, 10), warnDays: 30 });
|
||||
|
||||
receiver = await new Promise<http.Server>((resolve) => {
|
||||
const s = http.createServer((req, res) => {
|
||||
let raw = "";
|
||||
req.on("data", (c) => (raw += c));
|
||||
req.on("end", () => {
|
||||
received.push({ headers: req.headers, body: raw });
|
||||
res.statusCode = receiverStatus.shift() ?? 200;
|
||||
res.end("ok");
|
||||
});
|
||||
});
|
||||
s.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
receiverUrl = `http://127.0.0.1:${(receiver.address() as AddressInfo).port}/hook`;
|
||||
|
||||
const a = express();
|
||||
a.use("/metrics", metricsRouter); // no session: it is the bearer token that counts
|
||||
const api = express();
|
||||
api.use(express.json());
|
||||
api.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
api.use("/api/metrics-tokens", metricsTokensRouter);
|
||||
api.use("/api/event-webhooks", eventWebhooksRouter);
|
||||
a.use(api);
|
||||
app = await new Promise<http.Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}`;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
app.close();
|
||||
receiver.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const send = (method: string, path: string, body?: unknown) =>
|
||||
fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
|
||||
let secret = "";
|
||||
let tokenId = 0;
|
||||
|
||||
test("an administrator makes a token; the secret is shown once and only its hash is kept", async () => {
|
||||
const res = await send("POST", "/api/metrics-tokens", { label: "Prometheus" });
|
||||
assert.equal(res.status, 201);
|
||||
const body = await res.json();
|
||||
secret = body.secret;
|
||||
tokenId = body.token.id;
|
||||
assert.match(secret, /^hlm_m_/);
|
||||
const list = await (await send("GET", "/api/metrics-tokens")).json();
|
||||
assert.ok(!JSON.stringify(list).includes(secret) && !JSON.stringify(list).includes("tokenHash"));
|
||||
const [stored] = await db.select().from(metricsTokens);
|
||||
assert.notEqual(stored.tokenHash, secret);
|
||||
assert.equal(stored.tokenHash.length, 64);
|
||||
who = "operator";
|
||||
assert.equal((await send("POST", "/api/metrics-tokens", { label: "x" })).status, 403);
|
||||
assert.equal((await send("GET", "/api/metrics-tokens")).status, 403);
|
||||
who = "admin";
|
||||
assert.equal((await send("POST", "/api/metrics-tokens", { label: "" })).status, 400);
|
||||
});
|
||||
|
||||
test("/metrics wants the token in a header: nothing, a wrong one, or one in the address all get the same refusal", async () => {
|
||||
const none = await fetch(`${base}/metrics`);
|
||||
assert.equal(none.status, 401);
|
||||
assert.match(none.headers.get("www-authenticate") ?? "", /Bearer/);
|
||||
const refusal = await none.text();
|
||||
assert.equal((await fetch(`${base}/metrics`, { headers: { authorization: "Bearer hlm_m_wrongwrongwrongwrongwrong" } })).status, 401);
|
||||
assert.equal((await fetch(`${base}/metrics?token=${secret}`)).status, 401, "a token in the address is not accepted");
|
||||
assert.equal(await (await fetch(`${base}/metrics`, { headers: { authorization: "Bearer nope" } })).text(), refusal);
|
||||
});
|
||||
|
||||
test("with the token it answers in Prometheus's format, with the servers, alerts and the rest", async () => {
|
||||
const res = await fetch(`${base}/metrics`, { headers: { authorization: `Bearer ${secret}` } });
|
||||
assert.equal(res.status, 200);
|
||||
assert.match(res.headers.get("content-type") ?? "", /^text\/plain; version=0\.0\.4/);
|
||||
assert.equal(res.headers.get("cache-control"), "no-store");
|
||||
const text = await res.text();
|
||||
|
||||
assert.match(text, /# TYPE homelab_server_up gauge/);
|
||||
assert.match(text, /homelab_server_up\{server="elsa \\"main\\""\} 1/, "the name's quotes are escaped");
|
||||
assert.match(text, /homelab_server_up\{server="anna"\} 0/, "five hours silent is past the offline threshold");
|
||||
assert.ok(!text.includes('server="never"'), "a server that never reported has nothing to say");
|
||||
assert.match(text, /homelab_server_cpu_load_percent\{server="elsa \\"main\\""\} 12\.5/);
|
||||
assert.match(text, /homelab_server_memory_used_percent\{server="elsa \\"main\\""\} 25/);
|
||||
assert.match(text, /homelab_server_disk_used_percent\{server="elsa \\"main\\"",mount="C:\\\\"\} 40/, "a backslash in a mount is escaped");
|
||||
assert.match(text, /homelab_server_pending_updates\{server="elsa \\"main\\""\} 7/);
|
||||
assert.match(text, /homelab_server_pending_security_updates\{server="elsa \\"main\\""\} 2/);
|
||||
assert.match(text, /homelab_server_reboot_required\{server="elsa \\"main\\""\} 1/);
|
||||
assert.match(text, /homelab_server_agent_outdated\{server="elsa \\"main\\""\} 1/, "version 1 is behind");
|
||||
assert.match(text, /homelab_alerts\{severity="critical"\} \d+/);
|
||||
assert.match(text, /homelab_secret_expiry_days\{name="VPN key"\} 9/);
|
||||
assert.match(text, /homelab_maintenance_active_windows 0/);
|
||||
assert.match(text, /homelab_scrape_duration_seconds /);
|
||||
});
|
||||
|
||||
test("every line is valid exposition format, and each metric is described once", async () => {
|
||||
const text = await (await fetch(`${base}/metrics`, { headers: { authorization: `Bearer ${secret}` } })).text();
|
||||
const helps: string[] = [];
|
||||
for (const line of text.trim().split("\n")) {
|
||||
if (line.startsWith("# HELP ")) {
|
||||
helps.push(line.split(" ")[2]);
|
||||
continue;
|
||||
}
|
||||
if (line.startsWith("# TYPE ")) continue;
|
||||
assert.match(line, /^[a-zA-Z_:][a-zA-Z0-9_:]*(\{([a-zA-Z_][a-zA-Z0-9_]*="([^"\\]|\\.)*",?)*\})? -?\d+(\.\d+)?$/, line);
|
||||
}
|
||||
assert.equal(new Set(helps).size, helps.length, "no metric is declared twice");
|
||||
});
|
||||
|
||||
test("use is noted, and a removed token stops working at once", async () => {
|
||||
const [row] = await db.select().from(metricsTokens).where(eq(metricsTokens.id, tokenId));
|
||||
assert.ok(row.lastUsedAt);
|
||||
assert.equal((await send("DELETE", `/api/metrics-tokens/${tokenId}`)).status, 204);
|
||||
assert.equal((await fetch(`${base}/metrics`, { headers: { authorization: `Bearer ${secret}` } })).status, 401);
|
||||
assert.equal((await send("DELETE", `/api/metrics-tokens/${tokenId}`)).status, 404);
|
||||
const entries = (await db.select().from(auditLog)).filter((e) => e.category === "metrics_token");
|
||||
assert.deepEqual(entries.map((e) => e.action), ["create", "delete"]);
|
||||
assert.ok(entries.every((e) => !(e.detail ?? "").includes(secret)));
|
||||
});
|
||||
|
||||
// ── event webhooks ──
|
||||
|
||||
let hookId = 0;
|
||||
const HOOK_SECRET = "s3cret-for-signing";
|
||||
const waitFor = async (n: number, ms = 3000) => {
|
||||
for (let t = 0; t < ms && received.length < n; t += 25) await new Promise((r) => setTimeout(r, 25));
|
||||
};
|
||||
|
||||
test("a webhook is made with a secret that is never shown again, and kept encrypted", async () => {
|
||||
const res = await send("POST", "/api/event-webhooks", { label: "Home Assistant", url: receiverUrl, secret: HOOK_SECRET, events: ["alert.opened", "maintenance.started"] });
|
||||
assert.equal(res.status, 201);
|
||||
const hook = (await res.json()).webhook;
|
||||
hookId = hook.id;
|
||||
assert.equal(hook.hasSecret, true);
|
||||
assert.deepEqual(hook.events, ["alert.opened", "maintenance.started"]);
|
||||
assert.ok(!JSON.stringify(await (await send("GET", "/api/event-webhooks")).json()).includes(HOOK_SECRET));
|
||||
const [stored] = await db.select().from(eventWebhooks).where(eq(eventWebhooks.id, hookId));
|
||||
assert.match(stored.secret ?? "", /^enc:v1:/);
|
||||
const audit = (await db.select().from(auditLog)).find((e) => e.category === "webhook" && e.action === "create");
|
||||
assert.ok(!(audit?.detail ?? "").includes(receiverUrl) && !(audit?.detail ?? "").includes(HOOK_SECRET), "neither the address nor the secret goes in the audit log");
|
||||
});
|
||||
|
||||
test("a bad webhook is refused: not http, a password in the address, no events, only admins", async () => {
|
||||
for (const bad of [
|
||||
{ label: "x", url: "ftp://x.example/hook", events: ["alert.opened"] },
|
||||
{ label: "x", url: "https://user:pass@x.example/hook", events: ["alert.opened"] },
|
||||
{ label: "x", url: "not a url", events: ["alert.opened"] },
|
||||
{ label: "x", url: receiverUrl, events: [] },
|
||||
{ label: "x", url: receiverUrl, events: ["alert.exploded"] },
|
||||
{ label: "", url: receiverUrl, events: ["alert.opened"] },
|
||||
]) {
|
||||
assert.equal((await send("POST", "/api/event-webhooks", bad)).status, 400, JSON.stringify(bad));
|
||||
}
|
||||
who = "operator";
|
||||
assert.equal((await send("GET", "/api/event-webhooks")).status, 403);
|
||||
who = "admin";
|
||||
});
|
||||
|
||||
test("an event goes to the webhooks that want it, signed so the receiver can check it", async () => {
|
||||
received.length = 0;
|
||||
emitEvent("alert.opened", { alert: { key: "offline:server:1", severity: "critical", message: "elsa hasn't reported" } });
|
||||
emitEvent("alert.resolved", { alert: { key: "x", message: "not wanted by this webhook" } });
|
||||
await waitFor(1);
|
||||
await new Promise((r) => setTimeout(r, 150));
|
||||
assert.equal(received.length, 1, "only the event it asked for");
|
||||
const got = received[0];
|
||||
assert.equal(got.headers["x-hlm-event"], "alert.opened");
|
||||
assert.equal(got.headers["x-hlm-signature"], `sha256=${createHmac("sha256", HOOK_SECRET).update(got.body).digest("hex")}`);
|
||||
assert.equal(got.headers["x-hlm-signature"], signBody(HOOK_SECRET, got.body));
|
||||
const body = JSON.parse(got.body);
|
||||
assert.equal(body.event, "alert.opened");
|
||||
assert.equal(body.source, "homelab-manager");
|
||||
assert.equal(body.data.alert.message, "elsa hasn't reported");
|
||||
assert.equal(body.id, got.headers["x-hlm-delivery"]);
|
||||
const [row] = await db.select().from(eventWebhooks).where(eq(eventWebhooks.id, hookId));
|
||||
assert.equal(row.lastResult, "200");
|
||||
assert.ok(row.lastAttemptAt);
|
||||
});
|
||||
|
||||
test("a switched-off webhook hears nothing", async () => {
|
||||
assert.equal((await send("PATCH", `/api/event-webhooks/${hookId}`, { enabled: false })).status, 200);
|
||||
received.length = 0;
|
||||
emitEvent("alert.opened", { alert: { message: "x" } });
|
||||
await new Promise((r) => setTimeout(r, 300));
|
||||
assert.equal(received.length, 0);
|
||||
await send("PATCH", `/api/event-webhooks/${hookId}`, { enabled: true });
|
||||
});
|
||||
|
||||
test("a delivery that fails is tried once more, and what happened is noted", async () => {
|
||||
const [row] = await db.select().from(eventWebhooks).where(eq(eventWebhooks.id, hookId));
|
||||
received.length = 0;
|
||||
receiverStatus = [500, 200];
|
||||
const ok = await deliver(row, "alert.opened", { n: 1 }, { retryAfterMs: 10 });
|
||||
assert.deepEqual(ok, { ok: true, result: "200" });
|
||||
assert.equal(received.length, 2, "tried twice");
|
||||
assert.equal(received[0].headers["x-hlm-delivery"], received[1].headers["x-hlm-delivery"], "the same message both times");
|
||||
|
||||
receiverStatus = [503, 503];
|
||||
const bad = await deliver(row, "alert.opened", { n: 2 }, { retryAfterMs: 10 });
|
||||
assert.deepEqual(bad, { ok: false, result: "503" });
|
||||
assert.equal((await db.select().from(eventWebhooks).where(eq(eventWebhooks.id, hookId)))[0].lastResult, "503");
|
||||
|
||||
const dead = await deliver({ ...row, url: "http://127.0.0.1:1/nothing" }, "alert.opened", {}, { retryAfterMs: 10 });
|
||||
assert.equal(dead.ok, false);
|
||||
});
|
||||
|
||||
test("a test message goes out whatever the webhook is set to, and says how it went", async () => {
|
||||
received.length = 0;
|
||||
receiverStatus = [];
|
||||
const res = await send("POST", `/api/event-webhooks/${hookId}/test`);
|
||||
assert.deepEqual(await res.json(), { ok: true, result: "200" });
|
||||
assert.equal(received[0].headers["x-hlm-event"], "test");
|
||||
assert.equal((await send("POST", "/api/event-webhooks/99999/test")).status, 404);
|
||||
});
|
||||
|
||||
test("maintenance beginning and ending is noticed, however it came about, and a first look says nothing", async () => {
|
||||
resetMaintenanceEventsForTests();
|
||||
const [elsa] = await db.select().from(servers).where(eq(servers.name, 'elsa "main"'));
|
||||
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: elsa.id, reason: "before", startedAt: new Date(Date.now() - 60_000).toISOString(), endsAt: new Date(Date.now() + 3600_000).toISOString(), createdBy: "Ada" });
|
||||
assert.deepEqual(await checkMaintenanceEvents(), { started: [], ended: [] }, "already running when first looked at: not announced");
|
||||
|
||||
const [w] = await db.insert(maintenanceWindows).values({ targetType: "server", targetId: elsa.id, reason: "patching", startedAt: new Date(Date.now() - 1000).toISOString(), endsAt: new Date(Date.now() + 3600_000).toISOString(), createdBy: "Ada", source: "schedule:1:2026-10-17" }).returning();
|
||||
received.length = 0;
|
||||
const started = await checkMaintenanceEvents();
|
||||
assert.deepEqual(started, { started: [w.id], ended: [] });
|
||||
await waitFor(1);
|
||||
assert.equal(received[0].headers["x-hlm-event"], "maintenance.started");
|
||||
const payload = JSON.parse(received[0].body).data.window;
|
||||
assert.equal(payload.targetName, 'elsa "main"');
|
||||
assert.equal(payload.reason, "patching");
|
||||
assert.equal(payload.source, "schedule:1:2026-10-17");
|
||||
|
||||
// ended early
|
||||
await db.update(maintenanceWindows).set({ endsAt: new Date(Date.now() - 500).toISOString() }).where(eq(maintenanceWindows.id, w.id));
|
||||
assert.deepEqual(await checkMaintenanceEvents(), { started: [], ended: [w.id] });
|
||||
assert.deepEqual(await checkMaintenanceEvents(), { started: [], ended: [] }, "and only once");
|
||||
});
|
||||
|
||||
test("a webhook can be removed", async () => {
|
||||
assert.equal((await send("DELETE", `/api/event-webhooks/${hookId}`)).status, 204);
|
||||
assert.equal((await send("DELETE", `/api/event-webhooks/${hookId}`)).status, 404);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,144 @@
|
||||
/**
|
||||
* Runbooks: notes about what to do when something is wrong, attached to a server, integration or DNS provider.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-runbooks-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, integrations, auditLog } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { runbooksRouter } = await import("../src/routes/runbooks.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
describe("runbooks", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "operator";
|
||||
let elsa = 0;
|
||||
let proxmox = 0;
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "operator", name: "Olga", role: "operator" },
|
||||
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
||||
]);
|
||||
elsa = (await db.insert(servers).values({ name: "elsa", apiTokenHash: "a", apiTokenPrefix: "a" }).returning())[0].id;
|
||||
proxmox = (await db.insert(integrations).values({ type: "proxmox", name: "Felicia", baseUrl: "https://pve.lab", config: "{}" }).returning())[0].id;
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/runbooks", runbooksRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/runbooks`;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const send = (method: string, path: string, body?: unknown) =>
|
||||
fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
|
||||
let firstId = 0;
|
||||
|
||||
test("an operator writes one for a server, for everything or for one kind of alert", async () => {
|
||||
const any = await send("POST", "/", { targetType: "server", targetId: elsa, title: "If elsa is down", body: "Check the UPS first.\n\n- then the switch" });
|
||||
assert.equal(any.status, 201);
|
||||
const row = (await any.json()).runbook;
|
||||
firstId = row.id;
|
||||
assert.equal(row.category, null);
|
||||
assert.equal(row.updatedBy, "Olga");
|
||||
const disk = await send("POST", "/", { targetType: "server", targetId: elsa, title: "Disk full", body: "Prune docker.", category: "disk" });
|
||||
assert.equal(disk.status, 201);
|
||||
assert.equal((await disk.json()).runbook.category, "disk");
|
||||
assert.equal((await send("POST", "/", { targetType: "integration", targetId: proxmox, title: "Proxmox", body: "Restart pveproxy." })).status, 201);
|
||||
});
|
||||
|
||||
test("anyone signed in can read them, by thing, or as a list of titles", async () => {
|
||||
who = "viewer";
|
||||
const mine = await (await send("GET", `/?targetType=server&targetId=${elsa}`)).json();
|
||||
assert.deepEqual(mine.runbooks.map((r: { title: string }) => r.title), ["Disk full", "If elsa is down"]);
|
||||
assert.equal(mine.runbooks[1].body, "Check the UPS first.\n\n- then the switch");
|
||||
|
||||
const summary = await (await send("GET", "/summary")).json();
|
||||
assert.equal(summary.runbooks.length, 3);
|
||||
assert.ok(summary.runbooks.every((r: Record<string, unknown>) => !("body" in r)), "titles only: the texts are fetched for the one being read");
|
||||
assert.equal((await send("GET", "/?targetType=server")).status, 400);
|
||||
assert.equal((await send("GET", "/?targetType=moon&targetId=1")).status, 400);
|
||||
who = "operator";
|
||||
});
|
||||
|
||||
test("only operators change them", async () => {
|
||||
who = "viewer";
|
||||
assert.equal((await send("POST", "/", { targetType: "server", targetId: elsa, title: "x", body: "y" })).status, 403);
|
||||
assert.equal((await send("PUT", `/${firstId}`, { title: "x", body: "y" })).status, 403);
|
||||
assert.equal((await send("DELETE", `/${firstId}`)).status, 403);
|
||||
who = "operator";
|
||||
});
|
||||
|
||||
test("a bad one is refused: no such thing, no title, a made-up kind of alert, too long", async () => {
|
||||
for (const bad of [
|
||||
{ targetType: "server", targetId: 99999, title: "x", body: "y" },
|
||||
{ targetType: "server", targetId: elsa, title: " ", body: "y" },
|
||||
{ targetType: "server", targetId: elsa, title: "x", body: "y", category: "weather" },
|
||||
{ targetType: "server", targetId: elsa, title: "x", body: "z".repeat(20_001) },
|
||||
{ targetType: "toaster", targetId: 1, title: "x", body: "y" },
|
||||
]) {
|
||||
assert.equal((await send("POST", "/", bad)).status, 400, JSON.stringify(bad).slice(0, 70));
|
||||
}
|
||||
});
|
||||
|
||||
test("editing keeps who changed it, and it can be removed; both are in the audit log", async () => {
|
||||
const put = await send("PUT", `/${firstId}`, { title: "If elsa is down (updated)", body: "UPS, then switch, then call Bo.", category: "offline" });
|
||||
assert.equal(put.status, 200);
|
||||
const row = (await put.json()).runbook;
|
||||
assert.equal(row.title, "If elsa is down (updated)");
|
||||
assert.equal(row.category, "offline");
|
||||
assert.ok(row.updatedAt >= row.createdAt);
|
||||
assert.equal((await send("PUT", "/99999", { title: "x", body: "y" })).status, 404);
|
||||
|
||||
assert.equal((await send("DELETE", `/${firstId}`)).status, 204);
|
||||
assert.equal((await send("DELETE", `/${firstId}`)).status, 404);
|
||||
const actions = (await db.select().from(auditLog)).filter((e) => e.category === "runbook").map((e) => e.action);
|
||||
for (const a of ["create", "update", "delete"]) assert.ok(actions.includes(a), a);
|
||||
});
|
||||
|
||||
test("notes for something that has since been removed drop out of the list", async () => {
|
||||
await db.delete(integrations).where(eq(integrations.id, proxmox));
|
||||
const summary = await (await send("GET", "/summary")).json();
|
||||
assert.equal(summary.runbooks.length, 1, "only the remaining server note");
|
||||
});
|
||||
|
||||
test("there is a limit to how many there can be for one thing", async () => {
|
||||
const [{ count }] = [{ count: (await (await send("GET", `/?targetType=server&targetId=${elsa}`)).json()).runbooks.length }];
|
||||
for (let i = count; i < 20; i++) assert.equal((await send("POST", "/", { targetType: "server", targetId: elsa, title: `Note ${i}`, body: "x" })).status, 201);
|
||||
const over = await send("POST", "/", { targetType: "server", targetId: elsa, title: "One too many", body: "x" });
|
||||
assert.equal(over.status, 400);
|
||||
assert.equal((await over.json()).error, "too_many");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,188 @@
|
||||
/**
|
||||
* Roles from sign-in groups: reading the claim, the rules, what happens to people at sign-in, and the Users routes around it.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-groups-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { extractGroups, roleFromGroups } = await import("../src/auth/groups.js");
|
||||
const { upsertUserFromLogin } = await import("../src/auth/users.js");
|
||||
const { usersRouter } = await import("../src/routes/users.js");
|
||||
const { updateSettings } = await import("../src/services/settingsStore.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const rules = { roleFromGroups: true, groupsClaim: "groups", adminGroups: ["homelab-admins"], operatorGroups: ["homelab-ops"], otherwise: "viewer" as const };
|
||||
|
||||
describe("reading the groups out of a sign-in", () => {
|
||||
test("from any place the provider put them, as a list or a single name", () => {
|
||||
assert.deepEqual(extractGroups([{ groups: ["a", "b"] }], "groups"), ["a", "b"]);
|
||||
assert.deepEqual(extractGroups([{ groups: "solo" }], "groups"), ["solo"]);
|
||||
assert.deepEqual(extractGroups([{ sub: "x" }, { groups: ["u"] }], "groups"), ["u"], "the user-info answer counts too");
|
||||
assert.deepEqual(extractGroups([{ groups: ["a", "b"] }, { groups: ["b", "c"] }], "groups"), ["a", "b", "c"], "joined, without repeats");
|
||||
assert.deepEqual(extractGroups([{ roles: ["x"] }], "roles"), ["x"], "a claim of another name");
|
||||
assert.deepEqual(extractGroups([{ groups: [" padded ", "", 5, null, "ok"] }], "groups"), ["padded", "ok"], "only real names");
|
||||
});
|
||||
|
||||
test("not being told is not the same as being in no groups", () => {
|
||||
assert.equal(extractGroups([{ sub: "x" }], "groups"), null);
|
||||
assert.equal(extractGroups([{ groups: null }], "groups"), null);
|
||||
assert.deepEqual(extractGroups([{ groups: [] }], "groups"), [], "an empty list is an answer");
|
||||
assert.equal(extractGroups([], "groups"), null);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the rules", () => {
|
||||
test("admin beats operator, names ignore case and spaces, and the rest get what 'otherwise' says", () => {
|
||||
assert.equal(roleFromGroups(rules, ["homelab-ops", "homelab-admins"], "viewer"), "admin");
|
||||
assert.equal(roleFromGroups(rules, ["HomeLab-Ops "], "viewer"), "operator");
|
||||
assert.equal(roleFromGroups(rules, ["cooks"], "operator"), "viewer");
|
||||
assert.equal(roleFromGroups(rules, [], "admin"), "viewer");
|
||||
assert.equal(roleFromGroups({ ...rules, otherwise: "keep" }, ["cooks"], "operator"), "operator");
|
||||
assert.equal(roleFromGroups({ ...rules, adminGroups: [], operatorGroups: [] }, ["homelab-admins"], "viewer"), "viewer", "no groups named, no role from them");
|
||||
});
|
||||
});
|
||||
|
||||
describe("at sign-in", () => {
|
||||
const login = (sub: string, groups: string[] | null | undefined) => upsertUserFromLogin({ sub, email: `${sub}@lab`, name: sub, groups });
|
||||
const roleOf = async (sub: string) => (await db.select().from(users).where(eq(users.oidcSub, sub)))[0].role;
|
||||
|
||||
test("with the rules off, groups are remembered but roles stay as they are", async () => {
|
||||
const first = await login("founder", ["homelab-ops"]);
|
||||
assert.equal(first.user.role, "admin", "the first person ever is the administrator");
|
||||
assert.deepEqual(JSON.parse(first.user.groups!), ["homelab-ops"]);
|
||||
const second = await login("sam", ["homelab-admins"]);
|
||||
assert.equal(second.user.role, "viewer");
|
||||
assert.equal(second.roleChange, undefined);
|
||||
assert.equal((await login("sam", ["homelab-admins"])).user.role, "viewer", "and again on the next sign-in");
|
||||
});
|
||||
|
||||
test("with the rules on, a new person's groups decide, and the first person is still the administrator", async () => {
|
||||
await updateSettings({ signIn: rules });
|
||||
assert.equal((await login("olga", ["homelab-ops"])).user.role, "operator");
|
||||
assert.equal((await login("nobody", ["cooks"])).user.role, "viewer");
|
||||
assert.equal((await login("quiet", [])).user.role, "viewer");
|
||||
assert.equal((await login("unknown", null)).user.role, "viewer", "no groups reported: viewer, as before");
|
||||
});
|
||||
|
||||
test("an existing person's role follows their groups, and says what changed", async () => {
|
||||
const up = await login("sam", ["homelab-ops"]);
|
||||
assert.equal(up.user.role, "operator");
|
||||
assert.deepEqual(up.roleChange, { from: "viewer", to: "operator", groups: ["homelab-ops"] });
|
||||
const again = await login("sam", ["homelab-ops"]);
|
||||
assert.equal(again.roleChange, undefined, "nothing to say when nothing changed");
|
||||
const down = await login("sam", ["cooks"]);
|
||||
assert.equal(down.user.role, "viewer");
|
||||
assert.equal(down.roleChange?.from, "operator");
|
||||
});
|
||||
|
||||
test("a sign-in that carries no groups leaves roles alone, so a wrong claim name can't demote anyone", async () => {
|
||||
await login("sam", ["homelab-admins"]);
|
||||
assert.equal(await roleOf("sam"), "admin");
|
||||
const blind = await login("sam", null);
|
||||
assert.equal(blind.roleChange, undefined);
|
||||
assert.equal(await roleOf("sam"), "admin");
|
||||
assert.equal(await roleOf("founder"), "admin");
|
||||
await login("founder", undefined);
|
||||
assert.equal(await roleOf("founder"), "admin");
|
||||
});
|
||||
|
||||
test("the last administrator is never demoted by a group rule", async () => {
|
||||
// sam and founder are both admins now; demoting one is allowed…
|
||||
const d1 = await login("sam", ["cooks"]);
|
||||
assert.equal(d1.user.role, "viewer");
|
||||
// …but not the other, who is the last.
|
||||
const d2 = await login("founder", ["cooks"]);
|
||||
assert.equal(d2.user.role, "admin");
|
||||
assert.equal(d2.roleChange?.blocked, true);
|
||||
assert.equal(await roleOf("founder"), "admin");
|
||||
});
|
||||
|
||||
test("'keep' leaves people in no named group as they are", async () => {
|
||||
await updateSettings({ signIn: { ...rules, otherwise: "keep" } });
|
||||
await login("olga", ["cooks"]);
|
||||
assert.equal(await roleOf("olga"), "operator");
|
||||
await updateSettings({ signIn: { ...rules, otherwise: "viewer" } });
|
||||
});
|
||||
});
|
||||
|
||||
describe("the Users routes", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
|
||||
before(async () => {
|
||||
await db.update(users).set({ role: "admin" }).where(eq(users.oidcSub, "founder"));
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: "founder" } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/users", usersRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/users`;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const send = (method: string, path: string, body?: unknown) =>
|
||||
fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
|
||||
test("the list carries each person's groups and says whether roles follow them; nothing else leaks", async () => {
|
||||
const body = await (await send("GET", "/")).json();
|
||||
assert.equal(body.rolesFollowGroups, true);
|
||||
const olga = body.users.find((u: { name: string }) => u.name === "olga");
|
||||
assert.deepEqual(olga.groups, ["cooks"]);
|
||||
assert.deepEqual(Object.keys(olga).sort(), ["createdAt", "email", "groups", "id", "lastLoginAt", "name", "oidcSub", "role"]);
|
||||
});
|
||||
|
||||
test("a role can't be set by hand while groups decide it", async () => {
|
||||
const olga = (await db.select().from(users).where(eq(users.oidcSub, "olga")))[0];
|
||||
const res = await send("PATCH", `/${olga.id}/role`, { role: "viewer" });
|
||||
assert.equal(res.status, 409);
|
||||
assert.equal((await res.json()).error, "managed_by_groups");
|
||||
await updateSettings({ signIn: { roleFromGroups: false } });
|
||||
assert.equal((await send("PATCH", `/${olga.id}/role`, { role: "viewer" })).status, 200, "and it can again once they don't");
|
||||
});
|
||||
|
||||
test("the preview says what each person would be, and warns when nobody would be an administrator", async () => {
|
||||
await db.update(users).set({ groups: JSON.stringify(["homelab-admins"]) }).where(eq(users.oidcSub, "founder"));
|
||||
const ok = await (await send("POST", "/preview-group-roles", { adminGroups: ["homelab-admins"] })).json();
|
||||
assert.equal(ok.people.find((p: { name: string }) => p.name === "founder").wouldBe, "admin");
|
||||
assert.equal(ok.adminsAfter, 1);
|
||||
|
||||
const locked = await (await send("POST", "/preview-group-roles", { adminGroups: ["a-group-nobody-is-in"] })).json();
|
||||
assert.equal(locked.adminsAfter, 0, "that rule would leave no administrator");
|
||||
assert.equal(locked.people.find((p: { name: string }) => p.name === "founder").wouldBe, "viewer");
|
||||
|
||||
const never = locked.people.find((p: { name: string }) => p.name === "unknown");
|
||||
assert.equal(never.groups, null);
|
||||
assert.equal(never.wouldBe, null, "someone who has never signed in with groups can't be told");
|
||||
assert.equal((await send("POST", "/preview-group-roles", { adminGroups: "nope" })).status, 400);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user