A UPS in the map: read over NUT or SNMP, alerts, shutdown order
New UPS integration (NUT or SNMP v2c, read only). Monitoring -> UPS shows charge, runtime, load and mains voltage with history and the times mains power was lost. Mains lost and a nearly empty battery are critical alerts; battery to be replaced and a heavy load are warnings. The UPS is a node in the dependency map and counts as down while on battery, so servers that go offline with it are folded under its alert; the page lists what is on it in the order to shut it down. /metrics series and docs added. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3cb3e4988f
commit
232a5661ab
38 files changed
+8049
-16
No files matched your search
@@ -0,0 +1,31 @@
|
||||
CREATE TABLE `ups_readings` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`integration_id` integer NOT NULL,
|
||||
`at` text NOT NULL,
|
||||
`charge` real,
|
||||
`runtime_seconds` integer,
|
||||
`load_percent` real,
|
||||
`input_voltage` real,
|
||||
`on_battery` integer DEFAULT false NOT NULL,
|
||||
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX `ups_readings_lookup_idx` ON `ups_readings` (`integration_id`,`at`);--> statement-breakpoint
|
||||
CREATE TABLE `ups_state` (
|
||||
`integration_id` integer PRIMARY KEY NOT NULL,
|
||||
`model` text,
|
||||
`manufacturer` text,
|
||||
`status_text` text NOT NULL,
|
||||
`on_battery` integer DEFAULT false NOT NULL,
|
||||
`low_battery` integer DEFAULT false NOT NULL,
|
||||
`replace_battery` integer DEFAULT false NOT NULL,
|
||||
`overload` integer DEFAULT false NOT NULL,
|
||||
`charge` real,
|
||||
`runtime_seconds` integer,
|
||||
`load_percent` real,
|
||||
`input_voltage` real,
|
||||
`output_voltage` real,
|
||||
`on_battery_since` text,
|
||||
`updated_at` text NOT NULL,
|
||||
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
File diff suppressed because it is too large.
Load diff
@@ -379,6 +379,13 @@
|
||||
"when": 1792609540216,
|
||||
"tag": "0053_dashing_stark_industries",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 54,
|
||||
"version": "6",
|
||||
"when": 1792610229172,
|
||||
"tag": "0054_green_beast",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -55,6 +55,7 @@ import { storageRouter } from "./routes/storage.js";
|
||||
import { dataUsageRouter } from "./routes/dataUsage.js";
|
||||
import { alertActionsRouter } from "./routes/alertActions.js";
|
||||
import { homeAssistantRouter } from "./routes/homeAssistant.js";
|
||||
import { upsRouter } from "./routes/ups.js";
|
||||
import { publicApiRouter } from "./routes/publicApi.js";
|
||||
import { generatorRouter } from "./routes/generator.js";
|
||||
import { alertTicketsRouter } from "./routes/alertTickets.js";
|
||||
@@ -184,6 +185,7 @@ export function createApp() {
|
||||
app.use("/api/storage", storageRouter);
|
||||
app.use("/api/data-usage", dataUsageRouter);
|
||||
app.use("/api/home-assistant", homeAssistantRouter);
|
||||
app.use("/api/ups", upsRouter);
|
||||
app.use("/api/todos", todosRouter);
|
||||
app.use("/api/notes", notesRouter);
|
||||
app.use("/api/generator", generatorRouter);
|
||||
|
||||
@@ -552,6 +552,7 @@ export const integrationTypes = [
|
||||
"unifi",
|
||||
"npm",
|
||||
"homeassistant",
|
||||
"ups",
|
||||
] as const;
|
||||
export type IntegrationType = (typeof integrationTypes)[number];
|
||||
|
||||
@@ -1158,6 +1159,48 @@ export const haReadings = sqliteTable(
|
||||
(t) => [index("ha_readings_lookup_idx").on(t.integrationId, t.entityId, t.at)],
|
||||
);
|
||||
|
||||
// ─── UPS ────────────────────────────────────────────────────────────────────
|
||||
|
||||
// How a UPS (an integration of type "ups", read over NUT or SNMP by services/upsSync.ts) is right now: one row each, replaced at every read. `updatedAt` is the last
|
||||
// read that worked; `onBatterySince` is when mains power was lost, kept until it is back.
|
||||
export const upsState = sqliteTable("ups_state", {
|
||||
integrationId: integer("integration_id")
|
||||
.primaryKey()
|
||||
.references(() => integrations.id, { onDelete: "cascade" }),
|
||||
model: text("model"),
|
||||
manufacturer: text("manufacturer"),
|
||||
statusText: text("status_text").notNull(),
|
||||
onBattery: integer("on_battery", { mode: "boolean" }).notNull().default(false),
|
||||
lowBattery: integer("low_battery", { mode: "boolean" }).notNull().default(false),
|
||||
replaceBattery: integer("replace_battery", { mode: "boolean" }).notNull().default(false),
|
||||
overload: integer("overload", { mode: "boolean" }).notNull().default(false),
|
||||
charge: real("charge"),
|
||||
runtimeSeconds: integer("runtime_seconds"),
|
||||
loadPercent: real("load_percent"),
|
||||
inputVoltage: real("input_voltage"),
|
||||
outputVoltage: real("output_voltage"),
|
||||
onBatterySince: text("on_battery_since"), // ISO
|
||||
updatedAt: text("updated_at").notNull(), // ISO
|
||||
});
|
||||
|
||||
// A UPS at a moment, for the charts: one every five minutes, and every minute while it runs on battery. Kept 35 days.
|
||||
export const upsReadings = sqliteTable(
|
||||
"ups_readings",
|
||||
{
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
integrationId: integer("integration_id")
|
||||
.notNull()
|
||||
.references(() => integrations.id, { onDelete: "cascade" }),
|
||||
at: text("at").notNull(), // ISO
|
||||
charge: real("charge"),
|
||||
runtimeSeconds: integer("runtime_seconds"),
|
||||
loadPercent: real("load_percent"),
|
||||
inputVoltage: real("input_voltage"),
|
||||
onBattery: integer("on_battery", { mode: "boolean" }).notNull().default(false),
|
||||
},
|
||||
(t) => [index("ups_readings_lookup_idx").on(t.integrationId, t.at)],
|
||||
);
|
||||
|
||||
// ─── Notes and the to-do list ───────────────────────────────────────────────
|
||||
|
||||
// Free-form notes about a server or a network device — what is it, where does it sit, what did I do to it last time — one text per thing, drawn with the same small
|
||||
|
||||
@@ -27,6 +27,7 @@ import { initFlapMonitor } from "./services/flapMonitor.js";
|
||||
import { initSnoozeMonitor } from "./services/snoozeMonitor.js";
|
||||
import { initUnifiSync } from "./services/unifiSync.js";
|
||||
import { initHomeAssistantSync } from "./services/homeAssistantSync.js";
|
||||
import { initUpsSync } from "./services/upsSync.js";
|
||||
import { backfillMetricRollups } from "./services/serverMetrics.js";
|
||||
import { backfillEndpointHourly } from "./services/endpointChecks.js";
|
||||
import { initSelfBackupScheduler } from "./services/selfBackupScheduler.js";
|
||||
@@ -75,6 +76,7 @@ initFlapMonitor();
|
||||
initSnoozeMonitor();
|
||||
initUnifiSync();
|
||||
initHomeAssistantSync();
|
||||
initUpsSync();
|
||||
// The hourly and daily figures are built from what is already kept, once, the first time after they were added (in the background: nothing waits for it).
|
||||
void Promise.all([backfillMetricRollups(), backfillEndpointHourly()]).catch((err) => console.error("[history] couldn't build the hourly and daily figures:", err));
|
||||
|
||||
|
||||
@@ -99,6 +99,13 @@ export const INTEGRATION_FIELDS: Partial<Record<IntegrationType, IntegrationFiel
|
||||
{ key: "token", label: "Long-lived access token", secret: true, type: "password", placeholder: "made on your profile page in Home Assistant" },
|
||||
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
|
||||
],
|
||||
ups: [
|
||||
{ key: "host", label: "Host name or address", secret: false, placeholder: "the machine running NUT (upsd), or the UPS's network card" },
|
||||
{ key: "snmp", label: "Read it over SNMP (version 2c) instead of NUT", secret: false, type: "checkbox" },
|
||||
{ key: "port", label: "Port", secret: false, optional: true, placeholder: "3493 for NUT, 161 for SNMP" },
|
||||
{ key: "upsName", label: "NUT: the UPS's name", secret: false, optional: true, placeholder: "as in ups.conf; the first one listed when blank" },
|
||||
{ key: "community", label: "SNMP: read community", secret: true, optional: true, type: "password", placeholder: "public when blank" },
|
||||
],
|
||||
osticket: [
|
||||
{ key: "host", label: "Database host", secret: false, placeholder: "osticket-db.example.lan" },
|
||||
{ key: "port", label: "Database port", secret: false, optional: true, placeholder: "3306" },
|
||||
@@ -141,6 +148,9 @@ export function resolveBaseUrl(
|
||||
): string {
|
||||
const url = nonSecretFields.url;
|
||||
if (typeof url === "string" && url) return url;
|
||||
// A UPS has no web address; the host it is read from is what the card shows.
|
||||
const host = nonSecretFields.host;
|
||||
if (type === "ups" && typeof host === "string" && host) return host;
|
||||
return INTEGRATION_BASE_URLS[type] ?? "";
|
||||
}
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ import { createOsTicketAdapter } from "./osticket/adapter.js";
|
||||
import { createUnifiAdapter } from "./unifi/adapter.js";
|
||||
import { createNpmAdapter } from "./npm/adapter.js";
|
||||
import { createHomeAssistantAdapter } from "./homeassistant/adapter.js";
|
||||
import { createUpsAdapter } from "./ups/adapter.js";
|
||||
|
||||
export interface PingableAdapter {
|
||||
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
|
||||
@@ -53,6 +54,8 @@ export function createIntegrationAdapter(type: IntegrationType, config: Integrat
|
||||
return createNpmAdapter(config as any);
|
||||
case "homeassistant":
|
||||
return createHomeAssistantAdapter(config as any);
|
||||
case "ups":
|
||||
return createUpsAdapter(config as any);
|
||||
default:
|
||||
throw new Error(`Integration type "${type}" is not implemented yet`);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
/**
|
||||
* UPS adapter. Config: host; optional: port, snmp (read over SNMP instead of NUT), upsName (NUT), community (SNMP).
|
||||
*
|
||||
* A UPS is read, never controlled: NUT's `LIST VAR` or an SNMP GET of the UPS-MIB. What it says is the same either way (integrations/ups/types.ts).
|
||||
*/
|
||||
import { withDiagLogging } from "../../services/diagLog.js";
|
||||
import { NUT_PORT, readNut } from "./nut.js";
|
||||
import { readSnmp, SNMP_PORT } from "./snmp.js";
|
||||
import type { UpsConfig, UpsReading } from "./types.js";
|
||||
|
||||
export interface UpsAdapter {
|
||||
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
|
||||
read(): Promise<UpsReading>;
|
||||
}
|
||||
|
||||
const truthy = (v: unknown) => v === true || v === "true";
|
||||
|
||||
/** The port in the config, or the protocol's own. */
|
||||
export function portOf(config: UpsConfig): number {
|
||||
const asked = Number(config.port);
|
||||
return Number.isInteger(asked) && asked >= 1 && asked <= 65535 ? asked : truthy(config.snmp) ? SNMP_PORT : NUT_PORT;
|
||||
}
|
||||
|
||||
export function createUpsAdapter(config: UpsConfig): UpsAdapter {
|
||||
const host = String(config.host ?? "").trim();
|
||||
async function read(): Promise<UpsReading> {
|
||||
if (!host) throw new Error("no host given");
|
||||
return truthy(config.snmp) ? readSnmp(host, portOf(config), (config.community || "public").toString()) : readNut(host, portOf(config), config.upsName?.toString().trim() || null);
|
||||
}
|
||||
async function ping() {
|
||||
const start = Date.now();
|
||||
try {
|
||||
await read();
|
||||
return { ok: true, latencyMs: Date.now() - start };
|
||||
} catch (err) {
|
||||
return { ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
}
|
||||
}
|
||||
return withDiagLogging("ups", { ping, read });
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
/**
|
||||
* Network UPS Tools: `upsd` answers a short text protocol on TCP port 3493 and needs no login to read. `LIST VAR <ups>` gives every variable the driver has
|
||||
* (`battery.charge`, `ups.status`, …) between a BEGIN and an END line; `LIST UPS` names the UPSs it knows. Only reading is done here — never a command.
|
||||
*/
|
||||
import net from "node:net";
|
||||
import { resolveTarget } from "../../utils/networkGuard.js";
|
||||
import type { UpsReading } from "./types.js";
|
||||
|
||||
export const NUT_PORT = 3493;
|
||||
const TIMEOUT_MS = 5000;
|
||||
|
||||
/** `VAR ups battery.charge "100"` lines into a map; anything else in the text is ignored. */
|
||||
export function parseNutVars(text: string): Record<string, string> {
|
||||
const out: Record<string, string> = {};
|
||||
for (const line of text.split(/\r?\n/)) {
|
||||
const m = /^VAR\s+\S+\s+(\S+)\s+"((?:[^"\\]|\\.)*)"\s*$/.exec(line);
|
||||
if (m) out[m[1]] = m[2].replace(/\\(.)/g, "$1");
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The names in an answer to `LIST UPS`. */
|
||||
export function parseNutUpsNames(text: string): string[] {
|
||||
const out: string[] = [];
|
||||
for (const line of text.split(/\r?\n/)) {
|
||||
const m = /^UPS\s+(\S+)\s/.exec(line);
|
||||
if (m) out.push(m[1]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const num = (v: string | undefined): number | null => {
|
||||
if (v === undefined || v.trim() === "") return null;
|
||||
const n = Number(v);
|
||||
return Number.isFinite(n) ? n : null;
|
||||
};
|
||||
|
||||
/** The variables of one UPS as what the app keeps. `ups.status` is a list of flags: OL online, OB on battery, LB low battery, RB replace battery, OVER overload. */
|
||||
export function interpretNut(vars: Record<string, string>): UpsReading {
|
||||
const status = (vars["ups.status"] ?? "").trim();
|
||||
const flags = new Set(status.split(/\s+/).filter(Boolean));
|
||||
return {
|
||||
model: vars["device.model"] || vars["ups.model"] || null,
|
||||
manufacturer: vars["device.mfr"] || vars["ups.mfr"] || null,
|
||||
statusText: status || "unknown",
|
||||
onBattery: flags.has("OB"),
|
||||
lowBattery: flags.has("LB"),
|
||||
replaceBattery: flags.has("RB"),
|
||||
overload: flags.has("OVER"),
|
||||
charge: num(vars["battery.charge"]),
|
||||
runtimeSeconds: num(vars["battery.runtime"]),
|
||||
loadPercent: num(vars["ups.load"]),
|
||||
inputVoltage: num(vars["input.voltage"]),
|
||||
outputVoltage: num(vars["output.voltage"]),
|
||||
};
|
||||
}
|
||||
|
||||
/** Sends the commands one after the other on one connection, each answered by a BEGIN … END block (or an ERR line), and returns the answers. */
|
||||
function ask(host: string, port: number, commands: string[]): Promise<string[]> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const socket = net.connect({ host, port });
|
||||
const answers: string[] = [];
|
||||
let buffer = "";
|
||||
let index = 0;
|
||||
let finished = false;
|
||||
const done = (err: Error | null) => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
// After a good answer the connection is closed politely (LOGOUT is already sent); after a bad one it is simply dropped.
|
||||
if (err) socket.destroy();
|
||||
else {
|
||||
socket.end();
|
||||
setTimeout(() => socket.destroy(), 200).unref();
|
||||
}
|
||||
if (err) reject(err);
|
||||
else resolve(answers);
|
||||
};
|
||||
socket.setTimeout(TIMEOUT_MS, () => done(new Error(`no answer within ${TIMEOUT_MS / 1000} s`)));
|
||||
socket.on("error", (err: NodeJS.ErrnoException) => done(new Error(err.code === "ECONNREFUSED" ? `nothing is listening on port ${port} — is upsd running, and does it listen on this address (LISTEN in upsd.conf)?` : err.message)));
|
||||
socket.on("close", () => done(new Error("the connection closed before the answer was complete")));
|
||||
socket.on("connect", () => socket.write(`${commands[0]}\n`));
|
||||
socket.on("data", (chunk) => {
|
||||
buffer += chunk.toString("utf8");
|
||||
for (;;) {
|
||||
const err = /^ERR\s+(.*)$/m.exec(buffer);
|
||||
if (err) return done(new Error(`upsd says ${err[1].trim()}`));
|
||||
const end = /^END LIST .*$/m.exec(buffer);
|
||||
if (!end) return;
|
||||
const cut = end.index + end[0].length;
|
||||
answers.push(buffer.slice(0, cut));
|
||||
buffer = buffer.slice(cut);
|
||||
index++;
|
||||
if (index >= commands.length) {
|
||||
socket.write("LOGOUT\n");
|
||||
return done(null);
|
||||
}
|
||||
socket.write(`${commands[index]}\n`);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/** Asks the NUT server about one UPS: `upsName`, or the first it lists when none is given. */
|
||||
export async function readNut(host: string, port: number, upsName: string | null): Promise<UpsReading> {
|
||||
const address = await resolveTarget(host, { allowPrivate: true });
|
||||
let name = upsName;
|
||||
if (!name) {
|
||||
const [list] = await ask(address, port, ["LIST UPS"]);
|
||||
name = parseNutUpsNames(list)[0] ?? null;
|
||||
if (!name) throw new Error("the NUT server lists no UPS");
|
||||
}
|
||||
const [vars] = await ask(address, port, [`LIST VAR ${name}`]);
|
||||
const parsed = parseNutVars(vars);
|
||||
if (Object.keys(parsed).length === 0) throw new Error(`the NUT server gave no variables for “${name}”`);
|
||||
return interpretNut(parsed);
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
/**
|
||||
* SNMP, version 2c, only as far as asking for a few values: one GET request over UDP (port 161) naming the objects of the standard UPS-MIB (RFC 1628), which most UPSs
|
||||
* and their network cards answer. Hand-written, because a GET is a few dozen lines of BER and a whole SNMP library is far more than is needed (and nothing is ever set).
|
||||
*/
|
||||
import dgram from "node:dgram";
|
||||
import net from "node:net";
|
||||
import { resolveTarget } from "../../utils/networkGuard.js";
|
||||
import type { UpsReading } from "./types.js";
|
||||
|
||||
export const SNMP_PORT = 161;
|
||||
const TIMEOUT_MS = 2500;
|
||||
const TRIES = 2;
|
||||
|
||||
// ─── BER ────────────────────────────────────────────────────────────────────
|
||||
|
||||
const lengthBytes = (n: number): number[] => (n < 0x80 ? [n] : n < 0x100 ? [0x81, n] : [0x82, n >> 8, n & 0xff]);
|
||||
|
||||
function tlv(tag: number, content: number[] | Buffer): Buffer {
|
||||
const body = Buffer.from(content);
|
||||
return Buffer.concat([Buffer.from([tag, ...lengthBytes(body.length)]), body]);
|
||||
}
|
||||
|
||||
function encodeInt(n: number): Buffer {
|
||||
const bytes: number[] = [];
|
||||
let v = n;
|
||||
do {
|
||||
bytes.unshift(v & 0xff);
|
||||
v = Math.floor(v / 256);
|
||||
} while (v > 0);
|
||||
if (bytes[0] & 0x80) bytes.unshift(0);
|
||||
return tlv(0x02, bytes);
|
||||
}
|
||||
|
||||
/** "1.3.6.1.2.1.33.1.2.4.0" into its BER form. */
|
||||
export function encodeOid(oid: string): Buffer {
|
||||
const parts = oid.split(".").map(Number);
|
||||
const bytes: number[] = [parts[0] * 40 + parts[1]];
|
||||
for (const p of parts.slice(2)) {
|
||||
const chunk = [p & 0x7f];
|
||||
let v = Math.floor(p / 128);
|
||||
while (v > 0) {
|
||||
chunk.unshift((v & 0x7f) | 0x80);
|
||||
v = Math.floor(v / 128);
|
||||
}
|
||||
bytes.push(...chunk);
|
||||
}
|
||||
return tlv(0x06, bytes);
|
||||
}
|
||||
|
||||
/** A GET request for these objects, from this community, under this request id. */
|
||||
export function buildGet(community: string, requestId: number, oids: string[]): Buffer {
|
||||
const varbinds = oids.map((o) => tlv(0x30, Buffer.concat([encodeOid(o), tlv(0x05, [])])));
|
||||
const pdu = tlv(0xa0, Buffer.concat([encodeInt(requestId), encodeInt(0), encodeInt(0), tlv(0x30, Buffer.concat(varbinds))]));
|
||||
return tlv(0x30, Buffer.concat([encodeInt(1), tlv(0x04, Buffer.from(community, "utf8")), pdu]));
|
||||
}
|
||||
|
||||
interface Node {
|
||||
tag: number;
|
||||
value: Buffer;
|
||||
}
|
||||
|
||||
/** The TLVs one after the other in `buf`. */
|
||||
function children(buf: Buffer): Node[] {
|
||||
const out: Node[] = [];
|
||||
let i = 0;
|
||||
while (i < buf.length) {
|
||||
const tag = buf[i++];
|
||||
let len = buf[i++];
|
||||
if (len === undefined) throw new Error("the answer is cut short");
|
||||
if (len & 0x80) {
|
||||
const n = len & 0x7f;
|
||||
if (n === 0 || n > 3) throw new Error("the answer has a length that can't be read");
|
||||
len = 0;
|
||||
for (let k = 0; k < n; k++) len = len * 256 + buf[i++];
|
||||
}
|
||||
if (i + len > buf.length) throw new Error("the answer is cut short");
|
||||
out.push({ tag, value: buf.subarray(i, i + len) });
|
||||
i += len;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function decodeUint(buf: Buffer): number {
|
||||
let n = 0;
|
||||
for (const b of buf) n = n * 256 + b;
|
||||
return n;
|
||||
}
|
||||
|
||||
function decodeInt(buf: Buffer): number {
|
||||
if (buf.length === 0) return 0;
|
||||
let v = 0;
|
||||
for (const b of buf) v = v * 256 + b;
|
||||
// two's complement
|
||||
return buf[0] & 0x80 ? v - 2 ** (8 * buf.length) : v;
|
||||
}
|
||||
|
||||
function decodeOid(buf: Buffer): string {
|
||||
if (buf.length === 0) return "";
|
||||
const parts = [Math.floor(buf[0] / 40), buf[0] % 40];
|
||||
let v = 0;
|
||||
for (const b of buf.subarray(1)) {
|
||||
v = v * 128 + (b & 0x7f);
|
||||
if (!(b & 0x80)) {
|
||||
parts.push(v);
|
||||
v = 0;
|
||||
}
|
||||
}
|
||||
return parts.join(".");
|
||||
}
|
||||
|
||||
export type SnmpValue = number | string | null;
|
||||
|
||||
export interface SnmpAnswer {
|
||||
requestId: number;
|
||||
/** 0 when the agent had no complaint. */
|
||||
errorStatus: number;
|
||||
/** Object → its value; null where the agent has no such object. */
|
||||
values: Map<string, SnmpValue>;
|
||||
}
|
||||
|
||||
/** An answer to a GET: its request id and each object's value. Throws if it isn't one. */
|
||||
export function parseResponse(packet: Buffer): SnmpAnswer {
|
||||
const [message] = children(packet);
|
||||
if (!message || message.tag !== 0x30) throw new Error("that isn't an SNMP message");
|
||||
const [, , pdu] = children(message.value);
|
||||
if (!pdu || pdu.tag !== 0xa2) throw new Error("that isn't an answer to a request");
|
||||
const [id, errorStatus, , bindings] = children(pdu.value);
|
||||
const values = new Map<string, SnmpValue>();
|
||||
for (const vb of children(bindings.value)) {
|
||||
const [oid, value] = children(vb.value);
|
||||
let v: SnmpValue;
|
||||
switch (value.tag) {
|
||||
case 0x02:
|
||||
v = decodeInt(value.value);
|
||||
break;
|
||||
case 0x41: // counter32
|
||||
case 0x42: // gauge32
|
||||
case 0x43: // timeticks
|
||||
case 0x46: // counter64
|
||||
v = decodeUint(value.value);
|
||||
break;
|
||||
case 0x04:
|
||||
v = value.value.toString("utf8").replace(/\0+$/, "").trim();
|
||||
break;
|
||||
default:
|
||||
// null, noSuchObject, noSuchInstance, endOfMibView, or something not needed: there is no value here
|
||||
v = null;
|
||||
}
|
||||
values.set(decodeOid(oid.value), v);
|
||||
}
|
||||
return { requestId: decodeInt(id.value), errorStatus: decodeInt(errorStatus.value), values };
|
||||
}
|
||||
|
||||
// ─── The UPS-MIB ────────────────────────────────────────────────────────────
|
||||
|
||||
const B = "1.3.6.1.2.1.33.1";
|
||||
export const OIDS = {
|
||||
manufacturer: `${B}.1.1.0`,
|
||||
model: `${B}.1.2.0`,
|
||||
batteryStatus: `${B}.2.1.0`, // 1 unknown, 2 normal, 3 low, 4 depleted
|
||||
secondsOnBattery: `${B}.2.2.0`,
|
||||
minutesRemaining: `${B}.2.3.0`,
|
||||
chargeRemaining: `${B}.2.4.0`,
|
||||
inputVoltage: `${B}.3.3.1.3.1`,
|
||||
outputSource: `${B}.4.1.0`, // 3 normal, 4 bypass, 5 battery, 6 booster, 7 reducer
|
||||
outputVoltage: `${B}.4.4.1.2.1`,
|
||||
outputLoad: `${B}.4.4.1.5.1`,
|
||||
} as const;
|
||||
|
||||
const asNumber = (v: SnmpValue | undefined): number | null => (typeof v === "number" ? v : typeof v === "string" && v !== "" && Number.isFinite(Number(v)) ? Number(v) : null);
|
||||
const asText = (v: SnmpValue | undefined): string | null => (typeof v === "string" && v !== "" ? v : null);
|
||||
|
||||
const SOURCES: Record<number, string> = { 1: "other", 2: "no output", 3: "normal", 4: "bypass", 5: "on battery", 6: "booster", 7: "reducer" };
|
||||
const BATTERY: Record<number, string> = { 1: "battery unknown", 2: "battery normal", 3: "battery low", 4: "battery depleted" };
|
||||
|
||||
/** The objects of the UPS-MIB, as what the app keeps. */
|
||||
export function interpretSnmp(values: Map<string, SnmpValue>): UpsReading {
|
||||
const get = (oid: string) => values.get(oid);
|
||||
const source = asNumber(get(OIDS.outputSource));
|
||||
const battery = asNumber(get(OIDS.batteryStatus));
|
||||
const secondsOn = asNumber(get(OIDS.secondsOnBattery));
|
||||
const minutes = asNumber(get(OIDS.minutesRemaining));
|
||||
const onBattery = source === 5 || (source === null && secondsOn !== null && secondsOn > 0);
|
||||
const text = [source !== null ? (SOURCES[source] ?? `source ${source}`) : null, battery !== null ? (BATTERY[battery] ?? null) : null].filter(Boolean).join(", ");
|
||||
return {
|
||||
model: asText(get(OIDS.model)),
|
||||
manufacturer: asText(get(OIDS.manufacturer)),
|
||||
statusText: text || "unknown",
|
||||
onBattery,
|
||||
lowBattery: battery === 3 || battery === 4,
|
||||
replaceBattery: false,
|
||||
overload: false,
|
||||
charge: asNumber(get(OIDS.chargeRemaining)),
|
||||
runtimeSeconds: minutes !== null ? minutes * 60 : null,
|
||||
loadPercent: asNumber(get(OIDS.outputLoad)),
|
||||
inputVoltage: asNumber(get(OIDS.inputVoltage)),
|
||||
outputVoltage: asNumber(get(OIDS.outputVoltage)),
|
||||
};
|
||||
}
|
||||
|
||||
/** One request and its answer over UDP, tried a second time when nothing comes back. */
|
||||
function exchange(address: string, port: number, request: Buffer, requestId: number): Promise<SnmpAnswer> {
|
||||
const type = net.isIP(address) === 6 ? "udp6" : "udp4";
|
||||
return new Promise((resolve, reject) => {
|
||||
const socket = dgram.createSocket(type);
|
||||
let tries = 0;
|
||||
let timer: ReturnType<typeof setTimeout>;
|
||||
const finish = (err: Error | null, answer?: SnmpAnswer) => {
|
||||
clearTimeout(timer);
|
||||
socket.close();
|
||||
if (err) reject(err);
|
||||
else resolve(answer!);
|
||||
};
|
||||
const send = () => {
|
||||
tries++;
|
||||
socket.send(request, port, address, (err) => err && finish(err));
|
||||
timer = setTimeout(() => (tries < TRIES ? send() : finish(new Error(`no answer within ${(TIMEOUT_MS * TRIES) / 1000} s — is SNMP on, and is the community right?`))), TIMEOUT_MS);
|
||||
};
|
||||
socket.on("error", (err) => finish(err));
|
||||
socket.on("message", (msg) => {
|
||||
try {
|
||||
const answer = parseResponse(msg);
|
||||
if (answer.requestId === requestId) finish(null, answer);
|
||||
} catch {
|
||||
// something else arriving on this port: ignored
|
||||
}
|
||||
});
|
||||
send();
|
||||
});
|
||||
}
|
||||
|
||||
export async function readSnmp(host: string, port: number, community: string): Promise<UpsReading> {
|
||||
const address = await resolveTarget(host, { allowPrivate: true });
|
||||
const requestId = 1 + Math.floor(Math.random() * 0x7ffffff0);
|
||||
const answer = await exchange(address, port, buildGet(community, requestId, Object.values(OIDS)), requestId);
|
||||
if (answer.errorStatus !== 0) throw new Error(`the device refused the request (error ${answer.errorStatus})`);
|
||||
const known = [...answer.values.values()].filter((v) => v !== null).length;
|
||||
if (known === 0) throw new Error("the device answers, but has none of the standard UPS objects (UPS-MIB)");
|
||||
return interpretSnmp(answer.values);
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
/** What a UPS says about itself, the same whichever way it was asked (NUT or SNMP). Anything it doesn't say is null. */
|
||||
export interface UpsReading {
|
||||
model: string | null;
|
||||
manufacturer: string | null;
|
||||
/** The UPS's own words for how it is ("OL CHRG", "on battery"), to show. */
|
||||
statusText: string;
|
||||
/** Mains power is gone and it is running from its battery. */
|
||||
onBattery: boolean;
|
||||
/** The battery is nearly empty: shut things down now. */
|
||||
lowBattery: boolean;
|
||||
/** The UPS asks for its battery to be replaced. */
|
||||
replaceBattery: boolean;
|
||||
/** It is carrying more than it can. */
|
||||
overload: boolean;
|
||||
/** Battery charge, percent. */
|
||||
charge: number | null;
|
||||
/** How long the battery would last at the present load. */
|
||||
runtimeSeconds: number | null;
|
||||
/** How much of its capacity the load uses, percent. */
|
||||
loadPercent: number | null;
|
||||
inputVoltage: number | null;
|
||||
outputVoltage: number | null;
|
||||
}
|
||||
|
||||
export interface UpsConfig {
|
||||
host: string;
|
||||
port?: string;
|
||||
/** Read over SNMP (version 2c) instead of NUT. */
|
||||
snmp?: boolean | string;
|
||||
/** NUT: the UPS's name on that server; the first one it lists when left blank. */
|
||||
upsName?: string;
|
||||
/** SNMP: the read community; "public" when left blank. */
|
||||
community?: string;
|
||||
}
|
||||
@@ -27,6 +27,7 @@ import { attachMatchedServers, summarizeMonitors, toMatchableServers } from "../
|
||||
import { getSourceStatus } from "../services/integrationHealthMonitor.js";
|
||||
import { removeIntegrationData, syncUnifiIntegration } from "../services/unifiSync.js";
|
||||
import { removeHomeAssistantData, syncHomeAssistant } from "../services/homeAssistantSync.js";
|
||||
import { forgetUps, syncUps } from "../services/upsSync.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
import { ID_PATTERNS, isValidId } from "../utils/pathSegment.js";
|
||||
@@ -127,6 +128,7 @@ integrationsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res)
|
||||
// A new UniFi controller is read at once, so its equipment is there when the person opens the page (not waited for: a slow controller must not slow this).
|
||||
if (type === "unifi") void syncUnifiIntegration(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "homeassistant") void syncHomeAssistant(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "ups") void syncUps(created.id).then(() => invalidateAlertsCache());
|
||||
|
||||
res.status(201).json({
|
||||
integration: {
|
||||
@@ -229,6 +231,7 @@ integrationsRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req,
|
||||
|
||||
if (updated.type === "unifi" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncUnifiIntegration(updated.id).then(() => invalidateAlertsCache());
|
||||
if (updated.type === "homeassistant" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncHomeAssistant(updated.id).then(() => invalidateAlertsCache());
|
||||
if (updated.type === "ups" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncUps(updated.id).then(() => invalidateAlertsCache());
|
||||
|
||||
res.json({
|
||||
integration: {
|
||||
@@ -307,6 +310,7 @@ integrationsRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req,
|
||||
|
||||
if (existing.type === "unifi") await removeIntegrationData(id);
|
||||
if (existing.type === "homeassistant") await removeHomeAssistantData(id);
|
||||
if (existing.type === "ups") await forgetUps(id);
|
||||
await db.delete(integrations).where(eq(integrations.id, id));
|
||||
if (existing.credentialId) {
|
||||
await db.delete(integrationCredentials).where(eq(integrationCredentials.id, existing.credentialId));
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
import { Router } from "express";
|
||||
import { and, asc, eq, gte } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, upsReadings, upsState } from "../db/schema.js";
|
||||
import { requireAuth } from "../auth/middleware.js";
|
||||
import { loadGraph } from "../services/dependencies.js";
|
||||
import { bucketUps, isUpsFresh, READING_KEEP_DAYS, shutdownOrder, upsProblems, upsSyncStatuses } from "../services/upsSync.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const upsRouter = Router();
|
||||
upsRouter.use(requireAuth);
|
||||
|
||||
const DAY_MS = 86_400_000;
|
||||
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
|
||||
|
||||
const isSnmp = (config: string | null) => {
|
||||
try {
|
||||
const c = config ? (JSON.parse(config) as Record<string, unknown>) : {};
|
||||
return c.snmp === true || c.snmp === "true";
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
/** Every UPS, how it is now, and what is on it in the order to shut it down. Everyone signed in can see it, like the other pages made of stored readings. */
|
||||
upsRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const [rows, states, statuses, graph] = await Promise.all([
|
||||
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled, config: integrations.config }).from(integrations).where(eq(integrations.type, "ups")).orderBy(integrations.name),
|
||||
db.select().from(upsState),
|
||||
upsSyncStatuses(),
|
||||
loadGraph(),
|
||||
]);
|
||||
const now = new Date();
|
||||
const byId = new Map(states.map((s) => [s.integrationId, s]));
|
||||
res.json({
|
||||
ups: rows.map((r) => {
|
||||
const state = byId.get(r.id) ?? null;
|
||||
const status = statuses[r.id] ?? null;
|
||||
const fresh = isUpsFresh(status ?? undefined, now.getTime());
|
||||
const { order, looped } = shutdownOrder(graph.edges, `integration:${r.id}`);
|
||||
return {
|
||||
id: r.id,
|
||||
name: r.name,
|
||||
enabled: r.enabled,
|
||||
protocol: isSnmp(r.config) ? "snmp" : "nut",
|
||||
status,
|
||||
fresh,
|
||||
state,
|
||||
problems: state && fresh ? upsProblems(r.id, r.name, state, now) : [],
|
||||
// What needs this UPS, directly or through something else, in the order to shut it down (step 1 first).
|
||||
carries: order.map((o) => {
|
||||
const [type, id] = o.key.split(":");
|
||||
return { key: o.key, type, id: Number(id), name: graph.names.get(o.key) ?? o.key, step: o.step };
|
||||
}),
|
||||
orderLooped: looped,
|
||||
};
|
||||
}),
|
||||
});
|
||||
}));
|
||||
|
||||
/** One UPS's charge, load and mains voltage over time (readings are kept 35 days). */
|
||||
upsRouter.get("/:id/history", asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [row] = await db.select({ id: integrations.id, name: integrations.name, type: integrations.type }).from(integrations).where(eq(integrations.id, id)).limit(1);
|
||||
if (!row || row.type !== "ups") return res.status(404).json({ error: "not_found" });
|
||||
const asked = Number(req.query.days);
|
||||
const days = Number.isFinite(asked) ? Math.min(READING_KEEP_DAYS, Math.max(1, Math.round(asked))) : 7;
|
||||
const to = Date.now();
|
||||
const from = to - days * DAY_MS;
|
||||
const rows = await db
|
||||
.select()
|
||||
.from(upsReadings)
|
||||
.where(and(eq(upsReadings.integrationId, id), gte(upsReadings.at, new Date(from).toISOString())))
|
||||
.orderBy(asc(upsReadings.at));
|
||||
res.json({ ups: { id: row.id, name: row.name }, days, ...bucketUps(rows, from, to, 200) });
|
||||
}));
|
||||
@@ -72,6 +72,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
|
||||
unifi: { label: "UniFi", link: "/network/devices" },
|
||||
npm: { label: "Nginx Proxy Manager", link: "/proxy-hosts" },
|
||||
homeassistant: { label: "Home Assistant", link: "/home-assistant" },
|
||||
ups: { label: "UPS", link: "/ups" },
|
||||
secrets: { label: "Secrets", link: "/secrets" },
|
||||
domains: { label: "Domains", link: "/dns?view=domains" },
|
||||
endpoint: { label: "Endpoint", link: "/endpoints" },
|
||||
@@ -100,6 +101,8 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
|
||||
if (m) return { type: "synology", id: Number(m[1]), category: "disk" };
|
||||
m = /^automation:(semaphore|gitea):(\d+):/.exec(key);
|
||||
if (m) return { type: m[1], id: Number(m[2]), category: "automation" };
|
||||
m = /^ups-(?:battery|low|replace|load):(\d+)$/.exec(key);
|
||||
if (m) return { type: "ups", id: Number(m[1]), category: "monitoring" };
|
||||
return { type: "server", id: null, category: "disk" };
|
||||
}
|
||||
|
||||
@@ -213,22 +216,23 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
const offlineMatch = /^offline:(server|device):(\d+)$/.exec(c.key);
|
||||
const offlineNode = offlineMatch ? `${offlineMatch[1]}:${offlineMatch[2]}` : undefined;
|
||||
const parentOf = offlineNode ? cover.cause.get(offlineNode) : undefined;
|
||||
const upsOnBattery = /^ups-battery:(\d+)$/.exec(c.key);
|
||||
push({
|
||||
causedBy: parentOf !== undefined ? { name: cover.graph.names.get(parentOf) ?? parentOf, alertId: alertIdOfNode(parentOf, cover.graph) ?? undefined } : undefined,
|
||||
affects: offlineNode ? cover.taken.get(offlineNode)?.sort((a, b) => a.localeCompare(b)) : undefined,
|
||||
affects: offlineNode ? cover.taken.get(offlineNode)?.sort((a, b) => a.localeCompare(b)) : upsOnBattery ? cover.taken.get(`integration:${upsOnBattery[1]}`)?.sort((a, b) => a.localeCompare(b)) : undefined,
|
||||
category: origin.category,
|
||||
severity: c.severity ?? "warning",
|
||||
type: origin.type,
|
||||
message: c.message,
|
||||
key: c.key,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : undefined,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : undefined,
|
||||
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
|
||||
});
|
||||
}
|
||||
// Integrations the check couldn't read this time. (A whole-integration entry covers its nodes, so skip those.)
|
||||
if (held.has("server:rules")) cannotCheck("Alert rules", "couldn't be judged — see the server log", "Server and storage health");
|
||||
for (const h of held) {
|
||||
const m = /^(proxmox|synology|unifi):(\d+)(?::(.+))?$/.exec(h);
|
||||
const m = /^(proxmox|synology|unifi|ups):(\d+)(?::(.+))?$/.exec(h);
|
||||
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
|
||||
const row = intById.get(Number(m[2]));
|
||||
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
|
||||
|
||||
@@ -36,6 +36,7 @@ export const HISTORY: Record<string, HistoryMeta> = {
|
||||
heartbeat_pings: { time: "at", keep: 14 },
|
||||
storage_samples: { time: "at", keep: 400, note: "one reading an hour of each Proxmox storage and Synology volume" },
|
||||
ha_readings: { time: "at", keep: 35, note: "the watched Home Assistant numbers" },
|
||||
ups_readings: { time: "at", keep: 35, note: "a UPS's charge, load and mains voltage" },
|
||||
alert_incidents: { time: "resolved_at", keep: 400, where: "resolved_at IS NOT NULL", note: "trimmed 400 days after an incident resolved" },
|
||||
webhook_deliveries: { time: "created_at", keep: 14, note: "the last 100 per webhook, and none older than two weeks" },
|
||||
audit_log: { time: "created_at", keep: "log", note: "who did what" },
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
* device each is connected through, as a UniFi controller says), plus the dependencies written here. Servers, network devices and integrations are the things
|
||||
* that can be down and be needed. (DNS providers raise no alerts of their own, so there is
|
||||
* nothing to fold.) An integration is judged per kind of service, as its health is: it is folded only if every enabled instance of that
|
||||
* kind is explained.
|
||||
* kind is explained. A UPS is down while it runs on battery (services/upsSync.ts).
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
@@ -15,6 +15,7 @@ import { dependencies, integrations, networkDevices, networkServerLinks, servers
|
||||
import { getInternalFlag } from "./settingsStore.js";
|
||||
import { listActiveWindows } from "./maintenance.js";
|
||||
import { loadParentRows, resolveParents, type ParentLink } from "./serverParents.js";
|
||||
import { upsOnBatteryIds } from "./upsSync.js";
|
||||
|
||||
/** Where the health monitor keeps the problems it is announcing; read here so a service's alert can tell a server it needs is down. */
|
||||
export const HEALTH_STATE_FLAG = "healthActiveConditions";
|
||||
@@ -180,6 +181,8 @@ export function alertIdOfNode(key: string, graph: Graph): string | null {
|
||||
if (m[1] === "server") return `offline:offline:server:${m[2]}`;
|
||||
if (m[1] === "device") return `offline:offline:device:${m[2]}`;
|
||||
const row = graph.intRows.find((r) => r.id === Number(m[2]));
|
||||
// A UPS is "down" for the map while it runs on battery, and its alert is the one about that.
|
||||
if (row?.type === "ups") return `monitoring:ups-battery:${row.id}`;
|
||||
return row ? `integration:failing:${row.type}` : null;
|
||||
}
|
||||
|
||||
@@ -229,6 +232,8 @@ export async function dependencyCover(offlineServerIds: Iterable<number>, failin
|
||||
for (const id of offlineDeviceIds) down.add(nodeKey("device", id));
|
||||
const failing = new Set(failingTypes);
|
||||
for (const [type, ids] of graph.instances) if (failing.has(type)) for (const id of ids) down.add(nodeKey("integration", id));
|
||||
// A UPS running on battery counts as down: whatever it carries is about to follow it, and is folded under its alert if it does.
|
||||
for (const id of await upsOnBatteryIds()) if (graph.names.has(nodeKey("integration", id))) down.add(nodeKey("integration", id));
|
||||
const cause = coveredNodes(down, graph.edges);
|
||||
const taken = new Map<string, string[]>();
|
||||
for (const [child, parent] of cause) taken.set(parent, [...(taken.get(parent) ?? []), graph.names.get(child) ?? child]);
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, networkDevices, serverPorts, servers } from "../db/schema.js";
|
||||
import { integrations, networkDevices, serverPorts, servers, upsState } from "../db/schema.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
|
||||
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
|
||||
@@ -17,6 +17,7 @@ import { parseJson, type AgentPort } from "./agentPorts.js";
|
||||
import { diskLimit, parseThresholds, type ServerThresholds } from "./serverThresholds.js";
|
||||
import { computeRuleFindings, findingKey, listRules, type RuleFinding } from "./alertRules.js";
|
||||
import { recordStorageSamples } from "./storageHistory.js";
|
||||
import { isUpsFresh, upsProblems, upsSyncStatuses, type UpsStateRow } from "./upsSync.js";
|
||||
|
||||
const STATE_FLAG = HEALTH_STATE_FLAG;
|
||||
/** After a restart the agents haven't had a chance to report yet (they run every 15 min), so server-derived conditions are held rather than judged. */
|
||||
@@ -63,11 +64,18 @@ export interface UnifiSnapshot {
|
||||
integrationName: string;
|
||||
devices: { id: number; name: string; kind: string; model: string | null; state: string; offlineSince: string | null }[];
|
||||
}
|
||||
/** One UPS as last read, only offered while that reading is recent enough to be believed. */
|
||||
export interface UpsSnapshot {
|
||||
integrationId: number;
|
||||
integrationName: string;
|
||||
state: UpsStateRow;
|
||||
}
|
||||
export interface HealthSnapshot {
|
||||
servers: ServerSnapshot[];
|
||||
proxmox: ProxmoxSnapshot[];
|
||||
synology: SynologySnapshot[];
|
||||
unifi?: UnifiSnapshot[];
|
||||
ups?: UpsSnapshot[];
|
||||
/** What the administrator's own alert rules find on the servers (services/alertRules.ts). */
|
||||
rules?: RuleFinding[];
|
||||
}
|
||||
@@ -183,6 +191,11 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
}
|
||||
}
|
||||
|
||||
// A UPS that has lost mains power, is nearly empty, wants a new battery or is overloaded.
|
||||
for (const u of snapshot.ups ?? []) {
|
||||
for (const p of upsProblems(u.integrationId, u.integrationName, u.state, new Date(now))) out.push({ key: p.key, source: `ups:${u.integrationId}`, message: p.message, severity: p.severity });
|
||||
}
|
||||
|
||||
for (const px of snapshot.proxmox) {
|
||||
const seenShared = new Set<string>();
|
||||
for (const node of px.nodes) {
|
||||
@@ -395,6 +408,21 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
}
|
||||
}
|
||||
|
||||
// UPSs are read by their own pass every minute; what is judged here is what that pass stored, and only while it is recent.
|
||||
const upsRows = rows.filter((r) => r.type === "ups");
|
||||
if (upsRows.length > 0) {
|
||||
const statuses = await upsSyncStatuses();
|
||||
const nowMs = Date.now();
|
||||
for (const row of upsRows) {
|
||||
const [state] = await db.select().from(upsState).where(eq(upsState.integrationId, row.id)).limit(1);
|
||||
if (!state || !isUpsFresh(statuses[row.id], nowMs)) {
|
||||
held.add(`ups:${row.id}`);
|
||||
continue;
|
||||
}
|
||||
(snapshot.ups ??= []).push({ integrationId: row.id, integrationName: row.name, state });
|
||||
}
|
||||
}
|
||||
|
||||
for (const row of rows) {
|
||||
if (row.type !== "proxmox" && row.type !== "synology") continue;
|
||||
try {
|
||||
@@ -494,7 +522,8 @@ export async function runHealthCheck(now: number = Date.now()): Promise<{ added:
|
||||
// A host's message says what else it takes with it.
|
||||
const added = announced.map((c) => {
|
||||
const m = /^offline:(server|device):(\d+)$/.exec(c.key);
|
||||
const children = m ? cover.taken.get(`${m[1]}:${m[2]}`) : undefined;
|
||||
const ups = /^ups-battery:(\d+)$/.exec(c.key);
|
||||
const children = m ? cover.taken.get(`${m[1]}:${m[2]}`) : ups ? cover.taken.get(`integration:${ups[1]}`) : undefined;
|
||||
return children?.length ? { ...c, message: `${c.message} — also takes down ${alsoList(children)}` } : c;
|
||||
});
|
||||
|
||||
|
||||
@@ -37,6 +37,8 @@ export function subjectOfConditionKey(key: string): string | null {
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^automation:(?:semaphore|gitea):(\d+):/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^ups-(?:battery|low|replace|load):(\d+)$/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
@@ -3,8 +3,10 @@
|
||||
* severity, integrations, maintenance, things that expire. It reads what the app already knows — nothing here calls out to a server or an
|
||||
* integration on its own, so a scrape every fifteen seconds costs a few database reads (the alerts are the app's own minute-old result).
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers } from "../db/schema.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState } from "../db/schema.js";
|
||||
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
|
||||
import { latestTests } from "./restoreTests.js";
|
||||
import { heartbeatState, parseStamp } from "./heartbeats.js";
|
||||
import { env } from "../env.js";
|
||||
@@ -56,6 +58,25 @@ class Exposition {
|
||||
}
|
||||
}
|
||||
|
||||
/** Each UPS, for /metrics: whether it is on battery, how full the battery is, how long it would last and how loaded it is. */
|
||||
async function addUpsMetrics(out: Exposition, now: Date) {
|
||||
const fresh = new Set<number>();
|
||||
const nowMs = now.getTime();
|
||||
const statuses = await upsSyncStatuses();
|
||||
for (const [id, s] of Object.entries(statuses)) if (isUpsFresh(s, nowMs)) fresh.add(Number(id));
|
||||
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "ups"))).map((r) => [r.id, r]));
|
||||
for (const s of await db.select().from(upsState)) {
|
||||
const row = names.get(s.integrationId);
|
||||
if (!row || !row.enabled || !fresh.has(s.integrationId)) continue;
|
||||
const labels = { name: row.name };
|
||||
out.add("homelab_ups_on_battery", "gauge", "1 if the UPS has lost mains power and runs on its battery, else 0.", s.onBattery ? 1 : 0, labels);
|
||||
if (s.charge !== null) out.add("homelab_ups_battery_charge_percent", "gauge", "How full the UPS's battery is.", s.charge, labels);
|
||||
if (s.runtimeSeconds !== null) out.add("homelab_ups_runtime_seconds", "gauge", "How long the UPS's battery would last at the present load.", s.runtimeSeconds, labels);
|
||||
if (s.loadPercent !== null) out.add("homelab_ups_load_percent", "gauge", "How much of its capacity the UPS's load uses.", s.loadPercent, labels);
|
||||
if (s.inputVoltage !== null) out.add("homelab_ups_input_volts", "gauge", "The mains voltage the UPS sees.", s.inputVoltage, labels);
|
||||
}
|
||||
}
|
||||
|
||||
/** Endpoint checks and restore tests, for /metrics. */
|
||||
async function addEndpointMetrics(out: Exposition, now: Date) {
|
||||
for (const c of await db.select().from(endpointChecks)) {
|
||||
@@ -217,6 +238,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
out.add("homelab_scrape_duration_seconds", "gauge", "How long building this answer took.", (Date.now() - started) / 1000);
|
||||
out.add("homelab_scrape_timestamp_seconds", "gauge", "When this answer was built, as a Unix time.", nowSec);
|
||||
await addEndpointMetrics(out, now);
|
||||
await addUpsMetrics(out, now);
|
||||
await addHeartbeatMetrics(out, now);
|
||||
return out.toString();
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
/**
|
||||
* Keeps what each UPS says up to date: every minute the UPS integrations are read (NUT or SNMP — integrations/ups), the state is stored, and a reading is added to the
|
||||
* history (every five minutes, and every minute while it runs on battery). The page, the charts, the alerts and the dependency map are built from what is stored, so
|
||||
* none of them talks to a UPS, and one that can't be reached costs only how fresh they are.
|
||||
*
|
||||
* What makes an alert: mains power lost (the UPS runs on battery), a nearly empty battery, a battery to be replaced, and a load near its capacity. While a UPS is on
|
||||
* battery it counts as *down* in the dependency map, so the servers that go offline with it are folded under its alert instead of each being announced.
|
||||
*/
|
||||
import { and, desc, eq, gte, lt } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, upsReadings, upsState } from "../db/schema.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createUpsAdapter } from "../integrations/ups/adapter.js";
|
||||
import type { UpsReading } from "../integrations/ups/types.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export const SYNC_INTERVAL_MS = 60_000;
|
||||
/** After this long without a good read, what is stored is no longer believed: alerts for it are held rather than judged. */
|
||||
export const STALE_AFTER_MS = 5 * 60_000;
|
||||
export const READING_KEEP_DAYS = 35;
|
||||
const READING_GAP_MS = 5 * 60_000 - 20_000;
|
||||
/** A load this high (percent of capacity) is a warning. */
|
||||
export const OVERLOAD_PERCENT = 90;
|
||||
const STATUS_FLAG = "upsSync";
|
||||
|
||||
export interface UpsSyncStatus {
|
||||
/** The last read that worked (kept when a later one failed). */
|
||||
at: string;
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
async function readStatuses(): Promise<Record<string, UpsSyncStatus>> {
|
||||
try {
|
||||
const raw = await getInternalFlag(STATUS_FLAG);
|
||||
return raw ? (JSON.parse(raw) as Record<string, UpsSyncStatus>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export async function upsSyncStatuses(): Promise<Record<number, UpsSyncStatus>> {
|
||||
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
|
||||
}
|
||||
|
||||
export function isUpsFresh(status: UpsSyncStatus | undefined, nowMs: number): boolean {
|
||||
return !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
|
||||
}
|
||||
|
||||
export type UpsStateRow = typeof upsState.$inferSelect;
|
||||
|
||||
/** Writes down one read: the state (with when mains power was lost, kept until it is back) and, when it is time, a point in the history. */
|
||||
export async function storeReading(integrationId: number, r: UpsReading, now: Date = new Date()): Promise<void> {
|
||||
const nowIso = now.toISOString();
|
||||
const [before] = await db.select().from(upsState).where(eq(upsState.integrationId, integrationId)).limit(1);
|
||||
const onBatterySince = r.onBattery ? (before?.onBattery && before.onBatterySince ? before.onBatterySince : nowIso) : null;
|
||||
const values = {
|
||||
model: r.model,
|
||||
manufacturer: r.manufacturer,
|
||||
statusText: r.statusText,
|
||||
onBattery: r.onBattery,
|
||||
lowBattery: r.lowBattery,
|
||||
replaceBattery: r.replaceBattery,
|
||||
overload: r.overload,
|
||||
charge: r.charge,
|
||||
runtimeSeconds: r.runtimeSeconds,
|
||||
loadPercent: r.loadPercent,
|
||||
inputVoltage: r.inputVoltage,
|
||||
outputVoltage: r.outputVoltage,
|
||||
onBatterySince,
|
||||
updatedAt: nowIso,
|
||||
};
|
||||
await db.insert(upsState).values({ integrationId, ...values }).onConflictDoUpdate({ target: upsState.integrationId, set: values });
|
||||
|
||||
// Every minute while on battery (that is when it matters), and when it changes; otherwise every five.
|
||||
const [newest] = await db.select({ at: upsReadings.at }).from(upsReadings).where(eq(upsReadings.integrationId, integrationId)).orderBy(desc(upsReadings.at)).limit(1);
|
||||
const changed = !before || before.onBattery !== r.onBattery;
|
||||
if (changed || r.onBattery || !newest || now.getTime() - Date.parse(newest.at) >= READING_GAP_MS) {
|
||||
await db.insert(upsReadings).values({ integrationId, at: nowIso, charge: r.charge, runtimeSeconds: r.runtimeSeconds, loadPercent: r.loadPercent, inputVoltage: r.inputVoltage, onBattery: r.onBattery });
|
||||
}
|
||||
await db.delete(upsReadings).where(and(eq(upsReadings.integrationId, integrationId), lt(upsReadings.at, new Date(now.getTime() - READING_KEEP_DAYS * 86_400_000).toISOString())));
|
||||
}
|
||||
|
||||
/** Reads one UPS and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
|
||||
export async function syncUps(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<UpsReading>): Promise<UpsSyncStatus> {
|
||||
let status: UpsSyncStatus;
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
if (!loaded || loaded.integration.type !== "ups") throw new Error("not a UPS integration");
|
||||
const reading = await (read ? read(loaded.config) : createUpsAdapter(loaded.config as never).read());
|
||||
await storeReading(integrationId, reading, now);
|
||||
status = { at: now.toISOString(), ok: true, error: null };
|
||||
} catch (err) {
|
||||
const previous = (await readStatuses())[String(integrationId)];
|
||||
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
console.error(`[ups] couldn't read integration ${integrationId}:`, status.error);
|
||||
}
|
||||
try {
|
||||
const all = await readStatuses();
|
||||
all[String(integrationId)] = status;
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
} catch (err) {
|
||||
console.error("[ups] couldn't record the result:", err);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
export async function syncAllUps(now: Date = new Date()): Promise<Record<number, UpsSyncStatus>> {
|
||||
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "ups"), eq(integrations.enabled, true)));
|
||||
const out: Record<number, UpsSyncStatus> = {};
|
||||
for (const r of rows) out[r.id] = await syncUps(r.id, now);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Forgets how a UPS's last read went when it is removed (its state and history go with its row). */
|
||||
export async function forgetUps(integrationId: number): Promise<void> {
|
||||
const all = await readStatuses();
|
||||
if (String(integrationId) in all) {
|
||||
delete all[String(integrationId)];
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
}
|
||||
}
|
||||
|
||||
/** The ids of the UPSs that are running on battery, as last read and believed: they count as down in the dependency map. */
|
||||
export async function upsOnBatteryIds(now: Date = new Date()): Promise<number[]> {
|
||||
try {
|
||||
const rows = await db
|
||||
.select({ id: upsState.integrationId })
|
||||
.from(upsState)
|
||||
.innerJoin(integrations, eq(integrations.id, upsState.integrationId))
|
||||
.where(and(eq(upsState.onBattery, true), eq(integrations.enabled, true), gte(upsState.updatedAt, new Date(now.getTime() - STALE_AFTER_MS).toISOString())));
|
||||
return rows.map((r) => r.id);
|
||||
} catch (err) {
|
||||
console.error("[ups] couldn't tell which are on battery:", err);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// ─── What is wrong ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface UpsProblem {
|
||||
key: string;
|
||||
kind: "battery" | "low" | "replace" | "load";
|
||||
severity: "critical" | "warning";
|
||||
message: string;
|
||||
}
|
||||
|
||||
const minutesWords = (seconds: number) => {
|
||||
const m = Math.max(1, Math.round(seconds / 60));
|
||||
return m < 120 ? `${m} min` : `${Math.round(m / 60)} h`;
|
||||
};
|
||||
const since = (iso: string, now: Date) => {
|
||||
const m = Math.max(0, Math.floor((now.getTime() - Date.parse(iso)) / 60_000));
|
||||
return m < 1 ? "less than a minute" : m < 120 ? `${m} min` : `${Math.round(m / 60)} h`;
|
||||
};
|
||||
|
||||
/** Pure: what is wrong with one UPS, from what is stored for it. */
|
||||
export function upsProblems(id: number, name: string, s: Pick<UpsStateRow, "onBattery" | "lowBattery" | "replaceBattery" | "overload" | "charge" | "runtimeSeconds" | "loadPercent" | "onBatterySince">, now: Date): UpsProblem[] {
|
||||
const out: UpsProblem[] = [];
|
||||
const left = s.runtimeSeconds !== null ? `${minutesWords(s.runtimeSeconds)} left` : null;
|
||||
const charge = s.charge !== null ? `${Math.round(s.charge)} %` : null;
|
||||
if (s.onBattery) {
|
||||
const detail = [s.onBatterySince ? `for ${since(s.onBatterySince, now)}` : null, left, charge].filter(Boolean).join(", ");
|
||||
out.push({ key: `ups-battery:${id}`, kind: "battery", severity: "critical", message: `${name}: mains power has failed — running on battery${detail ? ` (${detail})` : ""}` });
|
||||
}
|
||||
if (s.lowBattery) {
|
||||
out.push({ key: `ups-low:${id}`, kind: "low", severity: "critical", message: `${name}: the battery is nearly empty${[charge, left].filter(Boolean).length ? ` (${[charge, left].filter(Boolean).join(", ")})` : ""} — shut down what it carries` });
|
||||
}
|
||||
if (s.replaceBattery) out.push({ key: `ups-replace:${id}`, kind: "replace", severity: "warning", message: `${name}: the UPS asks for its battery to be replaced` });
|
||||
if (s.overload || (s.loadPercent !== null && s.loadPercent >= OVERLOAD_PERCENT)) {
|
||||
out.push({ key: `ups-load:${id}`, kind: "load", severity: "warning", message: `${name} is carrying ${s.loadPercent !== null ? `${Math.round(s.loadPercent)} % of` : "more than"} what it can` });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ─── What is on it, and in what order to shut it down ───────────────────────
|
||||
|
||||
/**
|
||||
* Pure: everything that needs `key` (directly or through others), in the order to shut it down — what nothing else needs first, then what that needed, and so on, so
|
||||
* nothing is switched off while something that depends on it is still running. Each comes with its step (1 = first). `looped` is set if the order could not be settled
|
||||
* because some of them need each other.
|
||||
*/
|
||||
export function shutdownOrder(edges: Map<string, string[]>, key: string): { order: { key: string; step: number }[]; looped: boolean } {
|
||||
const reverse = new Map<string, string[]>();
|
||||
for (const [from, tos] of edges) for (const to of tos) reverse.set(to, [...(reverse.get(to) ?? []), from]);
|
||||
const set = new Set<string>();
|
||||
const queue = [key];
|
||||
while (queue.length > 0) {
|
||||
for (const d of reverse.get(queue.shift()!) ?? []) {
|
||||
if (d === key || set.has(d)) continue;
|
||||
set.add(d);
|
||||
queue.push(d);
|
||||
}
|
||||
}
|
||||
// A node's step is one more than the latest step of anything in the set that needs it.
|
||||
const step = new Map<string, number>();
|
||||
let looped = false;
|
||||
const visiting = new Set<string>();
|
||||
const stepOf = (n: string): number => {
|
||||
if (step.has(n)) return step.get(n)!;
|
||||
if (visiting.has(n)) {
|
||||
looped = true;
|
||||
return 1;
|
||||
}
|
||||
visiting.add(n);
|
||||
const above = (reverse.get(n) ?? []).filter((d) => set.has(d));
|
||||
const s = above.length === 0 ? 1 : 1 + Math.max(...above.map(stepOf));
|
||||
visiting.delete(n);
|
||||
step.set(n, s);
|
||||
return s;
|
||||
};
|
||||
for (const n of set) stepOf(n);
|
||||
return { order: [...set].map((k) => ({ key: k, step: step.get(k) ?? 1 })).sort((a, b) => a.step - b.step || a.key.localeCompare(b.key)), looped };
|
||||
}
|
||||
|
||||
// ─── History ────────────────────────────────────────────────────────────────
|
||||
|
||||
export interface UpsPoint {
|
||||
/** ISO. */
|
||||
t: string;
|
||||
charge: number | null;
|
||||
runtimeSeconds: number | null;
|
||||
loadPercent: number | null;
|
||||
inputVoltage: number | null;
|
||||
/** It was on battery at some time in this stretch. */
|
||||
onBattery: boolean;
|
||||
}
|
||||
|
||||
type ReadingRow = { at: string; charge: number | null; runtimeSeconds: number | null; loadPercent: number | null; inputVoltage: number | null; onBattery: boolean };
|
||||
|
||||
/** Pure: readings into at most `maxPoints` equal slices of [from, to] (each the average of what it holds), and the stretches spent on battery. */
|
||||
export function bucketUps(rows: ReadingRow[], from: number, to: number, maxPoints: number): { points: UpsPoint[]; onBatteryPeriods: { from: string; to: string; ongoing: boolean }[] } {
|
||||
const size = Math.max((to - from) / maxPoints, 1);
|
||||
const buckets = new Map<number, { t: number; n: number; sums: Record<"charge" | "runtimeSeconds" | "loadPercent" | "inputVoltage", { sum: number; n: number }>; onBattery: boolean }>();
|
||||
for (const r of rows) {
|
||||
const t = Date.parse(r.at);
|
||||
if (t < from || t > to) continue;
|
||||
const key = Math.min(maxPoints - 1, Math.floor((t - from) / size));
|
||||
const b = buckets.get(key) ?? { t: 0, n: 0, sums: { charge: { sum: 0, n: 0 }, runtimeSeconds: { sum: 0, n: 0 }, loadPercent: { sum: 0, n: 0 }, inputVoltage: { sum: 0, n: 0 } }, onBattery: false };
|
||||
b.t += t;
|
||||
b.n += 1;
|
||||
b.onBattery ||= r.onBattery;
|
||||
for (const k of ["charge", "runtimeSeconds", "loadPercent", "inputVoltage"] as const) {
|
||||
const v = r[k];
|
||||
if (v !== null) {
|
||||
b.sums[k].sum += v;
|
||||
b.sums[k].n += 1;
|
||||
}
|
||||
}
|
||||
buckets.set(key, b);
|
||||
}
|
||||
const avg = (x: { sum: number; n: number }) => (x.n === 0 ? null : Math.round((x.sum / x.n) * 10) / 10);
|
||||
const points = [...buckets.entries()].sort((a, b) => a[0] - b[0]).map(([, b]) => ({ t: new Date(b.t / b.n).toISOString(), charge: avg(b.sums.charge), runtimeSeconds: avg(b.sums.runtimeSeconds), loadPercent: avg(b.sums.loadPercent), inputVoltage: avg(b.sums.inputVoltage), onBattery: b.onBattery }));
|
||||
// A stretch on battery runs from the first reading that says so to the first that says it is back.
|
||||
const periods: { from: string; to: string; ongoing: boolean }[] = [];
|
||||
let start: string | null = null;
|
||||
let last: string | null = null;
|
||||
for (const r of [...rows].sort((a, b) => a.at.localeCompare(b.at))) {
|
||||
if (r.onBattery) {
|
||||
start ??= r.at;
|
||||
last = r.at;
|
||||
} else if (start !== null) {
|
||||
periods.push({ from: start, to: r.at, ongoing: false });
|
||||
start = null;
|
||||
}
|
||||
}
|
||||
// One still going has no end yet: it runs to the last reading that says it is on battery, and says so.
|
||||
if (start !== null && last !== null) periods.push({ from: start, to: last, ongoing: true });
|
||||
return { points, onBatteryPeriods: periods };
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initUpsSync(): void {
|
||||
if (timer) return;
|
||||
const pass = () => void syncAllUps().catch((err) => console.error("[ups] pass failed:", err));
|
||||
timer = setInterval(pass, SYNC_INTERVAL_MS);
|
||||
setTimeout(pass, 9_000);
|
||||
console.log(`[ups] reading UPSs every ${SYNC_INTERVAL_MS / 1000} seconds`);
|
||||
}
|
||||
@@ -0,0 +1,539 @@
|
||||
/**
|
||||
* A UPS: what NUT and SNMP say and how it is read, what is stored and when a history point is added, what is wrong with it, the order to shut down what it carries,
|
||||
* its place in the dependency map (down while on battery, so what goes offline with it is folded under its alert), and the page's and /metrics' view of it.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import dgram from "node:dgram";
|
||||
import net from "node:net";
|
||||
import { createServer as createHttpServer, type Server } from "node:http";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { networkInterfaces, tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-ups-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, integrations, dependencies, upsState, upsReadings } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const nut = await import("../src/integrations/ups/nut.js");
|
||||
const snmp = await import("../src/integrations/ups/snmp.js");
|
||||
const { createUpsAdapter, portOf } = await import("../src/integrations/ups/adapter.js");
|
||||
const ups = await import("../src/services/upsSync.js");
|
||||
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
|
||||
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
|
||||
const deps = await import("../src/services/dependencies.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { upsRouter } = await import("../src/routes/ups.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
const { setInternalFlag } = await import("../src/services/settingsStore.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
/** An address of this machine that isn't loopback (UPS addresses are checked like the endpoint checks'), if it has one. */
|
||||
function lanAddress(): string | null {
|
||||
for (const list of Object.values(networkInterfaces())) {
|
||||
for (const i of list ?? []) if (i.family === "IPv4" && !i.internal && !i.address.startsWith("169.254.")) return i.address;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
const LAN = lanAddress();
|
||||
|
||||
const NOW = new Date("2026-10-20T12:00:00Z");
|
||||
const MIN = 60_000;
|
||||
const at = (min: number) => new Date(NOW.getTime() - min * MIN).toISOString();
|
||||
|
||||
const VARS = {
|
||||
"device.mfr": "APC",
|
||||
"device.model": "Smart-UPS 1500",
|
||||
"ups.status": "OL CHRG",
|
||||
"battery.charge": "97",
|
||||
"battery.runtime": "2040",
|
||||
"ups.load": "34",
|
||||
"input.voltage": "231.4",
|
||||
"output.voltage": "230.0",
|
||||
};
|
||||
|
||||
describe("what NUT says", () => {
|
||||
const text = `BEGIN LIST VAR ups\r\n${Object.entries(VARS).map(([k, v]) => `VAR ups ${k} "${v}"`).join("\r\n")}\r\nVAR ups ups.firmware "a \\"quoted\\" word"\r\nEND LIST VAR ups\r\n`;
|
||||
|
||||
test("the variables, with their quotes", () => {
|
||||
const vars = nut.parseNutVars(text);
|
||||
assert.equal(vars["battery.charge"], "97");
|
||||
assert.equal(vars["ups.firmware"], 'a "quoted" word');
|
||||
assert.deepEqual(nut.parseNutVars("ERR UNKNOWN-UPS\n"), {});
|
||||
});
|
||||
|
||||
test("the names of the UPSs it knows", () => {
|
||||
assert.deepEqual(nut.parseNutUpsNames('BEGIN LIST UPS\nUPS rack "In the rack"\nUPS desk "Under the desk"\nEND LIST UPS\n'), ["rack", "desk"]);
|
||||
});
|
||||
|
||||
test("online, on battery, low, to be replaced and overloaded", () => {
|
||||
const r = nut.interpretNut(nut.parseNutVars(text));
|
||||
assert.deepEqual([r.model, r.manufacturer, r.onBattery, r.lowBattery, r.charge, r.runtimeSeconds, r.loadPercent, r.inputVoltage, r.outputVoltage], ["Smart-UPS 1500", "APC", false, false, 97, 2040, 34, 231.4, 230]);
|
||||
const flags = (s: string) => nut.interpretNut({ "ups.status": s });
|
||||
assert.deepEqual([flags("OB DISCHRG").onBattery, flags("OB DISCHRG").lowBattery], [true, false]);
|
||||
assert.deepEqual([flags("OB DISCHRG LB").onBattery, flags("OB DISCHRG LB").lowBattery], [true, true]);
|
||||
assert.equal(flags("OL RB").replaceBattery, true);
|
||||
assert.equal(flags("OL OVER").overload, true);
|
||||
const empty = nut.interpretNut({});
|
||||
assert.deepEqual([empty.statusText, empty.charge, empty.model, empty.onBattery], ["unknown", null, null, false]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("asking a NUT server", { skip: LAN === null && "this machine has no address other than loopback" }, () => {
|
||||
let server: net.Server;
|
||||
let port = 0;
|
||||
const seen: string[] = [];
|
||||
|
||||
before(async () => {
|
||||
server = net.createServer((socket) => {
|
||||
let buf = "";
|
||||
socket.on("error", () => {});
|
||||
socket.on("data", (d) => {
|
||||
buf += d.toString();
|
||||
let i;
|
||||
while ((i = buf.indexOf("\n")) >= 0) {
|
||||
const line = buf.slice(0, i).trim();
|
||||
buf = buf.slice(i + 1);
|
||||
seen.push(line);
|
||||
if (line === "LIST UPS") socket.write('BEGIN LIST UPS\nUPS rack "The rack"\nEND LIST UPS\n');
|
||||
else if (line === "LIST VAR rack") socket.write(`BEGIN LIST VAR rack\n${Object.entries(VARS).map(([k, v]) => `VAR rack ${k} "${v}"`).join("\n")}\nEND LIST VAR rack\n`);
|
||||
else if (line.startsWith("LIST VAR")) socket.write("ERR UNKNOWN-UPS\n");
|
||||
else if (line === "LOGOUT") socket.end("OK Goodbye\n");
|
||||
}
|
||||
});
|
||||
});
|
||||
await new Promise<void>((r) => server.listen(0, "0.0.0.0", r));
|
||||
port = (server.address() as AddressInfo).port;
|
||||
});
|
||||
after(() => server.close());
|
||||
|
||||
test("it reads the named UPS, or the first one listed when none is named", async () => {
|
||||
const named = await nut.readNut(LAN!, port, "rack");
|
||||
assert.deepEqual([named.model, named.charge, named.onBattery], ["Smart-UPS 1500", 97, false]);
|
||||
seen.length = 0;
|
||||
const first = await nut.readNut(LAN!, port, null);
|
||||
assert.equal(first.manufacturer, "APC");
|
||||
assert.deepEqual(seen.filter((l) => l !== "LOGOUT").slice(0, 2), ["LIST UPS", "LIST VAR rack"]);
|
||||
});
|
||||
|
||||
test("an unknown UPS, a closed port and an address it won't ask say so", async () => {
|
||||
await assert.rejects(nut.readNut(LAN!, port, "nope"), /upsd says UNKNOWN-UPS/);
|
||||
await assert.rejects(nut.readNut(LAN!, 1, "rack"), /nothing is listening on port 1/);
|
||||
await assert.rejects(nut.readNut("127.0.0.1", port, "rack"));
|
||||
});
|
||||
|
||||
test("the adapter takes its port from the config, or the protocol's own, and pings", async () => {
|
||||
assert.deepEqual([portOf({ host: "x" }), portOf({ host: "x", snmp: true }), portOf({ host: "x", snmp: "true", port: "1161" }), portOf({ host: "x", port: "bad" })], [3493, 161, 1161, 3493]);
|
||||
const ok = await createUpsAdapter({ host: LAN!, port: String(port), upsName: "rack" }).ping();
|
||||
assert.equal(ok.ok, true);
|
||||
const bad = await createUpsAdapter({ host: LAN!, port: String(port), upsName: "nope" }).ping();
|
||||
assert.deepEqual([bad.ok, /UNKNOWN-UPS/.test(bad.error ?? "")], [false, true]);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── SNMP, with a small agent of the test's own ─────────────────────────────
|
||||
|
||||
/** Just enough BER to answer a GET: the test's own encoder, so what is checked is not the code's. */
|
||||
const len = (n: number) => (n < 128 ? [n] : [0x81, n]);
|
||||
const t = (tag: number, body: number[]) => [tag, ...len(body.length), ...body];
|
||||
const int = (n: number) => t(0x02, [n >> 24 & 0xff, n >> 16 & 0xff, n >> 8 & 0xff, n & 0xff]);
|
||||
const oidBytes = (oid: string) => {
|
||||
const p = oid.split(".").map(Number);
|
||||
const out = [p[0] * 40 + p[1]];
|
||||
for (const n of p.slice(2)) out.push(...(n < 128 ? [n] : [(n >> 7) | 0x80, n & 0x7f]));
|
||||
return t(0x06, out);
|
||||
};
|
||||
/** The request id and the objects asked for, read straight from the bytes. */
|
||||
function readRequest(buf: Buffer): { id: number; community: string; oids: string[] } {
|
||||
let i = 2; // 30 len
|
||||
if (buf[1] & 0x80) i = 2 + (buf[1] & 0x7f);
|
||||
const tlv = () => {
|
||||
const tag = buf[i++];
|
||||
let l = buf[i++];
|
||||
if (l & 0x80) {
|
||||
const n = l & 0x7f;
|
||||
l = 0;
|
||||
for (let k = 0; k < n; k++) l = l * 256 + buf[i++];
|
||||
}
|
||||
const start = i;
|
||||
i += l;
|
||||
return { tag, start, end: i };
|
||||
};
|
||||
tlv(); // version
|
||||
const com = tlv();
|
||||
const community = buf.subarray(com.start, com.end).toString();
|
||||
const pdu = tlv();
|
||||
i = pdu.start;
|
||||
const rid = tlv();
|
||||
let id = 0;
|
||||
for (let k = rid.start; k < rid.end; k++) id = id * 256 + buf[k];
|
||||
tlv();
|
||||
tlv();
|
||||
const list = tlv();
|
||||
i = list.start;
|
||||
const oids: string[] = [];
|
||||
while (i < list.end) {
|
||||
const vb = tlv();
|
||||
const save = i;
|
||||
i = vb.start;
|
||||
const o = tlv();
|
||||
const bytes = buf.subarray(o.start, o.end);
|
||||
const parts = [Math.floor(bytes[0] / 40), bytes[0] % 40];
|
||||
let v = 0;
|
||||
for (const b of bytes.subarray(1)) {
|
||||
v = v * 128 + (b & 0x7f);
|
||||
if (!(b & 0x80)) {
|
||||
parts.push(v);
|
||||
v = 0;
|
||||
}
|
||||
}
|
||||
oids.push(parts.join("."));
|
||||
i = save;
|
||||
}
|
||||
return { id, community, oids };
|
||||
}
|
||||
|
||||
describe("what SNMP says", () => {
|
||||
test("an object id is written as the standard says", () => {
|
||||
assert.deepEqual([...snmp.encodeOid("1.3.6.1.2.1.33.1.2.4.0")], [0x06, 0x0a, 0x2b, 0x06, 0x01, 0x02, 0x01, 0x21, 0x01, 0x02, 0x04, 0x00]);
|
||||
assert.deepEqual([...snmp.encodeOid("1.3.6.1.4.1.318.1")], [0x06, 0x08, 0x2b, 0x06, 0x01, 0x04, 0x01, 0x82, 0x3e, 0x01], "318 takes two bytes");
|
||||
});
|
||||
|
||||
test("a request names the community, an id and the objects", () => {
|
||||
const req = snmp.buildGet("public", 123456, [snmp.OIDS.chargeRemaining, snmp.OIDS.outputSource]);
|
||||
assert.equal(req[0], 0x30);
|
||||
assert.deepEqual(readRequest(req), { id: 123456, community: "public", oids: [snmp.OIDS.chargeRemaining, snmp.OIDS.outputSource] });
|
||||
});
|
||||
|
||||
/** An answer as an agent would write it. */
|
||||
const answer = (id: number, binds: [string, number[]][]) => Buffer.from(t(0x30, [...int(1), ...t(0x04, [...Buffer.from("public")]), ...t(0xa2, [...int(id), ...int(0), ...int(0), ...t(0x30, binds.flatMap(([o, v]) => t(0x30, [...oidBytes(o), ...v])))])]));
|
||||
|
||||
test("an answer's numbers, text, counters and objects the device doesn't have", () => {
|
||||
const parsed = snmp.parseResponse(
|
||||
answer(77, [
|
||||
[snmp.OIDS.chargeRemaining, int(87)],
|
||||
[snmp.OIDS.model, t(0x04, [...Buffer.from("Back-UPS ES 700")])],
|
||||
[snmp.OIDS.secondsOnBattery, t(0x42, [0x01, 0x2c])],
|
||||
[snmp.OIDS.outputLoad, t(0x80, [])],
|
||||
[snmp.OIDS.inputVoltage, int(-5)],
|
||||
]),
|
||||
);
|
||||
assert.deepEqual([parsed.requestId, parsed.errorStatus], [77, 0]);
|
||||
assert.deepEqual([parsed.values.get(snmp.OIDS.chargeRemaining), parsed.values.get(snmp.OIDS.model), parsed.values.get(snmp.OIDS.secondsOnBattery), parsed.values.get(snmp.OIDS.outputLoad), parsed.values.get(snmp.OIDS.inputVoltage)], [87, "Back-UPS ES 700", 300, null, -5]);
|
||||
assert.throws(() => snmp.parseResponse(Buffer.from([0x30, 0x03, 0x02, 0x01])), /cut short|isn't/);
|
||||
assert.throws(() => snmp.parseResponse(Buffer.from("not snmp at all")));
|
||||
});
|
||||
|
||||
test("the UPS-MIB: on mains, on battery, low", () => {
|
||||
const m = (entries: [string, number | string | null][]) => new Map<string, number | string | null>(entries);
|
||||
const mains = snmp.interpretSnmp(m([[snmp.OIDS.outputSource, 3], [snmp.OIDS.batteryStatus, 2], [snmp.OIDS.chargeRemaining, 100], [snmp.OIDS.minutesRemaining, 42], [snmp.OIDS.outputLoad, 18], [snmp.OIDS.model, "SMX1500"], [snmp.OIDS.manufacturer, "APC"]]));
|
||||
assert.deepEqual([mains.onBattery, mains.lowBattery, mains.charge, mains.runtimeSeconds, mains.loadPercent, mains.model, mains.statusText], [false, false, 100, 2520, 18, "SMX1500", "normal, battery normal"]);
|
||||
const out = snmp.interpretSnmp(m([[snmp.OIDS.outputSource, 5], [snmp.OIDS.batteryStatus, 3]]));
|
||||
assert.deepEqual([out.onBattery, out.lowBattery, out.statusText], [true, true, "on battery, battery low"]);
|
||||
assert.equal(snmp.interpretSnmp(m([[snmp.OIDS.secondsOnBattery, 90]])).onBattery, true, "no output source, but seconds on battery");
|
||||
assert.equal(snmp.interpretSnmp(m([])).statusText, "unknown");
|
||||
});
|
||||
|
||||
describe("asking an agent", { skip: LAN === null && "this machine has no address other than loopback" }, () => {
|
||||
let agent: dgram.Socket;
|
||||
let port = 0;
|
||||
let community = "public";
|
||||
let silent = false;
|
||||
let values: [string, number[]][] = [];
|
||||
|
||||
before(async () => {
|
||||
agent = dgram.createSocket("udp4");
|
||||
agent.on("message", (msg, rinfo) => {
|
||||
if (silent) return;
|
||||
const req = readRequest(msg);
|
||||
if (req.community !== community) return; // a wrong community gets no answer
|
||||
const binds = req.oids.map((o) => [o, values.find(([k]) => k === o)?.[1] ?? t(0x80, [])] as [string, number[]]);
|
||||
agent.send(answer(req.id, binds), rinfo.port, rinfo.address);
|
||||
});
|
||||
await new Promise<void>((r) => agent.bind(0, "0.0.0.0", r));
|
||||
port = agent.address().port;
|
||||
});
|
||||
after(() => agent.close());
|
||||
|
||||
test("it reads the UPS", async () => {
|
||||
values = [[snmp.OIDS.outputSource, int(5)], [snmp.OIDS.batteryStatus, int(2)], [snmp.OIDS.chargeRemaining, int(64)], [snmp.OIDS.minutesRemaining, int(18)], [snmp.OIDS.model, t(0x04, [...Buffer.from("Eaton 5P")])]];
|
||||
const r = await snmp.readSnmp(LAN!, port, "public");
|
||||
assert.deepEqual([r.onBattery, r.charge, r.runtimeSeconds, r.model], [true, 64, 1080, "Eaton 5P"]);
|
||||
const viaAdapter = await createUpsAdapter({ host: LAN!, snmp: true, port: String(port) }).read();
|
||||
assert.equal(viaAdapter.charge, 64, "the adapter reads over SNMP when asked, with the community public when none is given");
|
||||
});
|
||||
|
||||
test("a device with none of the UPS objects is said not to be a UPS", async () => {
|
||||
values = [];
|
||||
await assert.rejects(snmp.readSnmp(LAN!, port, "public"), /none of the standard UPS objects/);
|
||||
});
|
||||
});
|
||||
|
||||
test("an address it won't ask is refused before anything is sent", async () => {
|
||||
await assert.rejects(snmp.readSnmp("127.0.0.1", 161, "public"));
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Stored, judged and drawn ───────────────────────────────────────────────
|
||||
|
||||
describe("what is stored", () => {
|
||||
let id = 0;
|
||||
const reading = (extra: Partial<import("../src/integrations/ups/types.js").UpsReading> = {}) => ({ model: "SMX", manufacturer: "APC", statusText: "OL", onBattery: false, lowBattery: false, replaceBattery: false, overload: false, charge: 100, runtimeSeconds: 1800, loadPercent: 30, inputVoltage: 230, outputVoltage: 230, ...extra });
|
||||
|
||||
before(async () => {
|
||||
id = (await db.insert(integrations).values({ type: "ups", name: "Rack UPS", baseUrl: "", config: JSON.stringify({ host: "nut.lan" }) }).returning({ id: integrations.id }))[0].id;
|
||||
});
|
||||
|
||||
test("the state is replaced at each read, and when mains power was lost is kept until it is back", async () => {
|
||||
await ups.storeReading(id, reading(), new Date(NOW.getTime() - 30 * MIN));
|
||||
let [s] = await db.select().from(upsState).where(eq(upsState.integrationId, id));
|
||||
assert.deepEqual([s.onBattery, s.onBatterySince, s.charge], [false, null, 100]);
|
||||
await ups.storeReading(id, reading({ onBattery: true, statusText: "OB DISCHRG", charge: 95 }), new Date(NOW.getTime() - 10 * MIN));
|
||||
await ups.storeReading(id, reading({ onBattery: true, statusText: "OB DISCHRG", charge: 80 }), new Date(NOW.getTime() - 5 * MIN));
|
||||
[s] = await db.select().from(upsState).where(eq(upsState.integrationId, id));
|
||||
assert.deepEqual([s.onBattery, s.onBatterySince, s.charge, s.statusText], [true, at(10), 80, "OB DISCHRG"]);
|
||||
await ups.storeReading(id, reading({ charge: 82 }), NOW);
|
||||
[s] = await db.select().from(upsState).where(eq(upsState.integrationId, id));
|
||||
assert.deepEqual([s.onBattery, s.onBatterySince], [false, null]);
|
||||
});
|
||||
|
||||
test("a point is added to the history every five minutes, every minute on battery, and when it changes", async () => {
|
||||
const rows = await db.select().from(upsReadings).where(eq(upsReadings.integrationId, id)).orderBy(upsReadings.at);
|
||||
// 30 min ago (first), 10 and 5 min ago (on battery), now (back on mains: a change)
|
||||
assert.deepEqual(rows.map((r) => r.at), [at(30), at(10), at(5), NOW.toISOString()]);
|
||||
await ups.storeReading(id, reading(), new Date(NOW.getTime() + 1 * MIN));
|
||||
await ups.storeReading(id, reading(), new Date(NOW.getTime() + 3 * MIN));
|
||||
assert.equal((await db.select().from(upsReadings).where(eq(upsReadings.integrationId, id))).length, 4, "mains, a minute and three minutes later: nothing new");
|
||||
await ups.storeReading(id, reading(), new Date(NOW.getTime() + 6 * MIN));
|
||||
assert.equal((await db.select().from(upsReadings).where(eq(upsReadings.integrationId, id))).length, 5);
|
||||
});
|
||||
|
||||
test("history older than 35 days is dropped", async () => {
|
||||
await db.insert(upsReadings).values({ integrationId: id, at: new Date(NOW.getTime() - 40 * 86_400_000).toISOString(), charge: 100, onBattery: false });
|
||||
await ups.storeReading(id, reading(), new Date(NOW.getTime() + 12 * MIN));
|
||||
assert.equal((await db.select().from(upsReadings).where(eq(upsReadings.integrationId, id))).every((r) => Date.parse(r.at) > NOW.getTime() - 36 * 86_400_000), true);
|
||||
});
|
||||
|
||||
test("a read that fails is said so, and keeps when the last good one was", async () => {
|
||||
const ok = await ups.syncUps(id, new Date(NOW.getTime() + 20 * MIN), async () => reading({ charge: 99 }));
|
||||
assert.deepEqual([ok.ok, ok.at], [true, new Date(NOW.getTime() + 20 * MIN).toISOString()]);
|
||||
const failed = await ups.syncUps(id, new Date(NOW.getTime() + 25 * MIN), async () => {
|
||||
throw new Error("no answer within 5 s");
|
||||
});
|
||||
assert.deepEqual([failed.ok, failed.error, failed.at], [false, "no answer within 5 s", ok.at]);
|
||||
const statuses = await ups.upsSyncStatuses();
|
||||
assert.equal(statuses[id].ok, false);
|
||||
assert.equal(ups.isUpsFresh(ok, Date.parse(ok.at) + 4 * MIN), true);
|
||||
assert.equal(ups.isUpsFresh(ok, Date.parse(ok.at) + 6 * MIN), false, "after five minutes it is no longer believed");
|
||||
assert.equal(ups.isUpsFresh(failed, Date.parse(ok.at)), false, "and a failed read is never fresh");
|
||||
const wrong = await ups.syncUps(999999, NOW, async () => reading());
|
||||
assert.equal(wrong.ok, false);
|
||||
});
|
||||
|
||||
test("the history is bucketed into averages and the stretches on battery are found", () => {
|
||||
const rows = [
|
||||
{ at: at(60), charge: 100, runtimeSeconds: 1800, loadPercent: 30, inputVoltage: 230, onBattery: false },
|
||||
{ at: at(50), charge: 100, runtimeSeconds: 1800, loadPercent: 40, inputVoltage: 230, onBattery: false },
|
||||
{ at: at(40), charge: 90, runtimeSeconds: 1200, loadPercent: 35, inputVoltage: 0, onBattery: true },
|
||||
{ at: at(35), charge: 70, runtimeSeconds: 800, loadPercent: 35, inputVoltage: 0, onBattery: true },
|
||||
{ at: at(30), charge: 70, runtimeSeconds: 800, loadPercent: 35, inputVoltage: 231, onBattery: false },
|
||||
{ at: at(5), charge: 98, runtimeSeconds: null, loadPercent: null, inputVoltage: 231, onBattery: true },
|
||||
];
|
||||
const r = ups.bucketUps(rows, NOW.getTime() - 61 * MIN, NOW.getTime(), 2);
|
||||
assert.equal(r.points.length, 2);
|
||||
assert.deepEqual([r.points[0].loadPercent, r.points[0].onBattery, r.points[1].onBattery], [35, true, true]);
|
||||
assert.deepEqual(r.onBatteryPeriods, [{ from: at(40), to: at(30), ongoing: false }, { from: at(5), to: at(5), ongoing: true }], "one that ended, and one still going");
|
||||
assert.deepEqual(ups.bucketUps([], 0, 10, 5), { points: [], onBatteryPeriods: [] });
|
||||
});
|
||||
});
|
||||
|
||||
describe("what is wrong", () => {
|
||||
const base = { onBattery: false, lowBattery: false, replaceBattery: false, overload: false, charge: 100, runtimeSeconds: 1800, loadPercent: 30, onBatterySince: null as string | null };
|
||||
const keys = (s: Partial<typeof base>) => ups.upsProblems(7, "Rack UPS", { ...base, ...s }, NOW).map((p) => p.key);
|
||||
|
||||
test("nothing is wrong with a UPS on mains", () => {
|
||||
assert.deepEqual(keys({}), []);
|
||||
});
|
||||
|
||||
test("mains power lost says for how long, how much is left and how full the battery is", () => {
|
||||
const [p] = ups.upsProblems(7, "Rack UPS", { ...base, onBattery: true, onBatterySince: at(4), charge: 84, runtimeSeconds: 1380 }, NOW);
|
||||
assert.deepEqual([p.key, p.severity, p.kind], ["ups-battery:7", "critical", "battery"]);
|
||||
assert.equal(p.message, "Rack UPS: mains power has failed — running on battery (for 4 min, 23 min left, 84 %)");
|
||||
assert.match(ups.upsProblems(7, "U", { ...base, onBattery: true, charge: null, runtimeSeconds: null }, NOW)[0].message, /^U: mains power has failed — running on battery$/, "with nothing more to say it says no more");
|
||||
});
|
||||
|
||||
test("a nearly empty battery, one to be replaced, and a load near the limit are each their own", () => {
|
||||
assert.deepEqual(keys({ onBattery: true, lowBattery: true }), ["ups-battery:7", "ups-low:7"]);
|
||||
assert.deepEqual(keys({ replaceBattery: true }), ["ups-replace:7"]);
|
||||
assert.deepEqual(keys({ loadPercent: 89 }), []);
|
||||
assert.deepEqual(keys({ loadPercent: 90 }), ["ups-load:7"]);
|
||||
assert.deepEqual(keys({ overload: true, loadPercent: null }), ["ups-load:7"]);
|
||||
assert.equal(ups.upsProblems(7, "R", { ...base, replaceBattery: true }, NOW)[0].severity, "warning");
|
||||
});
|
||||
});
|
||||
|
||||
describe("what it carries", () => {
|
||||
test("what nothing else needs is shut down first, then what that needed, so nothing goes while something still depends on it", () => {
|
||||
// guest → host → ups; nas → ups; app → guest, nas
|
||||
const edges = new Map<string, string[]>([
|
||||
["server:host", ["integration:1"]],
|
||||
["server:nas", ["integration:1"]],
|
||||
["server:guest", ["server:host"]],
|
||||
["server:app", ["server:guest", "server:nas"]],
|
||||
["server:other", ["integration:2"]],
|
||||
]);
|
||||
const { order, looped } = ups.shutdownOrder(edges, "integration:1");
|
||||
assert.equal(looped, false);
|
||||
assert.deepEqual(order.map((o) => [o.key, o.step]), [["server:app", 1], ["server:guest", 2], ["server:nas", 2], ["server:host", 3]]);
|
||||
assert.deepEqual(ups.shutdownOrder(edges, "integration:2").order, [{ key: "server:other", step: 1 }]);
|
||||
assert.deepEqual(ups.shutdownOrder(edges, "integration:9"), { order: [], looped: false });
|
||||
});
|
||||
|
||||
test("things that need each other don't hang it", () => {
|
||||
const edges = new Map<string, string[]>([["server:a", ["integration:1", "server:b"]], ["server:b", ["server:a"]]]);
|
||||
const r = ups.shutdownOrder(edges, "integration:1");
|
||||
assert.equal(r.looped, true);
|
||||
assert.equal(r.order.length, 2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("in the map, in alerts, in the health pass and on the page", () => {
|
||||
let upsId = 0;
|
||||
let hostId = 0;
|
||||
let guestId = 0;
|
||||
let aloneId = 0;
|
||||
let app: Server;
|
||||
let base = "";
|
||||
const state = (extra: Partial<typeof upsState.$inferInsert> = {}) => ({ integrationId: upsId, model: "SMX", manufacturer: "APC", statusText: "OB DISCHRG", onBattery: true, lowBattery: false, replaceBattery: false, overload: false, charge: 71, runtimeSeconds: 1500, loadPercent: 44, inputVoltage: 0, outputVoltage: 230, onBatterySince: at(6), updatedAt: new Date().toISOString(), ...extra });
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
|
||||
upsId = (await db.insert(integrations).values({ type: "ups", name: "Cupboard UPS", baseUrl: "", config: JSON.stringify({ host: "nut.lan" }) }).returning({ id: integrations.id }))[0].id;
|
||||
const gone = new Date(Date.now() - 3 * 3_600_000).toISOString();
|
||||
const mk = async (name: string, lastSeenAt: string) => (await db.insert(servers).values({ name, apiTokenHash: name, apiTokenPrefix: name, lastSeenAt }).returning({ id: servers.id }))[0].id;
|
||||
hostId = await mk("pve", gone);
|
||||
guestId = await mk("wiki", gone);
|
||||
aloneId = await mk("printer", gone);
|
||||
await db.insert(dependencies).values([
|
||||
{ fromType: "server", fromId: hostId, toType: "integration", toId: upsId },
|
||||
{ fromType: "server", fromId: guestId, toType: "server", toId: hostId },
|
||||
]);
|
||||
await db.insert(upsState).values(state());
|
||||
await setInternalFlag("upsSync", JSON.stringify({ [upsId]: { at: new Date().toISOString(), ok: true, error: null } }));
|
||||
|
||||
const a = express();
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/ups", upsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/ups`;
|
||||
});
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
test("a UPS on battery is down, so the servers that go offline with it are folded under it", async () => {
|
||||
assert.deepEqual(await ups.upsOnBatteryIds(), [upsId]);
|
||||
const cover = await deps.dependencyCover([hostId, guestId, aloneId], []);
|
||||
assert.equal(cover.cause.get(`server:${hostId}`), `integration:${upsId}`);
|
||||
assert.equal(cover.cause.get(`server:${guestId}`), `integration:${upsId}`, "through the host it runs on");
|
||||
assert.equal(cover.cause.has(`server:${aloneId}`), false, "one that doesn't need it stands alone");
|
||||
assert.deepEqual((cover.taken.get(`integration:${upsId}`) ?? []).sort(), ["pve", "wiki"]);
|
||||
assert.equal(deps.alertIdOfNode(`integration:${upsId}`, cover.graph), `monitoring:ups-battery:${upsId}`);
|
||||
// Back on mains, it stops explaining anything.
|
||||
await db.update(upsState).set({ onBattery: false }).where(eq(upsState.integrationId, upsId));
|
||||
assert.deepEqual(await ups.upsOnBatteryIds(), []);
|
||||
assert.equal((await deps.dependencyCover([hostId], [])).cause.size, 0);
|
||||
// And a reading that is no longer believed doesn't either.
|
||||
await db.update(upsState).set({ onBattery: true, updatedAt: at(30) }).where(eq(upsState.integrationId, upsId));
|
||||
assert.deepEqual(await ups.upsOnBatteryIds(), []);
|
||||
await db.update(upsState).set({ updatedAt: new Date().toISOString() }).where(eq(upsState.integrationId, upsId));
|
||||
});
|
||||
|
||||
test("the health pass judges it from what is stored, and holds it when the last read is old", async () => {
|
||||
let { snapshot, held } = await collectSnapshot();
|
||||
assert.deepEqual(snapshot.ups?.map((u) => u.integrationName), ["Cupboard UPS"]);
|
||||
assert.equal(held.has(`ups:${upsId}`), false);
|
||||
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
|
||||
const c = list.find((x) => x.key === `ups-battery:${upsId}`)!;
|
||||
assert.deepEqual([c.severity, c.source], ["critical", `ups:${upsId}`]);
|
||||
assert.match(c.message, /Cupboard UPS: mains power has failed/);
|
||||
assert.equal(subjectOfConditionKey(c.key), `integration:${upsId}`, "maintenance on it silences its alerts");
|
||||
for (const k of ["ups-low", "ups-replace", "ups-load"]) assert.equal(subjectOfConditionKey(`${k}:${upsId}`), `integration:${upsId}`);
|
||||
await setInternalFlag("upsSync", JSON.stringify({ [upsId]: { at: at(30), ok: true, error: null } }));
|
||||
({ snapshot, held } = await collectSnapshot());
|
||||
assert.deepEqual([snapshot.ups ?? [], held.has(`ups:${upsId}`)], [[], true]);
|
||||
await setInternalFlag("upsSync", JSON.stringify({ [upsId]: { at: new Date().toISOString(), ok: true, error: null } }));
|
||||
});
|
||||
|
||||
test("the Alerts page shows it, with what goes down with it, and the offline servers say it is why", async () => {
|
||||
invalidateAlertsCache();
|
||||
const report = await getAlerts(true);
|
||||
const a = report.alerts.find((x) => x.id === `monitoring:ups-battery:${upsId}`)!;
|
||||
assert.ok(a, "the alert is there");
|
||||
assert.deepEqual([a.severity, a.category, a.source, a.link, a.target?.type, a.target?.id], ["critical", "monitoring", "UPS", "/ups", "integration", upsId]);
|
||||
assert.deepEqual((a.affects ?? []).sort(), ["pve", "wiki"]);
|
||||
const host = report.alerts.find((x) => x.id === `offline:offline:server:${hostId}`)!;
|
||||
assert.deepEqual([host.causedBy?.name, host.causedBy?.alertId, host.silenced], ["Cupboard UPS", `monitoring:ups-battery:${upsId}`, true]);
|
||||
const alone = report.alerts.find((x) => x.id === `offline:offline:server:${aloneId}`)!;
|
||||
assert.equal(alone.causedBy, undefined);
|
||||
});
|
||||
|
||||
test("the page lists each UPS, how it is, what is on it in shutdown order, and its history", async () => {
|
||||
const res = await fetch(base, { headers: { connection: "close" } });
|
||||
assert.equal(res.status, 200, "everyone signed in can look");
|
||||
const body = (await res.json()) as any;
|
||||
const u = body.ups.find((x: any) => x.id === upsId);
|
||||
assert.deepEqual([u.name, u.protocol, u.fresh, u.state.onBattery, u.state.charge], ["Cupboard UPS", "nut", true, true, 71]);
|
||||
assert.deepEqual(u.problems.map((p: any) => p.key), [`ups-battery:${upsId}`]);
|
||||
assert.deepEqual(u.carries.map((c: any) => [c.name, c.step, c.type]), [["wiki", 1, "server"], ["pve", 2, "server"]]);
|
||||
assert.equal(JSON.stringify(body).includes("nut.lan"), false, "no address leaves the server");
|
||||
const hist = (await (await fetch(`${base}/${upsId}/history?days=1`, { headers: { connection: "close" } })).json()) as any;
|
||||
assert.equal(hist.ups.name, "Cupboard UPS");
|
||||
assert.ok(Array.isArray(hist.points) && Array.isArray(hist.onBatteryPeriods));
|
||||
assert.equal((await fetch(`${base}/999/history`, { headers: { connection: "close" } })).status, 404);
|
||||
assert.equal((await fetch(`${base}/abc/history`, { headers: { connection: "close" } })).status, 400);
|
||||
});
|
||||
|
||||
test("/metrics carries it while it is believed, and not after", async () => {
|
||||
let text = await buildMetrics();
|
||||
assert.match(text, /homelab_ups_on_battery\{name="Cupboard UPS"\} 1/);
|
||||
assert.match(text, /homelab_ups_battery_charge_percent\{name="Cupboard UPS"\} 71/);
|
||||
assert.match(text, /homelab_ups_runtime_seconds\{name="Cupboard UPS"\} 1500/);
|
||||
await setInternalFlag("upsSync", JSON.stringify({ [upsId]: { at: at(30), ok: true, error: null } }));
|
||||
text = await buildMetrics();
|
||||
assert.doesNotMatch(text, /homelab_ups_on_battery/);
|
||||
});
|
||||
|
||||
test("a UPS is added through the integrations form like any other, with its host the only thing needed", async () => {
|
||||
const { INTEGRATION_FIELDS, validateIntegrationConfig, splitIntegrationConfig } = await import("../src/integrations/fieldSchemas.js");
|
||||
assert.ok(INTEGRATION_FIELDS.ups);
|
||||
assert.deepEqual(validateIntegrationConfig("ups", {}), ["host"]);
|
||||
assert.deepEqual(validateIntegrationConfig("ups", { host: "nut.lan" }), []);
|
||||
const { secretFields, nonSecretFields } = splitIntegrationConfig("ups", { host: "nut.lan", snmp: true, community: "s3cret", upsName: "rack" });
|
||||
assert.deepEqual([Object.keys(secretFields), Object.keys(nonSecretFields).sort()], [["community"], ["host", "snmp", "upsName"]]);
|
||||
});
|
||||
});
|
||||
|
||||
// keep the unused-import checker quiet about the helper servers this file declares for readability
|
||||
void createHttpServer;
|
||||
Reference in new issue
Block a user