A UPS in the map: read over NUT or SNMP, alerts, shutdown order
New UPS integration (NUT or SNMP v2c, read only). Monitoring -> UPS shows charge, runtime, load and mains voltage with history and the times mains power was lost. Mains lost and a nearly empty battery are critical alerts; battery to be replaced and a heavy load are warnings. The UPS is a node in the dependency map and counts as down while on battery, so servers that go offline with it are folded under its alert; the page lists what is on it in the order to shut it down. /metrics series and docs added. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3cb3e4988f
commit
232a5661ab
38 files changed
+8049
-16
No files matched your search
@@ -0,0 +1,539 @@
|
||||
/**
|
||||
* A UPS: what NUT and SNMP say and how it is read, what is stored and when a history point is added, what is wrong with it, the order to shut down what it carries,
|
||||
* its place in the dependency map (down while on battery, so what goes offline with it is folded under its alert), and the page's and /metrics' view of it.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import dgram from "node:dgram";
|
||||
import net from "node:net";
|
||||
import { createServer as createHttpServer, type Server } from "node:http";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { networkInterfaces, tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-ups-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, integrations, dependencies, upsState, upsReadings } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const nut = await import("../src/integrations/ups/nut.js");
|
||||
const snmp = await import("../src/integrations/ups/snmp.js");
|
||||
const { createUpsAdapter, portOf } = await import("../src/integrations/ups/adapter.js");
|
||||
const ups = await import("../src/services/upsSync.js");
|
||||
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
|
||||
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
|
||||
const deps = await import("../src/services/dependencies.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { upsRouter } = await import("../src/routes/ups.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
const { setInternalFlag } = await import("../src/services/settingsStore.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
/** An address of this machine that isn't loopback (UPS addresses are checked like the endpoint checks'), if it has one. */
|
||||
function lanAddress(): string | null {
|
||||
for (const list of Object.values(networkInterfaces())) {
|
||||
for (const i of list ?? []) if (i.family === "IPv4" && !i.internal && !i.address.startsWith("169.254.")) return i.address;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
const LAN = lanAddress();
|
||||
|
||||
const NOW = new Date("2026-10-20T12:00:00Z");
|
||||
const MIN = 60_000;
|
||||
const at = (min: number) => new Date(NOW.getTime() - min * MIN).toISOString();
|
||||
|
||||
const VARS = {
|
||||
"device.mfr": "APC",
|
||||
"device.model": "Smart-UPS 1500",
|
||||
"ups.status": "OL CHRG",
|
||||
"battery.charge": "97",
|
||||
"battery.runtime": "2040",
|
||||
"ups.load": "34",
|
||||
"input.voltage": "231.4",
|
||||
"output.voltage": "230.0",
|
||||
};
|
||||
|
||||
describe("what NUT says", () => {
|
||||
const text = `BEGIN LIST VAR ups\r\n${Object.entries(VARS).map(([k, v]) => `VAR ups ${k} "${v}"`).join("\r\n")}\r\nVAR ups ups.firmware "a \\"quoted\\" word"\r\nEND LIST VAR ups\r\n`;
|
||||
|
||||
test("the variables, with their quotes", () => {
|
||||
const vars = nut.parseNutVars(text);
|
||||
assert.equal(vars["battery.charge"], "97");
|
||||
assert.equal(vars["ups.firmware"], 'a "quoted" word');
|
||||
assert.deepEqual(nut.parseNutVars("ERR UNKNOWN-UPS\n"), {});
|
||||
});
|
||||
|
||||
test("the names of the UPSs it knows", () => {
|
||||
assert.deepEqual(nut.parseNutUpsNames('BEGIN LIST UPS\nUPS rack "In the rack"\nUPS desk "Under the desk"\nEND LIST UPS\n'), ["rack", "desk"]);
|
||||
});
|
||||
|
||||
test("online, on battery, low, to be replaced and overloaded", () => {
|
||||
const r = nut.interpretNut(nut.parseNutVars(text));
|
||||
assert.deepEqual([r.model, r.manufacturer, r.onBattery, r.lowBattery, r.charge, r.runtimeSeconds, r.loadPercent, r.inputVoltage, r.outputVoltage], ["Smart-UPS 1500", "APC", false, false, 97, 2040, 34, 231.4, 230]);
|
||||
const flags = (s: string) => nut.interpretNut({ "ups.status": s });
|
||||
assert.deepEqual([flags("OB DISCHRG").onBattery, flags("OB DISCHRG").lowBattery], [true, false]);
|
||||
assert.deepEqual([flags("OB DISCHRG LB").onBattery, flags("OB DISCHRG LB").lowBattery], [true, true]);
|
||||
assert.equal(flags("OL RB").replaceBattery, true);
|
||||
assert.equal(flags("OL OVER").overload, true);
|
||||
const empty = nut.interpretNut({});
|
||||
assert.deepEqual([empty.statusText, empty.charge, empty.model, empty.onBattery], ["unknown", null, null, false]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("asking a NUT server", { skip: LAN === null && "this machine has no address other than loopback" }, () => {
|
||||
let server: net.Server;
|
||||
let port = 0;
|
||||
const seen: string[] = [];
|
||||
|
||||
before(async () => {
|
||||
server = net.createServer((socket) => {
|
||||
let buf = "";
|
||||
socket.on("error", () => {});
|
||||
socket.on("data", (d) => {
|
||||
buf += d.toString();
|
||||
let i;
|
||||
while ((i = buf.indexOf("\n")) >= 0) {
|
||||
const line = buf.slice(0, i).trim();
|
||||
buf = buf.slice(i + 1);
|
||||
seen.push(line);
|
||||
if (line === "LIST UPS") socket.write('BEGIN LIST UPS\nUPS rack "The rack"\nEND LIST UPS\n');
|
||||
else if (line === "LIST VAR rack") socket.write(`BEGIN LIST VAR rack\n${Object.entries(VARS).map(([k, v]) => `VAR rack ${k} "${v}"`).join("\n")}\nEND LIST VAR rack\n`);
|
||||
else if (line.startsWith("LIST VAR")) socket.write("ERR UNKNOWN-UPS\n");
|
||||
else if (line === "LOGOUT") socket.end("OK Goodbye\n");
|
||||
}
|
||||
});
|
||||
});
|
||||
await new Promise<void>((r) => server.listen(0, "0.0.0.0", r));
|
||||
port = (server.address() as AddressInfo).port;
|
||||
});
|
||||
after(() => server.close());
|
||||
|
||||
test("it reads the named UPS, or the first one listed when none is named", async () => {
|
||||
const named = await nut.readNut(LAN!, port, "rack");
|
||||
assert.deepEqual([named.model, named.charge, named.onBattery], ["Smart-UPS 1500", 97, false]);
|
||||
seen.length = 0;
|
||||
const first = await nut.readNut(LAN!, port, null);
|
||||
assert.equal(first.manufacturer, "APC");
|
||||
assert.deepEqual(seen.filter((l) => l !== "LOGOUT").slice(0, 2), ["LIST UPS", "LIST VAR rack"]);
|
||||
});
|
||||
|
||||
test("an unknown UPS, a closed port and an address it won't ask say so", async () => {
|
||||
await assert.rejects(nut.readNut(LAN!, port, "nope"), /upsd says UNKNOWN-UPS/);
|
||||
await assert.rejects(nut.readNut(LAN!, 1, "rack"), /nothing is listening on port 1/);
|
||||
await assert.rejects(nut.readNut("127.0.0.1", port, "rack"));
|
||||
});
|
||||
|
||||
test("the adapter takes its port from the config, or the protocol's own, and pings", async () => {
|
||||
assert.deepEqual([portOf({ host: "x" }), portOf({ host: "x", snmp: true }), portOf({ host: "x", snmp: "true", port: "1161" }), portOf({ host: "x", port: "bad" })], [3493, 161, 1161, 3493]);
|
||||
const ok = await createUpsAdapter({ host: LAN!, port: String(port), upsName: "rack" }).ping();
|
||||
assert.equal(ok.ok, true);
|
||||
const bad = await createUpsAdapter({ host: LAN!, port: String(port), upsName: "nope" }).ping();
|
||||
assert.deepEqual([bad.ok, /UNKNOWN-UPS/.test(bad.error ?? "")], [false, true]);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── SNMP, with a small agent of the test's own ─────────────────────────────
|
||||
|
||||
/** Just enough BER to answer a GET: the test's own encoder, so what is checked is not the code's. */
|
||||
const len = (n: number) => (n < 128 ? [n] : [0x81, n]);
|
||||
const t = (tag: number, body: number[]) => [tag, ...len(body.length), ...body];
|
||||
const int = (n: number) => t(0x02, [n >> 24 & 0xff, n >> 16 & 0xff, n >> 8 & 0xff, n & 0xff]);
|
||||
const oidBytes = (oid: string) => {
|
||||
const p = oid.split(".").map(Number);
|
||||
const out = [p[0] * 40 + p[1]];
|
||||
for (const n of p.slice(2)) out.push(...(n < 128 ? [n] : [(n >> 7) | 0x80, n & 0x7f]));
|
||||
return t(0x06, out);
|
||||
};
|
||||
/** The request id and the objects asked for, read straight from the bytes. */
|
||||
function readRequest(buf: Buffer): { id: number; community: string; oids: string[] } {
|
||||
let i = 2; // 30 len
|
||||
if (buf[1] & 0x80) i = 2 + (buf[1] & 0x7f);
|
||||
const tlv = () => {
|
||||
const tag = buf[i++];
|
||||
let l = buf[i++];
|
||||
if (l & 0x80) {
|
||||
const n = l & 0x7f;
|
||||
l = 0;
|
||||
for (let k = 0; k < n; k++) l = l * 256 + buf[i++];
|
||||
}
|
||||
const start = i;
|
||||
i += l;
|
||||
return { tag, start, end: i };
|
||||
};
|
||||
tlv(); // version
|
||||
const com = tlv();
|
||||
const community = buf.subarray(com.start, com.end).toString();
|
||||
const pdu = tlv();
|
||||
i = pdu.start;
|
||||
const rid = tlv();
|
||||
let id = 0;
|
||||
for (let k = rid.start; k < rid.end; k++) id = id * 256 + buf[k];
|
||||
tlv();
|
||||
tlv();
|
||||
const list = tlv();
|
||||
i = list.start;
|
||||
const oids: string[] = [];
|
||||
while (i < list.end) {
|
||||
const vb = tlv();
|
||||
const save = i;
|
||||
i = vb.start;
|
||||
const o = tlv();
|
||||
const bytes = buf.subarray(o.start, o.end);
|
||||
const parts = [Math.floor(bytes[0] / 40), bytes[0] % 40];
|
||||
let v = 0;
|
||||
for (const b of bytes.subarray(1)) {
|
||||
v = v * 128 + (b & 0x7f);
|
||||
if (!(b & 0x80)) {
|
||||
parts.push(v);
|
||||
v = 0;
|
||||
}
|
||||
}
|
||||
oids.push(parts.join("."));
|
||||
i = save;
|
||||
}
|
||||
return { id, community, oids };
|
||||
}
|
||||
|
||||
describe("what SNMP says", () => {
|
||||
test("an object id is written as the standard says", () => {
|
||||
assert.deepEqual([...snmp.encodeOid("1.3.6.1.2.1.33.1.2.4.0")], [0x06, 0x0a, 0x2b, 0x06, 0x01, 0x02, 0x01, 0x21, 0x01, 0x02, 0x04, 0x00]);
|
||||
assert.deepEqual([...snmp.encodeOid("1.3.6.1.4.1.318.1")], [0x06, 0x08, 0x2b, 0x06, 0x01, 0x04, 0x01, 0x82, 0x3e, 0x01], "318 takes two bytes");
|
||||
});
|
||||
|
||||
test("a request names the community, an id and the objects", () => {
|
||||
const req = snmp.buildGet("public", 123456, [snmp.OIDS.chargeRemaining, snmp.OIDS.outputSource]);
|
||||
assert.equal(req[0], 0x30);
|
||||
assert.deepEqual(readRequest(req), { id: 123456, community: "public", oids: [snmp.OIDS.chargeRemaining, snmp.OIDS.outputSource] });
|
||||
});
|
||||
|
||||
/** An answer as an agent would write it. */
|
||||
const answer = (id: number, binds: [string, number[]][]) => Buffer.from(t(0x30, [...int(1), ...t(0x04, [...Buffer.from("public")]), ...t(0xa2, [...int(id), ...int(0), ...int(0), ...t(0x30, binds.flatMap(([o, v]) => t(0x30, [...oidBytes(o), ...v])))])]));
|
||||
|
||||
test("an answer's numbers, text, counters and objects the device doesn't have", () => {
|
||||
const parsed = snmp.parseResponse(
|
||||
answer(77, [
|
||||
[snmp.OIDS.chargeRemaining, int(87)],
|
||||
[snmp.OIDS.model, t(0x04, [...Buffer.from("Back-UPS ES 700")])],
|
||||
[snmp.OIDS.secondsOnBattery, t(0x42, [0x01, 0x2c])],
|
||||
[snmp.OIDS.outputLoad, t(0x80, [])],
|
||||
[snmp.OIDS.inputVoltage, int(-5)],
|
||||
]),
|
||||
);
|
||||
assert.deepEqual([parsed.requestId, parsed.errorStatus], [77, 0]);
|
||||
assert.deepEqual([parsed.values.get(snmp.OIDS.chargeRemaining), parsed.values.get(snmp.OIDS.model), parsed.values.get(snmp.OIDS.secondsOnBattery), parsed.values.get(snmp.OIDS.outputLoad), parsed.values.get(snmp.OIDS.inputVoltage)], [87, "Back-UPS ES 700", 300, null, -5]);
|
||||
assert.throws(() => snmp.parseResponse(Buffer.from([0x30, 0x03, 0x02, 0x01])), /cut short|isn't/);
|
||||
assert.throws(() => snmp.parseResponse(Buffer.from("not snmp at all")));
|
||||
});
|
||||
|
||||
test("the UPS-MIB: on mains, on battery, low", () => {
|
||||
const m = (entries: [string, number | string | null][]) => new Map<string, number | string | null>(entries);
|
||||
const mains = snmp.interpretSnmp(m([[snmp.OIDS.outputSource, 3], [snmp.OIDS.batteryStatus, 2], [snmp.OIDS.chargeRemaining, 100], [snmp.OIDS.minutesRemaining, 42], [snmp.OIDS.outputLoad, 18], [snmp.OIDS.model, "SMX1500"], [snmp.OIDS.manufacturer, "APC"]]));
|
||||
assert.deepEqual([mains.onBattery, mains.lowBattery, mains.charge, mains.runtimeSeconds, mains.loadPercent, mains.model, mains.statusText], [false, false, 100, 2520, 18, "SMX1500", "normal, battery normal"]);
|
||||
const out = snmp.interpretSnmp(m([[snmp.OIDS.outputSource, 5], [snmp.OIDS.batteryStatus, 3]]));
|
||||
assert.deepEqual([out.onBattery, out.lowBattery, out.statusText], [true, true, "on battery, battery low"]);
|
||||
assert.equal(snmp.interpretSnmp(m([[snmp.OIDS.secondsOnBattery, 90]])).onBattery, true, "no output source, but seconds on battery");
|
||||
assert.equal(snmp.interpretSnmp(m([])).statusText, "unknown");
|
||||
});
|
||||
|
||||
describe("asking an agent", { skip: LAN === null && "this machine has no address other than loopback" }, () => {
|
||||
let agent: dgram.Socket;
|
||||
let port = 0;
|
||||
let community = "public";
|
||||
let silent = false;
|
||||
let values: [string, number[]][] = [];
|
||||
|
||||
before(async () => {
|
||||
agent = dgram.createSocket("udp4");
|
||||
agent.on("message", (msg, rinfo) => {
|
||||
if (silent) return;
|
||||
const req = readRequest(msg);
|
||||
if (req.community !== community) return; // a wrong community gets no answer
|
||||
const binds = req.oids.map((o) => [o, values.find(([k]) => k === o)?.[1] ?? t(0x80, [])] as [string, number[]]);
|
||||
agent.send(answer(req.id, binds), rinfo.port, rinfo.address);
|
||||
});
|
||||
await new Promise<void>((r) => agent.bind(0, "0.0.0.0", r));
|
||||
port = agent.address().port;
|
||||
});
|
||||
after(() => agent.close());
|
||||
|
||||
test("it reads the UPS", async () => {
|
||||
values = [[snmp.OIDS.outputSource, int(5)], [snmp.OIDS.batteryStatus, int(2)], [snmp.OIDS.chargeRemaining, int(64)], [snmp.OIDS.minutesRemaining, int(18)], [snmp.OIDS.model, t(0x04, [...Buffer.from("Eaton 5P")])]];
|
||||
const r = await snmp.readSnmp(LAN!, port, "public");
|
||||
assert.deepEqual([r.onBattery, r.charge, r.runtimeSeconds, r.model], [true, 64, 1080, "Eaton 5P"]);
|
||||
const viaAdapter = await createUpsAdapter({ host: LAN!, snmp: true, port: String(port) }).read();
|
||||
assert.equal(viaAdapter.charge, 64, "the adapter reads over SNMP when asked, with the community public when none is given");
|
||||
});
|
||||
|
||||
test("a device with none of the UPS objects is said not to be a UPS", async () => {
|
||||
values = [];
|
||||
await assert.rejects(snmp.readSnmp(LAN!, port, "public"), /none of the standard UPS objects/);
|
||||
});
|
||||
});
|
||||
|
||||
test("an address it won't ask is refused before anything is sent", async () => {
|
||||
await assert.rejects(snmp.readSnmp("127.0.0.1", 161, "public"));
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Stored, judged and drawn ───────────────────────────────────────────────
|
||||
|
||||
describe("what is stored", () => {
|
||||
let id = 0;
|
||||
const reading = (extra: Partial<import("../src/integrations/ups/types.js").UpsReading> = {}) => ({ model: "SMX", manufacturer: "APC", statusText: "OL", onBattery: false, lowBattery: false, replaceBattery: false, overload: false, charge: 100, runtimeSeconds: 1800, loadPercent: 30, inputVoltage: 230, outputVoltage: 230, ...extra });
|
||||
|
||||
before(async () => {
|
||||
id = (await db.insert(integrations).values({ type: "ups", name: "Rack UPS", baseUrl: "", config: JSON.stringify({ host: "nut.lan" }) }).returning({ id: integrations.id }))[0].id;
|
||||
});
|
||||
|
||||
test("the state is replaced at each read, and when mains power was lost is kept until it is back", async () => {
|
||||
await ups.storeReading(id, reading(), new Date(NOW.getTime() - 30 * MIN));
|
||||
let [s] = await db.select().from(upsState).where(eq(upsState.integrationId, id));
|
||||
assert.deepEqual([s.onBattery, s.onBatterySince, s.charge], [false, null, 100]);
|
||||
await ups.storeReading(id, reading({ onBattery: true, statusText: "OB DISCHRG", charge: 95 }), new Date(NOW.getTime() - 10 * MIN));
|
||||
await ups.storeReading(id, reading({ onBattery: true, statusText: "OB DISCHRG", charge: 80 }), new Date(NOW.getTime() - 5 * MIN));
|
||||
[s] = await db.select().from(upsState).where(eq(upsState.integrationId, id));
|
||||
assert.deepEqual([s.onBattery, s.onBatterySince, s.charge, s.statusText], [true, at(10), 80, "OB DISCHRG"]);
|
||||
await ups.storeReading(id, reading({ charge: 82 }), NOW);
|
||||
[s] = await db.select().from(upsState).where(eq(upsState.integrationId, id));
|
||||
assert.deepEqual([s.onBattery, s.onBatterySince], [false, null]);
|
||||
});
|
||||
|
||||
test("a point is added to the history every five minutes, every minute on battery, and when it changes", async () => {
|
||||
const rows = await db.select().from(upsReadings).where(eq(upsReadings.integrationId, id)).orderBy(upsReadings.at);
|
||||
// 30 min ago (first), 10 and 5 min ago (on battery), now (back on mains: a change)
|
||||
assert.deepEqual(rows.map((r) => r.at), [at(30), at(10), at(5), NOW.toISOString()]);
|
||||
await ups.storeReading(id, reading(), new Date(NOW.getTime() + 1 * MIN));
|
||||
await ups.storeReading(id, reading(), new Date(NOW.getTime() + 3 * MIN));
|
||||
assert.equal((await db.select().from(upsReadings).where(eq(upsReadings.integrationId, id))).length, 4, "mains, a minute and three minutes later: nothing new");
|
||||
await ups.storeReading(id, reading(), new Date(NOW.getTime() + 6 * MIN));
|
||||
assert.equal((await db.select().from(upsReadings).where(eq(upsReadings.integrationId, id))).length, 5);
|
||||
});
|
||||
|
||||
test("history older than 35 days is dropped", async () => {
|
||||
await db.insert(upsReadings).values({ integrationId: id, at: new Date(NOW.getTime() - 40 * 86_400_000).toISOString(), charge: 100, onBattery: false });
|
||||
await ups.storeReading(id, reading(), new Date(NOW.getTime() + 12 * MIN));
|
||||
assert.equal((await db.select().from(upsReadings).where(eq(upsReadings.integrationId, id))).every((r) => Date.parse(r.at) > NOW.getTime() - 36 * 86_400_000), true);
|
||||
});
|
||||
|
||||
test("a read that fails is said so, and keeps when the last good one was", async () => {
|
||||
const ok = await ups.syncUps(id, new Date(NOW.getTime() + 20 * MIN), async () => reading({ charge: 99 }));
|
||||
assert.deepEqual([ok.ok, ok.at], [true, new Date(NOW.getTime() + 20 * MIN).toISOString()]);
|
||||
const failed = await ups.syncUps(id, new Date(NOW.getTime() + 25 * MIN), async () => {
|
||||
throw new Error("no answer within 5 s");
|
||||
});
|
||||
assert.deepEqual([failed.ok, failed.error, failed.at], [false, "no answer within 5 s", ok.at]);
|
||||
const statuses = await ups.upsSyncStatuses();
|
||||
assert.equal(statuses[id].ok, false);
|
||||
assert.equal(ups.isUpsFresh(ok, Date.parse(ok.at) + 4 * MIN), true);
|
||||
assert.equal(ups.isUpsFresh(ok, Date.parse(ok.at) + 6 * MIN), false, "after five minutes it is no longer believed");
|
||||
assert.equal(ups.isUpsFresh(failed, Date.parse(ok.at)), false, "and a failed read is never fresh");
|
||||
const wrong = await ups.syncUps(999999, NOW, async () => reading());
|
||||
assert.equal(wrong.ok, false);
|
||||
});
|
||||
|
||||
test("the history is bucketed into averages and the stretches on battery are found", () => {
|
||||
const rows = [
|
||||
{ at: at(60), charge: 100, runtimeSeconds: 1800, loadPercent: 30, inputVoltage: 230, onBattery: false },
|
||||
{ at: at(50), charge: 100, runtimeSeconds: 1800, loadPercent: 40, inputVoltage: 230, onBattery: false },
|
||||
{ at: at(40), charge: 90, runtimeSeconds: 1200, loadPercent: 35, inputVoltage: 0, onBattery: true },
|
||||
{ at: at(35), charge: 70, runtimeSeconds: 800, loadPercent: 35, inputVoltage: 0, onBattery: true },
|
||||
{ at: at(30), charge: 70, runtimeSeconds: 800, loadPercent: 35, inputVoltage: 231, onBattery: false },
|
||||
{ at: at(5), charge: 98, runtimeSeconds: null, loadPercent: null, inputVoltage: 231, onBattery: true },
|
||||
];
|
||||
const r = ups.bucketUps(rows, NOW.getTime() - 61 * MIN, NOW.getTime(), 2);
|
||||
assert.equal(r.points.length, 2);
|
||||
assert.deepEqual([r.points[0].loadPercent, r.points[0].onBattery, r.points[1].onBattery], [35, true, true]);
|
||||
assert.deepEqual(r.onBatteryPeriods, [{ from: at(40), to: at(30), ongoing: false }, { from: at(5), to: at(5), ongoing: true }], "one that ended, and one still going");
|
||||
assert.deepEqual(ups.bucketUps([], 0, 10, 5), { points: [], onBatteryPeriods: [] });
|
||||
});
|
||||
});
|
||||
|
||||
describe("what is wrong", () => {
|
||||
const base = { onBattery: false, lowBattery: false, replaceBattery: false, overload: false, charge: 100, runtimeSeconds: 1800, loadPercent: 30, onBatterySince: null as string | null };
|
||||
const keys = (s: Partial<typeof base>) => ups.upsProblems(7, "Rack UPS", { ...base, ...s }, NOW).map((p) => p.key);
|
||||
|
||||
test("nothing is wrong with a UPS on mains", () => {
|
||||
assert.deepEqual(keys({}), []);
|
||||
});
|
||||
|
||||
test("mains power lost says for how long, how much is left and how full the battery is", () => {
|
||||
const [p] = ups.upsProblems(7, "Rack UPS", { ...base, onBattery: true, onBatterySince: at(4), charge: 84, runtimeSeconds: 1380 }, NOW);
|
||||
assert.deepEqual([p.key, p.severity, p.kind], ["ups-battery:7", "critical", "battery"]);
|
||||
assert.equal(p.message, "Rack UPS: mains power has failed — running on battery (for 4 min, 23 min left, 84 %)");
|
||||
assert.match(ups.upsProblems(7, "U", { ...base, onBattery: true, charge: null, runtimeSeconds: null }, NOW)[0].message, /^U: mains power has failed — running on battery$/, "with nothing more to say it says no more");
|
||||
});
|
||||
|
||||
test("a nearly empty battery, one to be replaced, and a load near the limit are each their own", () => {
|
||||
assert.deepEqual(keys({ onBattery: true, lowBattery: true }), ["ups-battery:7", "ups-low:7"]);
|
||||
assert.deepEqual(keys({ replaceBattery: true }), ["ups-replace:7"]);
|
||||
assert.deepEqual(keys({ loadPercent: 89 }), []);
|
||||
assert.deepEqual(keys({ loadPercent: 90 }), ["ups-load:7"]);
|
||||
assert.deepEqual(keys({ overload: true, loadPercent: null }), ["ups-load:7"]);
|
||||
assert.equal(ups.upsProblems(7, "R", { ...base, replaceBattery: true }, NOW)[0].severity, "warning");
|
||||
});
|
||||
});
|
||||
|
||||
describe("what it carries", () => {
|
||||
test("what nothing else needs is shut down first, then what that needed, so nothing goes while something still depends on it", () => {
|
||||
// guest → host → ups; nas → ups; app → guest, nas
|
||||
const edges = new Map<string, string[]>([
|
||||
["server:host", ["integration:1"]],
|
||||
["server:nas", ["integration:1"]],
|
||||
["server:guest", ["server:host"]],
|
||||
["server:app", ["server:guest", "server:nas"]],
|
||||
["server:other", ["integration:2"]],
|
||||
]);
|
||||
const { order, looped } = ups.shutdownOrder(edges, "integration:1");
|
||||
assert.equal(looped, false);
|
||||
assert.deepEqual(order.map((o) => [o.key, o.step]), [["server:app", 1], ["server:guest", 2], ["server:nas", 2], ["server:host", 3]]);
|
||||
assert.deepEqual(ups.shutdownOrder(edges, "integration:2").order, [{ key: "server:other", step: 1 }]);
|
||||
assert.deepEqual(ups.shutdownOrder(edges, "integration:9"), { order: [], looped: false });
|
||||
});
|
||||
|
||||
test("things that need each other don't hang it", () => {
|
||||
const edges = new Map<string, string[]>([["server:a", ["integration:1", "server:b"]], ["server:b", ["server:a"]]]);
|
||||
const r = ups.shutdownOrder(edges, "integration:1");
|
||||
assert.equal(r.looped, true);
|
||||
assert.equal(r.order.length, 2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("in the map, in alerts, in the health pass and on the page", () => {
|
||||
let upsId = 0;
|
||||
let hostId = 0;
|
||||
let guestId = 0;
|
||||
let aloneId = 0;
|
||||
let app: Server;
|
||||
let base = "";
|
||||
const state = (extra: Partial<typeof upsState.$inferInsert> = {}) => ({ integrationId: upsId, model: "SMX", manufacturer: "APC", statusText: "OB DISCHRG", onBattery: true, lowBattery: false, replaceBattery: false, overload: false, charge: 71, runtimeSeconds: 1500, loadPercent: 44, inputVoltage: 0, outputVoltage: 230, onBatterySince: at(6), updatedAt: new Date().toISOString(), ...extra });
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
|
||||
upsId = (await db.insert(integrations).values({ type: "ups", name: "Cupboard UPS", baseUrl: "", config: JSON.stringify({ host: "nut.lan" }) }).returning({ id: integrations.id }))[0].id;
|
||||
const gone = new Date(Date.now() - 3 * 3_600_000).toISOString();
|
||||
const mk = async (name: string, lastSeenAt: string) => (await db.insert(servers).values({ name, apiTokenHash: name, apiTokenPrefix: name, lastSeenAt }).returning({ id: servers.id }))[0].id;
|
||||
hostId = await mk("pve", gone);
|
||||
guestId = await mk("wiki", gone);
|
||||
aloneId = await mk("printer", gone);
|
||||
await db.insert(dependencies).values([
|
||||
{ fromType: "server", fromId: hostId, toType: "integration", toId: upsId },
|
||||
{ fromType: "server", fromId: guestId, toType: "server", toId: hostId },
|
||||
]);
|
||||
await db.insert(upsState).values(state());
|
||||
await setInternalFlag("upsSync", JSON.stringify({ [upsId]: { at: new Date().toISOString(), ok: true, error: null } }));
|
||||
|
||||
const a = express();
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/ups", upsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/ups`;
|
||||
});
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
test("a UPS on battery is down, so the servers that go offline with it are folded under it", async () => {
|
||||
assert.deepEqual(await ups.upsOnBatteryIds(), [upsId]);
|
||||
const cover = await deps.dependencyCover([hostId, guestId, aloneId], []);
|
||||
assert.equal(cover.cause.get(`server:${hostId}`), `integration:${upsId}`);
|
||||
assert.equal(cover.cause.get(`server:${guestId}`), `integration:${upsId}`, "through the host it runs on");
|
||||
assert.equal(cover.cause.has(`server:${aloneId}`), false, "one that doesn't need it stands alone");
|
||||
assert.deepEqual((cover.taken.get(`integration:${upsId}`) ?? []).sort(), ["pve", "wiki"]);
|
||||
assert.equal(deps.alertIdOfNode(`integration:${upsId}`, cover.graph), `monitoring:ups-battery:${upsId}`);
|
||||
// Back on mains, it stops explaining anything.
|
||||
await db.update(upsState).set({ onBattery: false }).where(eq(upsState.integrationId, upsId));
|
||||
assert.deepEqual(await ups.upsOnBatteryIds(), []);
|
||||
assert.equal((await deps.dependencyCover([hostId], [])).cause.size, 0);
|
||||
// And a reading that is no longer believed doesn't either.
|
||||
await db.update(upsState).set({ onBattery: true, updatedAt: at(30) }).where(eq(upsState.integrationId, upsId));
|
||||
assert.deepEqual(await ups.upsOnBatteryIds(), []);
|
||||
await db.update(upsState).set({ updatedAt: new Date().toISOString() }).where(eq(upsState.integrationId, upsId));
|
||||
});
|
||||
|
||||
test("the health pass judges it from what is stored, and holds it when the last read is old", async () => {
|
||||
let { snapshot, held } = await collectSnapshot();
|
||||
assert.deepEqual(snapshot.ups?.map((u) => u.integrationName), ["Cupboard UPS"]);
|
||||
assert.equal(held.has(`ups:${upsId}`), false);
|
||||
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
|
||||
const c = list.find((x) => x.key === `ups-battery:${upsId}`)!;
|
||||
assert.deepEqual([c.severity, c.source], ["critical", `ups:${upsId}`]);
|
||||
assert.match(c.message, /Cupboard UPS: mains power has failed/);
|
||||
assert.equal(subjectOfConditionKey(c.key), `integration:${upsId}`, "maintenance on it silences its alerts");
|
||||
for (const k of ["ups-low", "ups-replace", "ups-load"]) assert.equal(subjectOfConditionKey(`${k}:${upsId}`), `integration:${upsId}`);
|
||||
await setInternalFlag("upsSync", JSON.stringify({ [upsId]: { at: at(30), ok: true, error: null } }));
|
||||
({ snapshot, held } = await collectSnapshot());
|
||||
assert.deepEqual([snapshot.ups ?? [], held.has(`ups:${upsId}`)], [[], true]);
|
||||
await setInternalFlag("upsSync", JSON.stringify({ [upsId]: { at: new Date().toISOString(), ok: true, error: null } }));
|
||||
});
|
||||
|
||||
test("the Alerts page shows it, with what goes down with it, and the offline servers say it is why", async () => {
|
||||
invalidateAlertsCache();
|
||||
const report = await getAlerts(true);
|
||||
const a = report.alerts.find((x) => x.id === `monitoring:ups-battery:${upsId}`)!;
|
||||
assert.ok(a, "the alert is there");
|
||||
assert.deepEqual([a.severity, a.category, a.source, a.link, a.target?.type, a.target?.id], ["critical", "monitoring", "UPS", "/ups", "integration", upsId]);
|
||||
assert.deepEqual((a.affects ?? []).sort(), ["pve", "wiki"]);
|
||||
const host = report.alerts.find((x) => x.id === `offline:offline:server:${hostId}`)!;
|
||||
assert.deepEqual([host.causedBy?.name, host.causedBy?.alertId, host.silenced], ["Cupboard UPS", `monitoring:ups-battery:${upsId}`, true]);
|
||||
const alone = report.alerts.find((x) => x.id === `offline:offline:server:${aloneId}`)!;
|
||||
assert.equal(alone.causedBy, undefined);
|
||||
});
|
||||
|
||||
test("the page lists each UPS, how it is, what is on it in shutdown order, and its history", async () => {
|
||||
const res = await fetch(base, { headers: { connection: "close" } });
|
||||
assert.equal(res.status, 200, "everyone signed in can look");
|
||||
const body = (await res.json()) as any;
|
||||
const u = body.ups.find((x: any) => x.id === upsId);
|
||||
assert.deepEqual([u.name, u.protocol, u.fresh, u.state.onBattery, u.state.charge], ["Cupboard UPS", "nut", true, true, 71]);
|
||||
assert.deepEqual(u.problems.map((p: any) => p.key), [`ups-battery:${upsId}`]);
|
||||
assert.deepEqual(u.carries.map((c: any) => [c.name, c.step, c.type]), [["wiki", 1, "server"], ["pve", 2, "server"]]);
|
||||
assert.equal(JSON.stringify(body).includes("nut.lan"), false, "no address leaves the server");
|
||||
const hist = (await (await fetch(`${base}/${upsId}/history?days=1`, { headers: { connection: "close" } })).json()) as any;
|
||||
assert.equal(hist.ups.name, "Cupboard UPS");
|
||||
assert.ok(Array.isArray(hist.points) && Array.isArray(hist.onBatteryPeriods));
|
||||
assert.equal((await fetch(`${base}/999/history`, { headers: { connection: "close" } })).status, 404);
|
||||
assert.equal((await fetch(`${base}/abc/history`, { headers: { connection: "close" } })).status, 400);
|
||||
});
|
||||
|
||||
test("/metrics carries it while it is believed, and not after", async () => {
|
||||
let text = await buildMetrics();
|
||||
assert.match(text, /homelab_ups_on_battery\{name="Cupboard UPS"\} 1/);
|
||||
assert.match(text, /homelab_ups_battery_charge_percent\{name="Cupboard UPS"\} 71/);
|
||||
assert.match(text, /homelab_ups_runtime_seconds\{name="Cupboard UPS"\} 1500/);
|
||||
await setInternalFlag("upsSync", JSON.stringify({ [upsId]: { at: at(30), ok: true, error: null } }));
|
||||
text = await buildMetrics();
|
||||
assert.doesNotMatch(text, /homelab_ups_on_battery/);
|
||||
});
|
||||
|
||||
test("a UPS is added through the integrations form like any other, with its host the only thing needed", async () => {
|
||||
const { INTEGRATION_FIELDS, validateIntegrationConfig, splitIntegrationConfig } = await import("../src/integrations/fieldSchemas.js");
|
||||
assert.ok(INTEGRATION_FIELDS.ups);
|
||||
assert.deepEqual(validateIntegrationConfig("ups", {}), ["host"]);
|
||||
assert.deepEqual(validateIntegrationConfig("ups", { host: "nut.lan" }), []);
|
||||
const { secretFields, nonSecretFields } = splitIntegrationConfig("ups", { host: "nut.lan", snmp: true, community: "s3cret", upsName: "rack" });
|
||||
assert.deepEqual([Object.keys(secretFields), Object.keys(nonSecretFields).sort()], [["community"], ["host", "snmp", "upsName"]]);
|
||||
});
|
||||
});
|
||||
|
||||
// keep the unused-import checker quiet about the helper servers this file declares for readability
|
||||
void createHttpServer;
|
||||
Reference in new issue
Block a user