/** * A UPS: what NUT and SNMP say and how it is read, what is stored and when a history point is added, what is wrong with it, the order to shut down what it carries, * its place in the dependency map (down while on battery, so what goes offline with it is folded under its alert), and the page's and /metrics' view of it. * * Run with: npm test --workspace server */ import { test, describe, before, after } from "node:test"; import assert from "node:assert/strict"; import dgram from "node:dgram"; import net from "node:net"; import { createServer as createHttpServer, type Server } from "node:http"; import { mkdtempSync, rmSync } from "node:fs"; import { networkInterfaces, tmpdir } from "node:os"; import { join } from "node:path"; import type { AddressInfo } from "node:net"; const dir = mkdtempSync(join(tmpdir(), "hm-ups-")); process.env.DATABASE_PATH = join(dir, "test.sqlite"); process.env.SESSION_DIR = join(dir, "sessions"); process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); process.env.HEALTH_STARTUP_GRACE_MS = "0"; const express = (await import("express")).default; const { db } = await import("../src/db/client.js"); const { users, servers, integrations, dependencies, upsState, upsReadings } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const nut = await import("../src/integrations/ups/nut.js"); const snmp = await import("../src/integrations/ups/snmp.js"); const { createUpsAdapter, portOf } = await import("../src/integrations/ups/adapter.js"); const ups = await import("../src/services/upsSync.js"); const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js"); const { subjectOfConditionKey } = await import("../src/services/maintenance.js"); const deps = await import("../src/services/dependencies.js"); const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js"); const { upsRouter } = await import("../src/routes/ups.js"); const { buildMetrics } = await import("../src/services/metrics.js"); const { setInternalFlag } = await import("../src/services/settingsStore.js"); const { eq } = await import("drizzle-orm"); await runMigrations(); /** An address of this machine that isn't loopback (UPS addresses are checked like the endpoint checks'), if it has one. */ function lanAddress(): string | null { for (const list of Object.values(networkInterfaces())) { for (const i of list ?? []) if (i.family === "IPv4" && !i.internal && !i.address.startsWith("169.254.")) return i.address; } return null; } const LAN = lanAddress(); const NOW = new Date("2026-10-20T12:00:00Z"); const MIN = 60_000; const at = (min: number) => new Date(NOW.getTime() - min * MIN).toISOString(); const VARS = { "device.mfr": "APC", "device.model": "Smart-UPS 1500", "ups.status": "OL CHRG", "battery.charge": "97", "battery.runtime": "2040", "ups.load": "34", "input.voltage": "231.4", "output.voltage": "230.0", }; describe("what NUT says", () => { const text = `BEGIN LIST VAR ups\r\n${Object.entries(VARS).map(([k, v]) => `VAR ups ${k} "${v}"`).join("\r\n")}\r\nVAR ups ups.firmware "a \\"quoted\\" word"\r\nEND LIST VAR ups\r\n`; test("the variables, with their quotes", () => { const vars = nut.parseNutVars(text); assert.equal(vars["battery.charge"], "97"); assert.equal(vars["ups.firmware"], 'a "quoted" word'); assert.deepEqual(nut.parseNutVars("ERR UNKNOWN-UPS\n"), {}); }); test("the names of the UPSs it knows", () => { assert.deepEqual(nut.parseNutUpsNames('BEGIN LIST UPS\nUPS rack "In the rack"\nUPS desk "Under the desk"\nEND LIST UPS\n'), ["rack", "desk"]); }); test("online, on battery, low, to be replaced and overloaded", () => { const r = nut.interpretNut(nut.parseNutVars(text)); assert.deepEqual([r.model, r.manufacturer, r.onBattery, r.lowBattery, r.charge, r.runtimeSeconds, r.loadPercent, r.inputVoltage, r.outputVoltage], ["Smart-UPS 1500", "APC", false, false, 97, 2040, 34, 231.4, 230]); const flags = (s: string) => nut.interpretNut({ "ups.status": s }); assert.deepEqual([flags("OB DISCHRG").onBattery, flags("OB DISCHRG").lowBattery], [true, false]); assert.deepEqual([flags("OB DISCHRG LB").onBattery, flags("OB DISCHRG LB").lowBattery], [true, true]); assert.equal(flags("OL RB").replaceBattery, true); assert.equal(flags("OL OVER").overload, true); const empty = nut.interpretNut({}); assert.deepEqual([empty.statusText, empty.charge, empty.model, empty.onBattery], ["unknown", null, null, false]); }); }); describe("asking a NUT server", { skip: LAN === null && "this machine has no address other than loopback" }, () => { let server: net.Server; let port = 0; const seen: string[] = []; before(async () => { server = net.createServer((socket) => { let buf = ""; socket.on("error", () => {}); socket.on("data", (d) => { buf += d.toString(); let i; while ((i = buf.indexOf("\n")) >= 0) { const line = buf.slice(0, i).trim(); buf = buf.slice(i + 1); seen.push(line); if (line === "LIST UPS") socket.write('BEGIN LIST UPS\nUPS rack "The rack"\nEND LIST UPS\n'); else if (line === "LIST VAR rack") socket.write(`BEGIN LIST VAR rack\n${Object.entries(VARS).map(([k, v]) => `VAR rack ${k} "${v}"`).join("\n")}\nEND LIST VAR rack\n`); else if (line.startsWith("LIST VAR")) socket.write("ERR UNKNOWN-UPS\n"); else if (line === "LOGOUT") socket.end("OK Goodbye\n"); } }); }); await new Promise((r) => server.listen(0, "0.0.0.0", r)); port = (server.address() as AddressInfo).port; }); after(() => server.close()); test("it reads the named UPS, or the first one listed when none is named", async () => { const named = await nut.readNut(LAN!, port, "rack"); assert.deepEqual([named.model, named.charge, named.onBattery], ["Smart-UPS 1500", 97, false]); seen.length = 0; const first = await nut.readNut(LAN!, port, null); assert.equal(first.manufacturer, "APC"); assert.deepEqual(seen.filter((l) => l !== "LOGOUT").slice(0, 2), ["LIST UPS", "LIST VAR rack"]); }); test("an unknown UPS, a closed port and an address it won't ask say so", async () => { await assert.rejects(nut.readNut(LAN!, port, "nope"), /upsd says UNKNOWN-UPS/); await assert.rejects(nut.readNut(LAN!, 1, "rack"), /nothing is listening on port 1/); await assert.rejects(nut.readNut("127.0.0.1", port, "rack")); }); test("the adapter takes its port from the config, or the protocol's own, and pings", async () => { assert.deepEqual([portOf({ host: "x" }), portOf({ host: "x", snmp: true }), portOf({ host: "x", snmp: "true", port: "1161" }), portOf({ host: "x", port: "bad" })], [3493, 161, 1161, 3493]); const ok = await createUpsAdapter({ host: LAN!, port: String(port), upsName: "rack" }).ping(); assert.equal(ok.ok, true); const bad = await createUpsAdapter({ host: LAN!, port: String(port), upsName: "nope" }).ping(); assert.deepEqual([bad.ok, /UNKNOWN-UPS/.test(bad.error ?? "")], [false, true]); }); }); // ─── SNMP, with a small agent of the test's own ───────────────────────────── /** Just enough BER to answer a GET: the test's own encoder, so what is checked is not the code's. */ const len = (n: number) => (n < 128 ? [n] : [0x81, n]); const t = (tag: number, body: number[]) => [tag, ...len(body.length), ...body]; const int = (n: number) => t(0x02, [n >> 24 & 0xff, n >> 16 & 0xff, n >> 8 & 0xff, n & 0xff]); const oidBytes = (oid: string) => { const p = oid.split(".").map(Number); const out = [p[0] * 40 + p[1]]; for (const n of p.slice(2)) out.push(...(n < 128 ? [n] : [(n >> 7) | 0x80, n & 0x7f])); return t(0x06, out); }; /** The request id and the objects asked for, read straight from the bytes. */ function readRequest(buf: Buffer): { id: number; community: string; oids: string[] } { let i = 2; // 30 len if (buf[1] & 0x80) i = 2 + (buf[1] & 0x7f); const tlv = () => { const tag = buf[i++]; let l = buf[i++]; if (l & 0x80) { const n = l & 0x7f; l = 0; for (let k = 0; k < n; k++) l = l * 256 + buf[i++]; } const start = i; i += l; return { tag, start, end: i }; }; tlv(); // version const com = tlv(); const community = buf.subarray(com.start, com.end).toString(); const pdu = tlv(); i = pdu.start; const rid = tlv(); let id = 0; for (let k = rid.start; k < rid.end; k++) id = id * 256 + buf[k]; tlv(); tlv(); const list = tlv(); i = list.start; const oids: string[] = []; while (i < list.end) { const vb = tlv(); const save = i; i = vb.start; const o = tlv(); const bytes = buf.subarray(o.start, o.end); const parts = [Math.floor(bytes[0] / 40), bytes[0] % 40]; let v = 0; for (const b of bytes.subarray(1)) { v = v * 128 + (b & 0x7f); if (!(b & 0x80)) { parts.push(v); v = 0; } } oids.push(parts.join(".")); i = save; } return { id, community, oids }; } describe("what SNMP says", () => { test("an object id is written as the standard says", () => { assert.deepEqual([...snmp.encodeOid("1.3.6.1.2.1.33.1.2.4.0")], [0x06, 0x0a, 0x2b, 0x06, 0x01, 0x02, 0x01, 0x21, 0x01, 0x02, 0x04, 0x00]); assert.deepEqual([...snmp.encodeOid("1.3.6.1.4.1.318.1")], [0x06, 0x08, 0x2b, 0x06, 0x01, 0x04, 0x01, 0x82, 0x3e, 0x01], "318 takes two bytes"); }); test("a request names the community, an id and the objects", () => { const req = snmp.buildGet("public", 123456, [snmp.OIDS.chargeRemaining, snmp.OIDS.outputSource]); assert.equal(req[0], 0x30); assert.deepEqual(readRequest(req), { id: 123456, community: "public", oids: [snmp.OIDS.chargeRemaining, snmp.OIDS.outputSource] }); }); /** An answer as an agent would write it. */ const answer = (id: number, binds: [string, number[]][]) => Buffer.from(t(0x30, [...int(1), ...t(0x04, [...Buffer.from("public")]), ...t(0xa2, [...int(id), ...int(0), ...int(0), ...t(0x30, binds.flatMap(([o, v]) => t(0x30, [...oidBytes(o), ...v])))])])); test("an answer's numbers, text, counters and objects the device doesn't have", () => { const parsed = snmp.parseResponse( answer(77, [ [snmp.OIDS.chargeRemaining, int(87)], [snmp.OIDS.model, t(0x04, [...Buffer.from("Back-UPS ES 700")])], [snmp.OIDS.secondsOnBattery, t(0x42, [0x01, 0x2c])], [snmp.OIDS.outputLoad, t(0x80, [])], [snmp.OIDS.inputVoltage, int(-5)], ]), ); assert.deepEqual([parsed.requestId, parsed.errorStatus], [77, 0]); assert.deepEqual([parsed.values.get(snmp.OIDS.chargeRemaining), parsed.values.get(snmp.OIDS.model), parsed.values.get(snmp.OIDS.secondsOnBattery), parsed.values.get(snmp.OIDS.outputLoad), parsed.values.get(snmp.OIDS.inputVoltage)], [87, "Back-UPS ES 700", 300, null, -5]); assert.throws(() => snmp.parseResponse(Buffer.from([0x30, 0x03, 0x02, 0x01])), /cut short|isn't/); assert.throws(() => snmp.parseResponse(Buffer.from("not snmp at all"))); }); test("the UPS-MIB: on mains, on battery, low", () => { const m = (entries: [string, number | string | null][]) => new Map(entries); const mains = snmp.interpretSnmp(m([[snmp.OIDS.outputSource, 3], [snmp.OIDS.batteryStatus, 2], [snmp.OIDS.chargeRemaining, 100], [snmp.OIDS.minutesRemaining, 42], [snmp.OIDS.outputLoad, 18], [snmp.OIDS.model, "SMX1500"], [snmp.OIDS.manufacturer, "APC"]])); assert.deepEqual([mains.onBattery, mains.lowBattery, mains.charge, mains.runtimeSeconds, mains.loadPercent, mains.model, mains.statusText], [false, false, 100, 2520, 18, "SMX1500", "normal, battery normal"]); const out = snmp.interpretSnmp(m([[snmp.OIDS.outputSource, 5], [snmp.OIDS.batteryStatus, 3]])); assert.deepEqual([out.onBattery, out.lowBattery, out.statusText], [true, true, "on battery, battery low"]); assert.equal(snmp.interpretSnmp(m([[snmp.OIDS.secondsOnBattery, 90]])).onBattery, true, "no output source, but seconds on battery"); assert.equal(snmp.interpretSnmp(m([])).statusText, "unknown"); }); describe("asking an agent", { skip: LAN === null && "this machine has no address other than loopback" }, () => { let agent: dgram.Socket; let port = 0; let community = "public"; let silent = false; let values: [string, number[]][] = []; before(async () => { agent = dgram.createSocket("udp4"); agent.on("message", (msg, rinfo) => { if (silent) return; const req = readRequest(msg); if (req.community !== community) return; // a wrong community gets no answer const binds = req.oids.map((o) => [o, values.find(([k]) => k === o)?.[1] ?? t(0x80, [])] as [string, number[]]); agent.send(answer(req.id, binds), rinfo.port, rinfo.address); }); await new Promise((r) => agent.bind(0, "0.0.0.0", r)); port = agent.address().port; }); after(() => agent.close()); test("it reads the UPS", async () => { values = [[snmp.OIDS.outputSource, int(5)], [snmp.OIDS.batteryStatus, int(2)], [snmp.OIDS.chargeRemaining, int(64)], [snmp.OIDS.minutesRemaining, int(18)], [snmp.OIDS.model, t(0x04, [...Buffer.from("Eaton 5P")])]]; const r = await snmp.readSnmp(LAN!, port, "public"); assert.deepEqual([r.onBattery, r.charge, r.runtimeSeconds, r.model], [true, 64, 1080, "Eaton 5P"]); const viaAdapter = await createUpsAdapter({ host: LAN!, snmp: true, port: String(port) }).read(); assert.equal(viaAdapter.charge, 64, "the adapter reads over SNMP when asked, with the community public when none is given"); }); test("a device with none of the UPS objects is said not to be a UPS", async () => { values = []; await assert.rejects(snmp.readSnmp(LAN!, port, "public"), /none of the standard UPS objects/); }); }); test("an address it won't ask is refused before anything is sent", async () => { await assert.rejects(snmp.readSnmp("127.0.0.1", 161, "public")); }); }); // ─── Stored, judged and drawn ─────────────────────────────────────────────── describe("what is stored", () => { let id = 0; const reading = (extra: Partial = {}) => ({ model: "SMX", manufacturer: "APC", statusText: "OL", onBattery: false, lowBattery: false, replaceBattery: false, overload: false, charge: 100, runtimeSeconds: 1800, loadPercent: 30, inputVoltage: 230, outputVoltage: 230, ...extra }); before(async () => { id = (await db.insert(integrations).values({ type: "ups", name: "Rack UPS", baseUrl: "", config: JSON.stringify({ host: "nut.lan" }) }).returning({ id: integrations.id }))[0].id; }); test("the state is replaced at each read, and when mains power was lost is kept until it is back", async () => { await ups.storeReading(id, reading(), new Date(NOW.getTime() - 30 * MIN)); let [s] = await db.select().from(upsState).where(eq(upsState.integrationId, id)); assert.deepEqual([s.onBattery, s.onBatterySince, s.charge], [false, null, 100]); await ups.storeReading(id, reading({ onBattery: true, statusText: "OB DISCHRG", charge: 95 }), new Date(NOW.getTime() - 10 * MIN)); await ups.storeReading(id, reading({ onBattery: true, statusText: "OB DISCHRG", charge: 80 }), new Date(NOW.getTime() - 5 * MIN)); [s] = await db.select().from(upsState).where(eq(upsState.integrationId, id)); assert.deepEqual([s.onBattery, s.onBatterySince, s.charge, s.statusText], [true, at(10), 80, "OB DISCHRG"]); await ups.storeReading(id, reading({ charge: 82 }), NOW); [s] = await db.select().from(upsState).where(eq(upsState.integrationId, id)); assert.deepEqual([s.onBattery, s.onBatterySince], [false, null]); }); test("a point is added to the history every five minutes, every minute on battery, and when it changes", async () => { const rows = await db.select().from(upsReadings).where(eq(upsReadings.integrationId, id)).orderBy(upsReadings.at); // 30 min ago (first), 10 and 5 min ago (on battery), now (back on mains: a change) assert.deepEqual(rows.map((r) => r.at), [at(30), at(10), at(5), NOW.toISOString()]); await ups.storeReading(id, reading(), new Date(NOW.getTime() + 1 * MIN)); await ups.storeReading(id, reading(), new Date(NOW.getTime() + 3 * MIN)); assert.equal((await db.select().from(upsReadings).where(eq(upsReadings.integrationId, id))).length, 4, "mains, a minute and three minutes later: nothing new"); await ups.storeReading(id, reading(), new Date(NOW.getTime() + 6 * MIN)); assert.equal((await db.select().from(upsReadings).where(eq(upsReadings.integrationId, id))).length, 5); }); test("history older than 35 days is dropped", async () => { await db.insert(upsReadings).values({ integrationId: id, at: new Date(NOW.getTime() - 40 * 86_400_000).toISOString(), charge: 100, onBattery: false }); await ups.storeReading(id, reading(), new Date(NOW.getTime() + 12 * MIN)); assert.equal((await db.select().from(upsReadings).where(eq(upsReadings.integrationId, id))).every((r) => Date.parse(r.at) > NOW.getTime() - 36 * 86_400_000), true); }); test("a read that fails is said so, and keeps when the last good one was", async () => { const ok = await ups.syncUps(id, new Date(NOW.getTime() + 20 * MIN), async () => reading({ charge: 99 })); assert.deepEqual([ok.ok, ok.at], [true, new Date(NOW.getTime() + 20 * MIN).toISOString()]); const failed = await ups.syncUps(id, new Date(NOW.getTime() + 25 * MIN), async () => { throw new Error("no answer within 5 s"); }); assert.deepEqual([failed.ok, failed.error, failed.at], [false, "no answer within 5 s", ok.at]); const statuses = await ups.upsSyncStatuses(); assert.equal(statuses[id].ok, false); assert.equal(ups.isUpsFresh(ok, Date.parse(ok.at) + 4 * MIN), true); assert.equal(ups.isUpsFresh(ok, Date.parse(ok.at) + 6 * MIN), false, "after five minutes it is no longer believed"); assert.equal(ups.isUpsFresh(failed, Date.parse(ok.at)), false, "and a failed read is never fresh"); const wrong = await ups.syncUps(999999, NOW, async () => reading()); assert.equal(wrong.ok, false); }); test("the history is bucketed into averages and the stretches on battery are found", () => { const rows = [ { at: at(60), charge: 100, runtimeSeconds: 1800, loadPercent: 30, inputVoltage: 230, onBattery: false }, { at: at(50), charge: 100, runtimeSeconds: 1800, loadPercent: 40, inputVoltage: 230, onBattery: false }, { at: at(40), charge: 90, runtimeSeconds: 1200, loadPercent: 35, inputVoltage: 0, onBattery: true }, { at: at(35), charge: 70, runtimeSeconds: 800, loadPercent: 35, inputVoltage: 0, onBattery: true }, { at: at(30), charge: 70, runtimeSeconds: 800, loadPercent: 35, inputVoltage: 231, onBattery: false }, { at: at(5), charge: 98, runtimeSeconds: null, loadPercent: null, inputVoltage: 231, onBattery: true }, ]; const r = ups.bucketUps(rows, NOW.getTime() - 61 * MIN, NOW.getTime(), 2); assert.equal(r.points.length, 2); assert.deepEqual([r.points[0].loadPercent, r.points[0].onBattery, r.points[1].onBattery], [35, true, true]); assert.deepEqual(r.onBatteryPeriods, [{ from: at(40), to: at(30), ongoing: false }, { from: at(5), to: at(5), ongoing: true }], "one that ended, and one still going"); assert.deepEqual(ups.bucketUps([], 0, 10, 5), { points: [], onBatteryPeriods: [] }); }); }); describe("what is wrong", () => { const base = { onBattery: false, lowBattery: false, replaceBattery: false, overload: false, charge: 100, runtimeSeconds: 1800, loadPercent: 30, onBatterySince: null as string | null }; const keys = (s: Partial) => ups.upsProblems(7, "Rack UPS", { ...base, ...s }, NOW).map((p) => p.key); test("nothing is wrong with a UPS on mains", () => { assert.deepEqual(keys({}), []); }); test("mains power lost says for how long, how much is left and how full the battery is", () => { const [p] = ups.upsProblems(7, "Rack UPS", { ...base, onBattery: true, onBatterySince: at(4), charge: 84, runtimeSeconds: 1380 }, NOW); assert.deepEqual([p.key, p.severity, p.kind], ["ups-battery:7", "critical", "battery"]); assert.equal(p.message, "Rack UPS: mains power has failed — running on battery (for 4 min, 23 min left, 84 %)"); assert.match(ups.upsProblems(7, "U", { ...base, onBattery: true, charge: null, runtimeSeconds: null }, NOW)[0].message, /^U: mains power has failed — running on battery$/, "with nothing more to say it says no more"); }); test("a nearly empty battery, one to be replaced, and a load near the limit are each their own", () => { assert.deepEqual(keys({ onBattery: true, lowBattery: true }), ["ups-battery:7", "ups-low:7"]); assert.deepEqual(keys({ replaceBattery: true }), ["ups-replace:7"]); assert.deepEqual(keys({ loadPercent: 89 }), []); assert.deepEqual(keys({ loadPercent: 90 }), ["ups-load:7"]); assert.deepEqual(keys({ overload: true, loadPercent: null }), ["ups-load:7"]); assert.equal(ups.upsProblems(7, "R", { ...base, replaceBattery: true }, NOW)[0].severity, "warning"); }); }); describe("what it carries", () => { test("what nothing else needs is shut down first, then what that needed, so nothing goes while something still depends on it", () => { // guest → host → ups; nas → ups; app → guest, nas const edges = new Map([ ["server:host", ["integration:1"]], ["server:nas", ["integration:1"]], ["server:guest", ["server:host"]], ["server:app", ["server:guest", "server:nas"]], ["server:other", ["integration:2"]], ]); const { order, looped } = ups.shutdownOrder(edges, "integration:1"); assert.equal(looped, false); assert.deepEqual(order.map((o) => [o.key, o.step]), [["server:app", 1], ["server:guest", 2], ["server:nas", 2], ["server:host", 3]]); assert.deepEqual(ups.shutdownOrder(edges, "integration:2").order, [{ key: "server:other", step: 1 }]); assert.deepEqual(ups.shutdownOrder(edges, "integration:9"), { order: [], looped: false }); }); test("things that need each other don't hang it", () => { const edges = new Map([["server:a", ["integration:1", "server:b"]], ["server:b", ["server:a"]]]); const r = ups.shutdownOrder(edges, "integration:1"); assert.equal(r.looped, true); assert.equal(r.order.length, 2); }); }); describe("in the map, in alerts, in the health pass and on the page", () => { let upsId = 0; let hostId = 0; let guestId = 0; let aloneId = 0; let app: Server; let base = ""; const state = (extra: Partial = {}) => ({ integrationId: upsId, model: "SMX", manufacturer: "APC", statusText: "OB DISCHRG", onBattery: true, lowBattery: false, replaceBattery: false, overload: false, charge: 71, runtimeSeconds: 1500, loadPercent: 44, inputVoltage: 0, outputVoltage: 230, onBatterySince: at(6), updatedAt: new Date().toISOString(), ...extra }); before(async () => { await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]); upsId = (await db.insert(integrations).values({ type: "ups", name: "Cupboard UPS", baseUrl: "", config: JSON.stringify({ host: "nut.lan" }) }).returning({ id: integrations.id }))[0].id; const gone = new Date(Date.now() - 3 * 3_600_000).toISOString(); const mk = async (name: string, lastSeenAt: string) => (await db.insert(servers).values({ name, apiTokenHash: name, apiTokenPrefix: name, lastSeenAt }).returning({ id: servers.id }))[0].id; hostId = await mk("pve", gone); guestId = await mk("wiki", gone); aloneId = await mk("printer", gone); await db.insert(dependencies).values([ { fromType: "server", fromId: hostId, toType: "integration", toId: upsId }, { fromType: "server", fromId: guestId, toType: "server", toId: hostId }, ]); await db.insert(upsState).values(state()); await setInternalFlag("upsSync", JSON.stringify({ [upsId]: { at: new Date().toISOString(), ok: true, error: null } })); const a = express(); a.use((req, _res, next) => { (req as unknown as { session: unknown }).session = { user: { sub: "viewer" } }; next(); }); a.use("/api/ups", upsRouter); app = await new Promise((resolve) => { const s = a.listen(0, "127.0.0.1", () => resolve(s)); }); base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/ups`; }); after(() => { app.close(); try { rmSync(dir, { recursive: true, force: true }); } catch { // Windows keeps the database file locked until the process exits. } }); test("a UPS on battery is down, so the servers that go offline with it are folded under it", async () => { assert.deepEqual(await ups.upsOnBatteryIds(), [upsId]); const cover = await deps.dependencyCover([hostId, guestId, aloneId], []); assert.equal(cover.cause.get(`server:${hostId}`), `integration:${upsId}`); assert.equal(cover.cause.get(`server:${guestId}`), `integration:${upsId}`, "through the host it runs on"); assert.equal(cover.cause.has(`server:${aloneId}`), false, "one that doesn't need it stands alone"); assert.deepEqual((cover.taken.get(`integration:${upsId}`) ?? []).sort(), ["pve", "wiki"]); assert.equal(deps.alertIdOfNode(`integration:${upsId}`, cover.graph), `monitoring:ups-battery:${upsId}`); // Back on mains, it stops explaining anything. await db.update(upsState).set({ onBattery: false }).where(eq(upsState.integrationId, upsId)); assert.deepEqual(await ups.upsOnBatteryIds(), []); assert.equal((await deps.dependencyCover([hostId], [])).cause.size, 0); // And a reading that is no longer believed doesn't either. await db.update(upsState).set({ onBattery: true, updatedAt: at(30) }).where(eq(upsState.integrationId, upsId)); assert.deepEqual(await ups.upsOnBatteryIds(), []); await db.update(upsState).set({ updatedAt: new Date().toISOString() }).where(eq(upsState.integrationId, upsId)); }); test("the health pass judges it from what is stored, and holds it when the last read is old", async () => { let { snapshot, held } = await collectSnapshot(); assert.deepEqual(snapshot.ups?.map((u) => u.integrationName), ["Cupboard UPS"]); assert.equal(held.has(`ups:${upsId}`), false); const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true }); const c = list.find((x) => x.key === `ups-battery:${upsId}`)!; assert.deepEqual([c.severity, c.source], ["critical", `ups:${upsId}`]); assert.match(c.message, /Cupboard UPS: mains power has failed/); assert.equal(subjectOfConditionKey(c.key), `integration:${upsId}`, "maintenance on it silences its alerts"); for (const k of ["ups-low", "ups-replace", "ups-load"]) assert.equal(subjectOfConditionKey(`${k}:${upsId}`), `integration:${upsId}`); await setInternalFlag("upsSync", JSON.stringify({ [upsId]: { at: at(30), ok: true, error: null } })); ({ snapshot, held } = await collectSnapshot()); assert.deepEqual([snapshot.ups ?? [], held.has(`ups:${upsId}`)], [[], true]); await setInternalFlag("upsSync", JSON.stringify({ [upsId]: { at: new Date().toISOString(), ok: true, error: null } })); }); test("the Alerts page shows it, with what goes down with it, and the offline servers say it is why", async () => { invalidateAlertsCache(); const report = await getAlerts(true); const a = report.alerts.find((x) => x.id === `monitoring:ups-battery:${upsId}`)!; assert.ok(a, "the alert is there"); assert.deepEqual([a.severity, a.category, a.source, a.link, a.target?.type, a.target?.id], ["critical", "monitoring", "UPS", "/ups", "integration", upsId]); assert.deepEqual((a.affects ?? []).sort(), ["pve", "wiki"]); const host = report.alerts.find((x) => x.id === `offline:offline:server:${hostId}`)!; assert.deepEqual([host.causedBy?.name, host.causedBy?.alertId, host.silenced], ["Cupboard UPS", `monitoring:ups-battery:${upsId}`, true]); const alone = report.alerts.find((x) => x.id === `offline:offline:server:${aloneId}`)!; assert.equal(alone.causedBy, undefined); }); test("the page lists each UPS, how it is, what is on it in shutdown order, and its history", async () => { const res = await fetch(base, { headers: { connection: "close" } }); assert.equal(res.status, 200, "everyone signed in can look"); const body = (await res.json()) as any; const u = body.ups.find((x: any) => x.id === upsId); assert.deepEqual([u.name, u.protocol, u.fresh, u.state.onBattery, u.state.charge], ["Cupboard UPS", "nut", true, true, 71]); assert.deepEqual(u.problems.map((p: any) => p.key), [`ups-battery:${upsId}`]); assert.deepEqual(u.carries.map((c: any) => [c.name, c.step, c.type]), [["wiki", 1, "server"], ["pve", 2, "server"]]); assert.equal(JSON.stringify(body).includes("nut.lan"), false, "no address leaves the server"); const hist = (await (await fetch(`${base}/${upsId}/history?days=1`, { headers: { connection: "close" } })).json()) as any; assert.equal(hist.ups.name, "Cupboard UPS"); assert.ok(Array.isArray(hist.points) && Array.isArray(hist.onBatteryPeriods)); assert.equal((await fetch(`${base}/999/history`, { headers: { connection: "close" } })).status, 404); assert.equal((await fetch(`${base}/abc/history`, { headers: { connection: "close" } })).status, 400); }); test("/metrics carries it while it is believed, and not after", async () => { let text = await buildMetrics(); assert.match(text, /homelab_ups_on_battery\{name="Cupboard UPS"\} 1/); assert.match(text, /homelab_ups_battery_charge_percent\{name="Cupboard UPS"\} 71/); assert.match(text, /homelab_ups_runtime_seconds\{name="Cupboard UPS"\} 1500/); await setInternalFlag("upsSync", JSON.stringify({ [upsId]: { at: at(30), ok: true, error: null } })); text = await buildMetrics(); assert.doesNotMatch(text, /homelab_ups_on_battery/); }); test("a UPS is added through the integrations form like any other, with its host the only thing needed", async () => { const { INTEGRATION_FIELDS, validateIntegrationConfig, splitIntegrationConfig } = await import("../src/integrations/fieldSchemas.js"); assert.ok(INTEGRATION_FIELDS.ups); assert.deepEqual(validateIntegrationConfig("ups", {}), ["host"]); assert.deepEqual(validateIntegrationConfig("ups", { host: "nut.lan" }), []); const { secretFields, nonSecretFields } = splitIntegrationConfig("ups", { host: "nut.lan", snmp: true, community: "s3cret", upsName: "rack" }); assert.deepEqual([Object.keys(secretFields), Object.keys(nonSecretFields).sort()], [["community"], ["host", "snmp", "upsName"]]); }); }); // keep the unused-import checker quiet about the helper servers this file declares for readability void createHttpServer;