A UPS in the map: read over NUT or SNMP, alerts, shutdown order

New UPS integration (NUT or SNMP v2c, read only). Monitoring -> UPS shows charge,
runtime, load and mains voltage with history and the times mains power was lost.
Mains lost and a nearly empty battery are critical alerts; battery to be replaced
and a heavy load are warnings. The UPS is a node in the dependency map and counts
as down while on battery, so servers that go offline with it are folded under its
alert; the page lists what is on it in the order to shut it down. /metrics series
and docs added.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-21 21:47:31 +02:00
1 parent 3cb3e4988f
commit 232a5661ab
38 files changed
+8049 -16

No files matched your search

+2
View File
@@ -55,6 +55,7 @@ import { storageRouter } from "./routes/storage.js";
import { dataUsageRouter } from "./routes/dataUsage.js";
import { alertActionsRouter } from "./routes/alertActions.js";
import { homeAssistantRouter } from "./routes/homeAssistant.js";
import { upsRouter } from "./routes/ups.js";
import { publicApiRouter } from "./routes/publicApi.js";
import { generatorRouter } from "./routes/generator.js";
import { alertTicketsRouter } from "./routes/alertTickets.js";
@@ -184,6 +185,7 @@ export function createApp() {
app.use("/api/storage", storageRouter);
app.use("/api/data-usage", dataUsageRouter);
app.use("/api/home-assistant", homeAssistantRouter);
app.use("/api/ups", upsRouter);
app.use("/api/todos", todosRouter);
app.use("/api/notes", notesRouter);
app.use("/api/generator", generatorRouter);
+43
View File
@@ -552,6 +552,7 @@ export const integrationTypes = [
"unifi",
"npm",
"homeassistant",
"ups",
] as const;
export type IntegrationType = (typeof integrationTypes)[number];
@@ -1158,6 +1159,48 @@ export const haReadings = sqliteTable(
(t) => [index("ha_readings_lookup_idx").on(t.integrationId, t.entityId, t.at)],
);
// ─── UPS ────────────────────────────────────────────────────────────────────
// How a UPS (an integration of type "ups", read over NUT or SNMP by services/upsSync.ts) is right now: one row each, replaced at every read. `updatedAt` is the last
// read that worked; `onBatterySince` is when mains power was lost, kept until it is back.
export const upsState = sqliteTable("ups_state", {
integrationId: integer("integration_id")
.primaryKey()
.references(() => integrations.id, { onDelete: "cascade" }),
model: text("model"),
manufacturer: text("manufacturer"),
statusText: text("status_text").notNull(),
onBattery: integer("on_battery", { mode: "boolean" }).notNull().default(false),
lowBattery: integer("low_battery", { mode: "boolean" }).notNull().default(false),
replaceBattery: integer("replace_battery", { mode: "boolean" }).notNull().default(false),
overload: integer("overload", { mode: "boolean" }).notNull().default(false),
charge: real("charge"),
runtimeSeconds: integer("runtime_seconds"),
loadPercent: real("load_percent"),
inputVoltage: real("input_voltage"),
outputVoltage: real("output_voltage"),
onBatterySince: text("on_battery_since"), // ISO
updatedAt: text("updated_at").notNull(), // ISO
});
// A UPS at a moment, for the charts: one every five minutes, and every minute while it runs on battery. Kept 35 days.
export const upsReadings = sqliteTable(
"ups_readings",
{
id: integer("id").primaryKey({ autoIncrement: true }),
integrationId: integer("integration_id")
.notNull()
.references(() => integrations.id, { onDelete: "cascade" }),
at: text("at").notNull(), // ISO
charge: real("charge"),
runtimeSeconds: integer("runtime_seconds"),
loadPercent: real("load_percent"),
inputVoltage: real("input_voltage"),
onBattery: integer("on_battery", { mode: "boolean" }).notNull().default(false),
},
(t) => [index("ups_readings_lookup_idx").on(t.integrationId, t.at)],
);
// ─── Notes and the to-do list ───────────────────────────────────────────────
// Free-form notes about a server or a network device — what is it, where does it sit, what did I do to it last time — one text per thing, drawn with the same small
+2
View File
@@ -27,6 +27,7 @@ import { initFlapMonitor } from "./services/flapMonitor.js";
import { initSnoozeMonitor } from "./services/snoozeMonitor.js";
import { initUnifiSync } from "./services/unifiSync.js";
import { initHomeAssistantSync } from "./services/homeAssistantSync.js";
import { initUpsSync } from "./services/upsSync.js";
import { backfillMetricRollups } from "./services/serverMetrics.js";
import { backfillEndpointHourly } from "./services/endpointChecks.js";
import { initSelfBackupScheduler } from "./services/selfBackupScheduler.js";
@@ -75,6 +76,7 @@ initFlapMonitor();
initSnoozeMonitor();
initUnifiSync();
initHomeAssistantSync();
initUpsSync();
// The hourly and daily figures are built from what is already kept, once, the first time after they were added (in the background: nothing waits for it).
void Promise.all([backfillMetricRollups(), backfillEndpointHourly()]).catch((err) => console.error("[history] couldn't build the hourly and daily figures:", err));
+10
View File
@@ -99,6 +99,13 @@ export const INTEGRATION_FIELDS: Partial<Record<IntegrationType, IntegrationFiel
{ key: "token", label: "Long-lived access token", secret: true, type: "password", placeholder: "made on your profile page in Home Assistant" },
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
],
ups: [
{ key: "host", label: "Host name or address", secret: false, placeholder: "the machine running NUT (upsd), or the UPS's network card" },
{ key: "snmp", label: "Read it over SNMP (version 2c) instead of NUT", secret: false, type: "checkbox" },
{ key: "port", label: "Port", secret: false, optional: true, placeholder: "3493 for NUT, 161 for SNMP" },
{ key: "upsName", label: "NUT: the UPS's name", secret: false, optional: true, placeholder: "as in ups.conf; the first one listed when blank" },
{ key: "community", label: "SNMP: read community", secret: true, optional: true, type: "password", placeholder: "public when blank" },
],
osticket: [
{ key: "host", label: "Database host", secret: false, placeholder: "osticket-db.example.lan" },
{ key: "port", label: "Database port", secret: false, optional: true, placeholder: "3306" },
@@ -141,6 +148,9 @@ export function resolveBaseUrl(
): string {
const url = nonSecretFields.url;
if (typeof url === "string" && url) return url;
// A UPS has no web address; the host it is read from is what the card shows.
const host = nonSecretFields.host;
if (type === "ups" && typeof host === "string" && host) return host;
return INTEGRATION_BASE_URLS[type] ?? "";
}
+3
View File
@@ -13,6 +13,7 @@ import { createOsTicketAdapter } from "./osticket/adapter.js";
import { createUnifiAdapter } from "./unifi/adapter.js";
import { createNpmAdapter } from "./npm/adapter.js";
import { createHomeAssistantAdapter } from "./homeassistant/adapter.js";
import { createUpsAdapter } from "./ups/adapter.js";
export interface PingableAdapter {
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
@@ -53,6 +54,8 @@ export function createIntegrationAdapter(type: IntegrationType, config: Integrat
return createNpmAdapter(config as any);
case "homeassistant":
return createHomeAssistantAdapter(config as any);
case "ups":
return createUpsAdapter(config as any);
default:
throw new Error(`Integration type "${type}" is not implemented yet`);
}
+40
View File
@@ -0,0 +1,40 @@
/**
* UPS adapter. Config: host; optional: port, snmp (read over SNMP instead of NUT), upsName (NUT), community (SNMP).
*
* A UPS is read, never controlled: NUT's `LIST VAR` or an SNMP GET of the UPS-MIB. What it says is the same either way (integrations/ups/types.ts).
*/
import { withDiagLogging } from "../../services/diagLog.js";
import { NUT_PORT, readNut } from "./nut.js";
import { readSnmp, SNMP_PORT } from "./snmp.js";
import type { UpsConfig, UpsReading } from "./types.js";
export interface UpsAdapter {
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
read(): Promise<UpsReading>;
}
const truthy = (v: unknown) => v === true || v === "true";
/** The port in the config, or the protocol's own. */
export function portOf(config: UpsConfig): number {
const asked = Number(config.port);
return Number.isInteger(asked) && asked >= 1 && asked <= 65535 ? asked : truthy(config.snmp) ? SNMP_PORT : NUT_PORT;
}
export function createUpsAdapter(config: UpsConfig): UpsAdapter {
const host = String(config.host ?? "").trim();
async function read(): Promise<UpsReading> {
if (!host) throw new Error("no host given");
return truthy(config.snmp) ? readSnmp(host, portOf(config), (config.community || "public").toString()) : readNut(host, portOf(config), config.upsName?.toString().trim() || null);
}
async function ping() {
const start = Date.now();
try {
await read();
return { ok: true, latencyMs: Date.now() - start };
} catch (err) {
return { ok: false, error: err instanceof Error ? err.message : String(err) };
}
}
return withDiagLogging("ups", { ping, read });
}
+116
View File
@@ -0,0 +1,116 @@
/**
* Network UPS Tools: `upsd` answers a short text protocol on TCP port 3493 and needs no login to read. `LIST VAR <ups>` gives every variable the driver has
* (`battery.charge`, `ups.status`, …) between a BEGIN and an END line; `LIST UPS` names the UPSs it knows. Only reading is done here — never a command.
*/
import net from "node:net";
import { resolveTarget } from "../../utils/networkGuard.js";
import type { UpsReading } from "./types.js";
export const NUT_PORT = 3493;
const TIMEOUT_MS = 5000;
/** `VAR ups battery.charge "100"` lines into a map; anything else in the text is ignored. */
export function parseNutVars(text: string): Record<string, string> {
const out: Record<string, string> = {};
for (const line of text.split(/\r?\n/)) {
const m = /^VAR\s+\S+\s+(\S+)\s+"((?:[^"\\]|\\.)*)"\s*$/.exec(line);
if (m) out[m[1]] = m[2].replace(/\\(.)/g, "$1");
}
return out;
}
/** The names in an answer to `LIST UPS`. */
export function parseNutUpsNames(text: string): string[] {
const out: string[] = [];
for (const line of text.split(/\r?\n/)) {
const m = /^UPS\s+(\S+)\s/.exec(line);
if (m) out.push(m[1]);
}
return out;
}
const num = (v: string | undefined): number | null => {
if (v === undefined || v.trim() === "") return null;
const n = Number(v);
return Number.isFinite(n) ? n : null;
};
/** The variables of one UPS as what the app keeps. `ups.status` is a list of flags: OL online, OB on battery, LB low battery, RB replace battery, OVER overload. */
export function interpretNut(vars: Record<string, string>): UpsReading {
const status = (vars["ups.status"] ?? "").trim();
const flags = new Set(status.split(/\s+/).filter(Boolean));
return {
model: vars["device.model"] || vars["ups.model"] || null,
manufacturer: vars["device.mfr"] || vars["ups.mfr"] || null,
statusText: status || "unknown",
onBattery: flags.has("OB"),
lowBattery: flags.has("LB"),
replaceBattery: flags.has("RB"),
overload: flags.has("OVER"),
charge: num(vars["battery.charge"]),
runtimeSeconds: num(vars["battery.runtime"]),
loadPercent: num(vars["ups.load"]),
inputVoltage: num(vars["input.voltage"]),
outputVoltage: num(vars["output.voltage"]),
};
}
/** Sends the commands one after the other on one connection, each answered by a BEGIN … END block (or an ERR line), and returns the answers. */
function ask(host: string, port: number, commands: string[]): Promise<string[]> {
return new Promise((resolve, reject) => {
const socket = net.connect({ host, port });
const answers: string[] = [];
let buffer = "";
let index = 0;
let finished = false;
const done = (err: Error | null) => {
if (finished) return;
finished = true;
// After a good answer the connection is closed politely (LOGOUT is already sent); after a bad one it is simply dropped.
if (err) socket.destroy();
else {
socket.end();
setTimeout(() => socket.destroy(), 200).unref();
}
if (err) reject(err);
else resolve(answers);
};
socket.setTimeout(TIMEOUT_MS, () => done(new Error(`no answer within ${TIMEOUT_MS / 1000} s`)));
socket.on("error", (err: NodeJS.ErrnoException) => done(new Error(err.code === "ECONNREFUSED" ? `nothing is listening on port ${port} — is upsd running, and does it listen on this address (LISTEN in upsd.conf)?` : err.message)));
socket.on("close", () => done(new Error("the connection closed before the answer was complete")));
socket.on("connect", () => socket.write(`${commands[0]}\n`));
socket.on("data", (chunk) => {
buffer += chunk.toString("utf8");
for (;;) {
const err = /^ERR\s+(.*)$/m.exec(buffer);
if (err) return done(new Error(`upsd says ${err[1].trim()}`));
const end = /^END LIST .*$/m.exec(buffer);
if (!end) return;
const cut = end.index + end[0].length;
answers.push(buffer.slice(0, cut));
buffer = buffer.slice(cut);
index++;
if (index >= commands.length) {
socket.write("LOGOUT\n");
return done(null);
}
socket.write(`${commands[index]}\n`);
}
});
});
}
/** Asks the NUT server about one UPS: `upsName`, or the first it lists when none is given. */
export async function readNut(host: string, port: number, upsName: string | null): Promise<UpsReading> {
const address = await resolveTarget(host, { allowPrivate: true });
let name = upsName;
if (!name) {
const [list] = await ask(address, port, ["LIST UPS"]);
name = parseNutUpsNames(list)[0] ?? null;
if (!name) throw new Error("the NUT server lists no UPS");
}
const [vars] = await ask(address, port, [`LIST VAR ${name}`]);
const parsed = parseNutVars(vars);
if (Object.keys(parsed).length === 0) throw new Error(`the NUT server gave no variables for “${name}”`);
return interpretNut(parsed);
}
+240
View File
@@ -0,0 +1,240 @@
/**
* SNMP, version 2c, only as far as asking for a few values: one GET request over UDP (port 161) naming the objects of the standard UPS-MIB (RFC 1628), which most UPSs
* and their network cards answer. Hand-written, because a GET is a few dozen lines of BER and a whole SNMP library is far more than is needed (and nothing is ever set).
*/
import dgram from "node:dgram";
import net from "node:net";
import { resolveTarget } from "../../utils/networkGuard.js";
import type { UpsReading } from "./types.js";
export const SNMP_PORT = 161;
const TIMEOUT_MS = 2500;
const TRIES = 2;
// ─── BER ────────────────────────────────────────────────────────────────────
const lengthBytes = (n: number): number[] => (n < 0x80 ? [n] : n < 0x100 ? [0x81, n] : [0x82, n >> 8, n & 0xff]);
function tlv(tag: number, content: number[] | Buffer): Buffer {
const body = Buffer.from(content);
return Buffer.concat([Buffer.from([tag, ...lengthBytes(body.length)]), body]);
}
function encodeInt(n: number): Buffer {
const bytes: number[] = [];
let v = n;
do {
bytes.unshift(v & 0xff);
v = Math.floor(v / 256);
} while (v > 0);
if (bytes[0] & 0x80) bytes.unshift(0);
return tlv(0x02, bytes);
}
/** "1.3.6.1.2.1.33.1.2.4.0" into its BER form. */
export function encodeOid(oid: string): Buffer {
const parts = oid.split(".").map(Number);
const bytes: number[] = [parts[0] * 40 + parts[1]];
for (const p of parts.slice(2)) {
const chunk = [p & 0x7f];
let v = Math.floor(p / 128);
while (v > 0) {
chunk.unshift((v & 0x7f) | 0x80);
v = Math.floor(v / 128);
}
bytes.push(...chunk);
}
return tlv(0x06, bytes);
}
/** A GET request for these objects, from this community, under this request id. */
export function buildGet(community: string, requestId: number, oids: string[]): Buffer {
const varbinds = oids.map((o) => tlv(0x30, Buffer.concat([encodeOid(o), tlv(0x05, [])])));
const pdu = tlv(0xa0, Buffer.concat([encodeInt(requestId), encodeInt(0), encodeInt(0), tlv(0x30, Buffer.concat(varbinds))]));
return tlv(0x30, Buffer.concat([encodeInt(1), tlv(0x04, Buffer.from(community, "utf8")), pdu]));
}
interface Node {
tag: number;
value: Buffer;
}
/** The TLVs one after the other in `buf`. */
function children(buf: Buffer): Node[] {
const out: Node[] = [];
let i = 0;
while (i < buf.length) {
const tag = buf[i++];
let len = buf[i++];
if (len === undefined) throw new Error("the answer is cut short");
if (len & 0x80) {
const n = len & 0x7f;
if (n === 0 || n > 3) throw new Error("the answer has a length that can't be read");
len = 0;
for (let k = 0; k < n; k++) len = len * 256 + buf[i++];
}
if (i + len > buf.length) throw new Error("the answer is cut short");
out.push({ tag, value: buf.subarray(i, i + len) });
i += len;
}
return out;
}
function decodeUint(buf: Buffer): number {
let n = 0;
for (const b of buf) n = n * 256 + b;
return n;
}
function decodeInt(buf: Buffer): number {
if (buf.length === 0) return 0;
let v = 0;
for (const b of buf) v = v * 256 + b;
// two's complement
return buf[0] & 0x80 ? v - 2 ** (8 * buf.length) : v;
}
function decodeOid(buf: Buffer): string {
if (buf.length === 0) return "";
const parts = [Math.floor(buf[0] / 40), buf[0] % 40];
let v = 0;
for (const b of buf.subarray(1)) {
v = v * 128 + (b & 0x7f);
if (!(b & 0x80)) {
parts.push(v);
v = 0;
}
}
return parts.join(".");
}
export type SnmpValue = number | string | null;
export interface SnmpAnswer {
requestId: number;
/** 0 when the agent had no complaint. */
errorStatus: number;
/** Object → its value; null where the agent has no such object. */
values: Map<string, SnmpValue>;
}
/** An answer to a GET: its request id and each object's value. Throws if it isn't one. */
export function parseResponse(packet: Buffer): SnmpAnswer {
const [message] = children(packet);
if (!message || message.tag !== 0x30) throw new Error("that isn't an SNMP message");
const [, , pdu] = children(message.value);
if (!pdu || pdu.tag !== 0xa2) throw new Error("that isn't an answer to a request");
const [id, errorStatus, , bindings] = children(pdu.value);
const values = new Map<string, SnmpValue>();
for (const vb of children(bindings.value)) {
const [oid, value] = children(vb.value);
let v: SnmpValue;
switch (value.tag) {
case 0x02:
v = decodeInt(value.value);
break;
case 0x41: // counter32
case 0x42: // gauge32
case 0x43: // timeticks
case 0x46: // counter64
v = decodeUint(value.value);
break;
case 0x04:
v = value.value.toString("utf8").replace(/\0+$/, "").trim();
break;
default:
// null, noSuchObject, noSuchInstance, endOfMibView, or something not needed: there is no value here
v = null;
}
values.set(decodeOid(oid.value), v);
}
return { requestId: decodeInt(id.value), errorStatus: decodeInt(errorStatus.value), values };
}
// ─── The UPS-MIB ────────────────────────────────────────────────────────────
const B = "1.3.6.1.2.1.33.1";
export const OIDS = {
manufacturer: `${B}.1.1.0`,
model: `${B}.1.2.0`,
batteryStatus: `${B}.2.1.0`, // 1 unknown, 2 normal, 3 low, 4 depleted
secondsOnBattery: `${B}.2.2.0`,
minutesRemaining: `${B}.2.3.0`,
chargeRemaining: `${B}.2.4.0`,
inputVoltage: `${B}.3.3.1.3.1`,
outputSource: `${B}.4.1.0`, // 3 normal, 4 bypass, 5 battery, 6 booster, 7 reducer
outputVoltage: `${B}.4.4.1.2.1`,
outputLoad: `${B}.4.4.1.5.1`,
} as const;
const asNumber = (v: SnmpValue | undefined): number | null => (typeof v === "number" ? v : typeof v === "string" && v !== "" && Number.isFinite(Number(v)) ? Number(v) : null);
const asText = (v: SnmpValue | undefined): string | null => (typeof v === "string" && v !== "" ? v : null);
const SOURCES: Record<number, string> = { 1: "other", 2: "no output", 3: "normal", 4: "bypass", 5: "on battery", 6: "booster", 7: "reducer" };
const BATTERY: Record<number, string> = { 1: "battery unknown", 2: "battery normal", 3: "battery low", 4: "battery depleted" };
/** The objects of the UPS-MIB, as what the app keeps. */
export function interpretSnmp(values: Map<string, SnmpValue>): UpsReading {
const get = (oid: string) => values.get(oid);
const source = asNumber(get(OIDS.outputSource));
const battery = asNumber(get(OIDS.batteryStatus));
const secondsOn = asNumber(get(OIDS.secondsOnBattery));
const minutes = asNumber(get(OIDS.minutesRemaining));
const onBattery = source === 5 || (source === null && secondsOn !== null && secondsOn > 0);
const text = [source !== null ? (SOURCES[source] ?? `source ${source}`) : null, battery !== null ? (BATTERY[battery] ?? null) : null].filter(Boolean).join(", ");
return {
model: asText(get(OIDS.model)),
manufacturer: asText(get(OIDS.manufacturer)),
statusText: text || "unknown",
onBattery,
lowBattery: battery === 3 || battery === 4,
replaceBattery: false,
overload: false,
charge: asNumber(get(OIDS.chargeRemaining)),
runtimeSeconds: minutes !== null ? minutes * 60 : null,
loadPercent: asNumber(get(OIDS.outputLoad)),
inputVoltage: asNumber(get(OIDS.inputVoltage)),
outputVoltage: asNumber(get(OIDS.outputVoltage)),
};
}
/** One request and its answer over UDP, tried a second time when nothing comes back. */
function exchange(address: string, port: number, request: Buffer, requestId: number): Promise<SnmpAnswer> {
const type = net.isIP(address) === 6 ? "udp6" : "udp4";
return new Promise((resolve, reject) => {
const socket = dgram.createSocket(type);
let tries = 0;
let timer: ReturnType<typeof setTimeout>;
const finish = (err: Error | null, answer?: SnmpAnswer) => {
clearTimeout(timer);
socket.close();
if (err) reject(err);
else resolve(answer!);
};
const send = () => {
tries++;
socket.send(request, port, address, (err) => err && finish(err));
timer = setTimeout(() => (tries < TRIES ? send() : finish(new Error(`no answer within ${(TIMEOUT_MS * TRIES) / 1000} s — is SNMP on, and is the community right?`))), TIMEOUT_MS);
};
socket.on("error", (err) => finish(err));
socket.on("message", (msg) => {
try {
const answer = parseResponse(msg);
if (answer.requestId === requestId) finish(null, answer);
} catch {
// something else arriving on this port: ignored
}
});
send();
});
}
export async function readSnmp(host: string, port: number, community: string): Promise<UpsReading> {
const address = await resolveTarget(host, { allowPrivate: true });
const requestId = 1 + Math.floor(Math.random() * 0x7ffffff0);
const answer = await exchange(address, port, buildGet(community, requestId, Object.values(OIDS)), requestId);
if (answer.errorStatus !== 0) throw new Error(`the device refused the request (error ${answer.errorStatus})`);
const known = [...answer.values.values()].filter((v) => v !== null).length;
if (known === 0) throw new Error("the device answers, but has none of the standard UPS objects (UPS-MIB)");
return interpretSnmp(answer.values);
}
+34
View File
@@ -0,0 +1,34 @@
/** What a UPS says about itself, the same whichever way it was asked (NUT or SNMP). Anything it doesn't say is null. */
export interface UpsReading {
model: string | null;
manufacturer: string | null;
/** The UPS's own words for how it is ("OL CHRG", "on battery"), to show. */
statusText: string;
/** Mains power is gone and it is running from its battery. */
onBattery: boolean;
/** The battery is nearly empty: shut things down now. */
lowBattery: boolean;
/** The UPS asks for its battery to be replaced. */
replaceBattery: boolean;
/** It is carrying more than it can. */
overload: boolean;
/** Battery charge, percent. */
charge: number | null;
/** How long the battery would last at the present load. */
runtimeSeconds: number | null;
/** How much of its capacity the load uses, percent. */
loadPercent: number | null;
inputVoltage: number | null;
outputVoltage: number | null;
}
export interface UpsConfig {
host: string;
port?: string;
/** Read over SNMP (version 2c) instead of NUT. */
snmp?: boolean | string;
/** NUT: the UPS's name on that server; the first one it lists when left blank. */
upsName?: string;
/** SNMP: the read community; "public" when left blank. */
community?: string;
}
+4
View File
@@ -27,6 +27,7 @@ import { attachMatchedServers, summarizeMonitors, toMatchableServers } from "../
import { getSourceStatus } from "../services/integrationHealthMonitor.js";
import { removeIntegrationData, syncUnifiIntegration } from "../services/unifiSync.js";
import { removeHomeAssistantData, syncHomeAssistant } from "../services/homeAssistantSync.js";
import { forgetUps, syncUps } from "../services/upsSync.js";
import { invalidateAlertsCache } from "../services/alerts.js";
import { asyncHandler } from "../utils/asyncHandler.js";
import { ID_PATTERNS, isValidId } from "../utils/pathSegment.js";
@@ -127,6 +128,7 @@ integrationsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res)
// A new UniFi controller is read at once, so its equipment is there when the person opens the page (not waited for: a slow controller must not slow this).
if (type === "unifi") void syncUnifiIntegration(created.id).then(() => invalidateAlertsCache());
if (type === "homeassistant") void syncHomeAssistant(created.id).then(() => invalidateAlertsCache());
if (type === "ups") void syncUps(created.id).then(() => invalidateAlertsCache());
res.status(201).json({
integration: {
@@ -229,6 +231,7 @@ integrationsRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req,
if (updated.type === "unifi" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncUnifiIntegration(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "homeassistant" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncHomeAssistant(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "ups" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncUps(updated.id).then(() => invalidateAlertsCache());
res.json({
integration: {
@@ -307,6 +310,7 @@ integrationsRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req,
if (existing.type === "unifi") await removeIntegrationData(id);
if (existing.type === "homeassistant") await removeHomeAssistantData(id);
if (existing.type === "ups") await forgetUps(id);
await db.delete(integrations).where(eq(integrations.id, id));
if (existing.credentialId) {
await db.delete(integrationCredentials).where(eq(integrationCredentials.id, existing.credentialId));
+77
View File
@@ -0,0 +1,77 @@
import { Router } from "express";
import { and, asc, eq, gte } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, upsReadings, upsState } from "../db/schema.js";
import { requireAuth } from "../auth/middleware.js";
import { loadGraph } from "../services/dependencies.js";
import { bucketUps, isUpsFresh, READING_KEEP_DAYS, shutdownOrder, upsProblems, upsSyncStatuses } from "../services/upsSync.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const upsRouter = Router();
upsRouter.use(requireAuth);
const DAY_MS = 86_400_000;
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
const isSnmp = (config: string | null) => {
try {
const c = config ? (JSON.parse(config) as Record<string, unknown>) : {};
return c.snmp === true || c.snmp === "true";
} catch {
return false;
}
};
/** Every UPS, how it is now, and what is on it in the order to shut it down. Everyone signed in can see it, like the other pages made of stored readings. */
upsRouter.get("/", asyncHandler(async (_req, res) => {
const [rows, states, statuses, graph] = await Promise.all([
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled, config: integrations.config }).from(integrations).where(eq(integrations.type, "ups")).orderBy(integrations.name),
db.select().from(upsState),
upsSyncStatuses(),
loadGraph(),
]);
const now = new Date();
const byId = new Map(states.map((s) => [s.integrationId, s]));
res.json({
ups: rows.map((r) => {
const state = byId.get(r.id) ?? null;
const status = statuses[r.id] ?? null;
const fresh = isUpsFresh(status ?? undefined, now.getTime());
const { order, looped } = shutdownOrder(graph.edges, `integration:${r.id}`);
return {
id: r.id,
name: r.name,
enabled: r.enabled,
protocol: isSnmp(r.config) ? "snmp" : "nut",
status,
fresh,
state,
problems: state && fresh ? upsProblems(r.id, r.name, state, now) : [],
// What needs this UPS, directly or through something else, in the order to shut it down (step 1 first).
carries: order.map((o) => {
const [type, id] = o.key.split(":");
return { key: o.key, type, id: Number(id), name: graph.names.get(o.key) ?? o.key, step: o.step };
}),
orderLooped: looped,
};
}),
});
}));
/** One UPS's charge, load and mains voltage over time (readings are kept 35 days). */
upsRouter.get("/:id/history", asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [row] = await db.select({ id: integrations.id, name: integrations.name, type: integrations.type }).from(integrations).where(eq(integrations.id, id)).limit(1);
if (!row || row.type !== "ups") return res.status(404).json({ error: "not_found" });
const asked = Number(req.query.days);
const days = Number.isFinite(asked) ? Math.min(READING_KEEP_DAYS, Math.max(1, Math.round(asked))) : 7;
const to = Date.now();
const from = to - days * DAY_MS;
const rows = await db
.select()
.from(upsReadings)
.where(and(eq(upsReadings.integrationId, id), gte(upsReadings.at, new Date(from).toISOString())))
.orderBy(asc(upsReadings.at));
res.json({ ups: { id: row.id, name: row.name }, days, ...bucketUps(rows, from, to, 200) });
}));
+7 -3
View File
@@ -72,6 +72,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
unifi: { label: "UniFi", link: "/network/devices" },
npm: { label: "Nginx Proxy Manager", link: "/proxy-hosts" },
homeassistant: { label: "Home Assistant", link: "/home-assistant" },
ups: { label: "UPS", link: "/ups" },
secrets: { label: "Secrets", link: "/secrets" },
domains: { label: "Domains", link: "/dns?view=domains" },
endpoint: { label: "Endpoint", link: "/endpoints" },
@@ -100,6 +101,8 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
if (m) return { type: "synology", id: Number(m[1]), category: "disk" };
m = /^automation:(semaphore|gitea):(\d+):/.exec(key);
if (m) return { type: m[1], id: Number(m[2]), category: "automation" };
m = /^ups-(?:battery|low|replace|load):(\d+)$/.exec(key);
if (m) return { type: "ups", id: Number(m[1]), category: "monitoring" };
return { type: "server", id: null, category: "disk" };
}
@@ -213,22 +216,23 @@ export async function collectAlerts(): Promise<AlertsReport> {
const offlineMatch = /^offline:(server|device):(\d+)$/.exec(c.key);
const offlineNode = offlineMatch ? `${offlineMatch[1]}:${offlineMatch[2]}` : undefined;
const parentOf = offlineNode ? cover.cause.get(offlineNode) : undefined;
const upsOnBattery = /^ups-battery:(\d+)$/.exec(c.key);
push({
causedBy: parentOf !== undefined ? { name: cover.graph.names.get(parentOf) ?? parentOf, alertId: alertIdOfNode(parentOf, cover.graph) ?? undefined } : undefined,
affects: offlineNode ? cover.taken.get(offlineNode)?.sort((a, b) => a.localeCompare(b)) : undefined,
affects: offlineNode ? cover.taken.get(offlineNode)?.sort((a, b) => a.localeCompare(b)) : upsOnBattery ? cover.taken.get(`integration:${upsOnBattery[1]}`)?.sort((a, b) => a.localeCompare(b)) : undefined,
category: origin.category,
severity: c.severity ?? "warning",
type: origin.type,
message: c.message,
key: c.key,
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : undefined,
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : undefined,
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
});
}
// Integrations the check couldn't read this time. (A whole-integration entry covers its nodes, so skip those.)
if (held.has("server:rules")) cannotCheck("Alert rules", "couldn't be judged — see the server log", "Server and storage health");
for (const h of held) {
const m = /^(proxmox|synology|unifi):(\d+)(?::(.+))?$/.exec(h);
const m = /^(proxmox|synology|unifi|ups):(\d+)(?::(.+))?$/.exec(h);
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
const row = intById.get(Number(m[2]));
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
+1
View File
@@ -36,6 +36,7 @@ export const HISTORY: Record<string, HistoryMeta> = {
heartbeat_pings: { time: "at", keep: 14 },
storage_samples: { time: "at", keep: 400, note: "one reading an hour of each Proxmox storage and Synology volume" },
ha_readings: { time: "at", keep: 35, note: "the watched Home Assistant numbers" },
ups_readings: { time: "at", keep: 35, note: "a UPS's charge, load and mains voltage" },
alert_incidents: { time: "resolved_at", keep: 400, where: "resolved_at IS NOT NULL", note: "trimmed 400 days after an incident resolved" },
webhook_deliveries: { time: "created_at", keep: 14, note: "the last 100 per webhook, and none older than two weeks" },
audit_log: { time: "created_at", keep: "log", note: "who did what" },
+6 -1
View File
@@ -7,7 +7,7 @@
* device each is connected through, as a UniFi controller says), plus the dependencies written here. Servers, network devices and integrations are the things
* that can be down and be needed. (DNS providers raise no alerts of their own, so there is
* nothing to fold.) An integration is judged per kind of service, as its health is: it is folded only if every enabled instance of that
* kind is explained.
* kind is explained. A UPS is down while it runs on battery (services/upsSync.ts).
*/
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
@@ -15,6 +15,7 @@ import { dependencies, integrations, networkDevices, networkServerLinks, servers
import { getInternalFlag } from "./settingsStore.js";
import { listActiveWindows } from "./maintenance.js";
import { loadParentRows, resolveParents, type ParentLink } from "./serverParents.js";
import { upsOnBatteryIds } from "./upsSync.js";
/** Where the health monitor keeps the problems it is announcing; read here so a service's alert can tell a server it needs is down. */
export const HEALTH_STATE_FLAG = "healthActiveConditions";
@@ -180,6 +181,8 @@ export function alertIdOfNode(key: string, graph: Graph): string | null {
if (m[1] === "server") return `offline:offline:server:${m[2]}`;
if (m[1] === "device") return `offline:offline:device:${m[2]}`;
const row = graph.intRows.find((r) => r.id === Number(m[2]));
// A UPS is "down" for the map while it runs on battery, and its alert is the one about that.
if (row?.type === "ups") return `monitoring:ups-battery:${row.id}`;
return row ? `integration:failing:${row.type}` : null;
}
@@ -229,6 +232,8 @@ export async function dependencyCover(offlineServerIds: Iterable<number>, failin
for (const id of offlineDeviceIds) down.add(nodeKey("device", id));
const failing = new Set(failingTypes);
for (const [type, ids] of graph.instances) if (failing.has(type)) for (const id of ids) down.add(nodeKey("integration", id));
// A UPS running on battery counts as down: whatever it carries is about to follow it, and is folded under its alert if it does.
for (const id of await upsOnBatteryIds()) if (graph.names.has(nodeKey("integration", id))) down.add(nodeKey("integration", id));
const cause = coveredNodes(down, graph.edges);
const taken = new Map<string, string[]>();
for (const [child, parent] of cause) taken.set(parent, [...(taken.get(parent) ?? []), graph.names.get(child) ?? child]);
+31 -2
View File
@@ -1,6 +1,6 @@
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, networkDevices, serverPorts, servers } from "../db/schema.js";
import { integrations, networkDevices, serverPorts, servers, upsState } from "../db/schema.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
@@ -17,6 +17,7 @@ import { parseJson, type AgentPort } from "./agentPorts.js";
import { diskLimit, parseThresholds, type ServerThresholds } from "./serverThresholds.js";
import { computeRuleFindings, findingKey, listRules, type RuleFinding } from "./alertRules.js";
import { recordStorageSamples } from "./storageHistory.js";
import { isUpsFresh, upsProblems, upsSyncStatuses, type UpsStateRow } from "./upsSync.js";
const STATE_FLAG = HEALTH_STATE_FLAG;
/** After a restart the agents haven't had a chance to report yet (they run every 15 min), so server-derived conditions are held rather than judged. */
@@ -63,11 +64,18 @@ export interface UnifiSnapshot {
integrationName: string;
devices: { id: number; name: string; kind: string; model: string | null; state: string; offlineSince: string | null }[];
}
/** One UPS as last read, only offered while that reading is recent enough to be believed. */
export interface UpsSnapshot {
integrationId: number;
integrationName: string;
state: UpsStateRow;
}
export interface HealthSnapshot {
servers: ServerSnapshot[];
proxmox: ProxmoxSnapshot[];
synology: SynologySnapshot[];
unifi?: UnifiSnapshot[];
ups?: UpsSnapshot[];
/** What the administrator's own alert rules find on the servers (services/alertRules.ts). */
rules?: RuleFinding[];
}
@@ -183,6 +191,11 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
}
}
// A UPS that has lost mains power, is nearly empty, wants a new battery or is overloaded.
for (const u of snapshot.ups ?? []) {
for (const p of upsProblems(u.integrationId, u.integrationName, u.state, new Date(now))) out.push({ key: p.key, source: `ups:${u.integrationId}`, message: p.message, severity: p.severity });
}
for (const px of snapshot.proxmox) {
const seenShared = new Set<string>();
for (const node of px.nodes) {
@@ -395,6 +408,21 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
}
}
// UPSs are read by their own pass every minute; what is judged here is what that pass stored, and only while it is recent.
const upsRows = rows.filter((r) => r.type === "ups");
if (upsRows.length > 0) {
const statuses = await upsSyncStatuses();
const nowMs = Date.now();
for (const row of upsRows) {
const [state] = await db.select().from(upsState).where(eq(upsState.integrationId, row.id)).limit(1);
if (!state || !isUpsFresh(statuses[row.id], nowMs)) {
held.add(`ups:${row.id}`);
continue;
}
(snapshot.ups ??= []).push({ integrationId: row.id, integrationName: row.name, state });
}
}
for (const row of rows) {
if (row.type !== "proxmox" && row.type !== "synology") continue;
try {
@@ -494,7 +522,8 @@ export async function runHealthCheck(now: number = Date.now()): Promise<{ added:
// A host's message says what else it takes with it.
const added = announced.map((c) => {
const m = /^offline:(server|device):(\d+)$/.exec(c.key);
const children = m ? cover.taken.get(`${m[1]}:${m[2]}`) : undefined;
const ups = /^ups-battery:(\d+)$/.exec(c.key);
const children = m ? cover.taken.get(`${m[1]}:${m[2]}`) : ups ? cover.taken.get(`integration:${ups[1]}`) : undefined;
return children?.length ? { ...c, message: `${c.message} — also takes down ${alsoList(children)}` } : c;
});
+2
View File
@@ -37,6 +37,8 @@ export function subjectOfConditionKey(key: string): string | null {
if (m) return `integration:${m[1]}`;
m = /^automation:(?:semaphore|gitea):(\d+):/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^ups-(?:battery|low|replace|load):(\d+)$/.exec(key);
if (m) return `integration:${m[1]}`;
return null;
}
+23 -1
View File
@@ -3,8 +3,10 @@
* severity, integrations, maintenance, things that expire. It reads what the app already knows — nothing here calls out to a server or an
* integration on its own, so a scrape every fifteen seconds costs a few database reads (the alerts are the app's own minute-old result).
*/
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers } from "../db/schema.js";
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState } from "../db/schema.js";
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
import { latestTests } from "./restoreTests.js";
import { heartbeatState, parseStamp } from "./heartbeats.js";
import { env } from "../env.js";
@@ -56,6 +58,25 @@ class Exposition {
}
}
/** Each UPS, for /metrics: whether it is on battery, how full the battery is, how long it would last and how loaded it is. */
async function addUpsMetrics(out: Exposition, now: Date) {
const fresh = new Set<number>();
const nowMs = now.getTime();
const statuses = await upsSyncStatuses();
for (const [id, s] of Object.entries(statuses)) if (isUpsFresh(s, nowMs)) fresh.add(Number(id));
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "ups"))).map((r) => [r.id, r]));
for (const s of await db.select().from(upsState)) {
const row = names.get(s.integrationId);
if (!row || !row.enabled || !fresh.has(s.integrationId)) continue;
const labels = { name: row.name };
out.add("homelab_ups_on_battery", "gauge", "1 if the UPS has lost mains power and runs on its battery, else 0.", s.onBattery ? 1 : 0, labels);
if (s.charge !== null) out.add("homelab_ups_battery_charge_percent", "gauge", "How full the UPS's battery is.", s.charge, labels);
if (s.runtimeSeconds !== null) out.add("homelab_ups_runtime_seconds", "gauge", "How long the UPS's battery would last at the present load.", s.runtimeSeconds, labels);
if (s.loadPercent !== null) out.add("homelab_ups_load_percent", "gauge", "How much of its capacity the UPS's load uses.", s.loadPercent, labels);
if (s.inputVoltage !== null) out.add("homelab_ups_input_volts", "gauge", "The mains voltage the UPS sees.", s.inputVoltage, labels);
}
}
/** Endpoint checks and restore tests, for /metrics. */
async function addEndpointMetrics(out: Exposition, now: Date) {
for (const c of await db.select().from(endpointChecks)) {
@@ -217,6 +238,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
out.add("homelab_scrape_duration_seconds", "gauge", "How long building this answer took.", (Date.now() - started) / 1000);
out.add("homelab_scrape_timestamp_seconds", "gauge", "When this answer was built, as a Unix time.", nowSec);
await addEndpointMetrics(out, now);
await addUpsMetrics(out, now);
await addHeartbeatMetrics(out, now);
return out.toString();
}
+280
View File
@@ -0,0 +1,280 @@
/**
* Keeps what each UPS says up to date: every minute the UPS integrations are read (NUT or SNMP — integrations/ups), the state is stored, and a reading is added to the
* history (every five minutes, and every minute while it runs on battery). The page, the charts, the alerts and the dependency map are built from what is stored, so
* none of them talks to a UPS, and one that can't be reached costs only how fresh they are.
*
* What makes an alert: mains power lost (the UPS runs on battery), a nearly empty battery, a battery to be replaced, and a load near its capacity. While a UPS is on
* battery it counts as *down* in the dependency map, so the servers that go offline with it are folded under its alert instead of each being announced.
*/
import { and, desc, eq, gte, lt } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, upsReadings, upsState } from "../db/schema.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createUpsAdapter } from "../integrations/ups/adapter.js";
import type { UpsReading } from "../integrations/ups/types.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
export const SYNC_INTERVAL_MS = 60_000;
/** After this long without a good read, what is stored is no longer believed: alerts for it are held rather than judged. */
export const STALE_AFTER_MS = 5 * 60_000;
export const READING_KEEP_DAYS = 35;
const READING_GAP_MS = 5 * 60_000 - 20_000;
/** A load this high (percent of capacity) is a warning. */
export const OVERLOAD_PERCENT = 90;
const STATUS_FLAG = "upsSync";
export interface UpsSyncStatus {
/** The last read that worked (kept when a later one failed). */
at: string;
ok: boolean;
error: string | null;
}
async function readStatuses(): Promise<Record<string, UpsSyncStatus>> {
try {
const raw = await getInternalFlag(STATUS_FLAG);
return raw ? (JSON.parse(raw) as Record<string, UpsSyncStatus>) : {};
} catch {
return {};
}
}
export async function upsSyncStatuses(): Promise<Record<number, UpsSyncStatus>> {
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
}
export function isUpsFresh(status: UpsSyncStatus | undefined, nowMs: number): boolean {
return !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
}
export type UpsStateRow = typeof upsState.$inferSelect;
/** Writes down one read: the state (with when mains power was lost, kept until it is back) and, when it is time, a point in the history. */
export async function storeReading(integrationId: number, r: UpsReading, now: Date = new Date()): Promise<void> {
const nowIso = now.toISOString();
const [before] = await db.select().from(upsState).where(eq(upsState.integrationId, integrationId)).limit(1);
const onBatterySince = r.onBattery ? (before?.onBattery && before.onBatterySince ? before.onBatterySince : nowIso) : null;
const values = {
model: r.model,
manufacturer: r.manufacturer,
statusText: r.statusText,
onBattery: r.onBattery,
lowBattery: r.lowBattery,
replaceBattery: r.replaceBattery,
overload: r.overload,
charge: r.charge,
runtimeSeconds: r.runtimeSeconds,
loadPercent: r.loadPercent,
inputVoltage: r.inputVoltage,
outputVoltage: r.outputVoltage,
onBatterySince,
updatedAt: nowIso,
};
await db.insert(upsState).values({ integrationId, ...values }).onConflictDoUpdate({ target: upsState.integrationId, set: values });
// Every minute while on battery (that is when it matters), and when it changes; otherwise every five.
const [newest] = await db.select({ at: upsReadings.at }).from(upsReadings).where(eq(upsReadings.integrationId, integrationId)).orderBy(desc(upsReadings.at)).limit(1);
const changed = !before || before.onBattery !== r.onBattery;
if (changed || r.onBattery || !newest || now.getTime() - Date.parse(newest.at) >= READING_GAP_MS) {
await db.insert(upsReadings).values({ integrationId, at: nowIso, charge: r.charge, runtimeSeconds: r.runtimeSeconds, loadPercent: r.loadPercent, inputVoltage: r.inputVoltage, onBattery: r.onBattery });
}
await db.delete(upsReadings).where(and(eq(upsReadings.integrationId, integrationId), lt(upsReadings.at, new Date(now.getTime() - READING_KEEP_DAYS * 86_400_000).toISOString())));
}
/** Reads one UPS and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
export async function syncUps(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<UpsReading>): Promise<UpsSyncStatus> {
let status: UpsSyncStatus;
try {
const loaded = await loadIntegrationConfig(integrationId);
if (!loaded || loaded.integration.type !== "ups") throw new Error("not a UPS integration");
const reading = await (read ? read(loaded.config) : createUpsAdapter(loaded.config as never).read());
await storeReading(integrationId, reading, now);
status = { at: now.toISOString(), ok: true, error: null };
} catch (err) {
const previous = (await readStatuses())[String(integrationId)];
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
console.error(`[ups] couldn't read integration ${integrationId}:`, status.error);
}
try {
const all = await readStatuses();
all[String(integrationId)] = status;
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
} catch (err) {
console.error("[ups] couldn't record the result:", err);
}
return status;
}
export async function syncAllUps(now: Date = new Date()): Promise<Record<number, UpsSyncStatus>> {
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "ups"), eq(integrations.enabled, true)));
const out: Record<number, UpsSyncStatus> = {};
for (const r of rows) out[r.id] = await syncUps(r.id, now);
return out;
}
/** Forgets how a UPS's last read went when it is removed (its state and history go with its row). */
export async function forgetUps(integrationId: number): Promise<void> {
const all = await readStatuses();
if (String(integrationId) in all) {
delete all[String(integrationId)];
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
}
}
/** The ids of the UPSs that are running on battery, as last read and believed: they count as down in the dependency map. */
export async function upsOnBatteryIds(now: Date = new Date()): Promise<number[]> {
try {
const rows = await db
.select({ id: upsState.integrationId })
.from(upsState)
.innerJoin(integrations, eq(integrations.id, upsState.integrationId))
.where(and(eq(upsState.onBattery, true), eq(integrations.enabled, true), gte(upsState.updatedAt, new Date(now.getTime() - STALE_AFTER_MS).toISOString())));
return rows.map((r) => r.id);
} catch (err) {
console.error("[ups] couldn't tell which are on battery:", err);
return [];
}
}
// ─── What is wrong ──────────────────────────────────────────────────────────
export interface UpsProblem {
key: string;
kind: "battery" | "low" | "replace" | "load";
severity: "critical" | "warning";
message: string;
}
const minutesWords = (seconds: number) => {
const m = Math.max(1, Math.round(seconds / 60));
return m < 120 ? `${m} min` : `${Math.round(m / 60)} h`;
};
const since = (iso: string, now: Date) => {
const m = Math.max(0, Math.floor((now.getTime() - Date.parse(iso)) / 60_000));
return m < 1 ? "less than a minute" : m < 120 ? `${m} min` : `${Math.round(m / 60)} h`;
};
/** Pure: what is wrong with one UPS, from what is stored for it. */
export function upsProblems(id: number, name: string, s: Pick<UpsStateRow, "onBattery" | "lowBattery" | "replaceBattery" | "overload" | "charge" | "runtimeSeconds" | "loadPercent" | "onBatterySince">, now: Date): UpsProblem[] {
const out: UpsProblem[] = [];
const left = s.runtimeSeconds !== null ? `${minutesWords(s.runtimeSeconds)} left` : null;
const charge = s.charge !== null ? `${Math.round(s.charge)} %` : null;
if (s.onBattery) {
const detail = [s.onBatterySince ? `for ${since(s.onBatterySince, now)}` : null, left, charge].filter(Boolean).join(", ");
out.push({ key: `ups-battery:${id}`, kind: "battery", severity: "critical", message: `${name}: mains power has failed — running on battery${detail ? ` (${detail})` : ""}` });
}
if (s.lowBattery) {
out.push({ key: `ups-low:${id}`, kind: "low", severity: "critical", message: `${name}: the battery is nearly empty${[charge, left].filter(Boolean).length ? ` (${[charge, left].filter(Boolean).join(", ")})` : ""} — shut down what it carries` });
}
if (s.replaceBattery) out.push({ key: `ups-replace:${id}`, kind: "replace", severity: "warning", message: `${name}: the UPS asks for its battery to be replaced` });
if (s.overload || (s.loadPercent !== null && s.loadPercent >= OVERLOAD_PERCENT)) {
out.push({ key: `ups-load:${id}`, kind: "load", severity: "warning", message: `${name} is carrying ${s.loadPercent !== null ? `${Math.round(s.loadPercent)} % of` : "more than"} what it can` });
}
return out;
}
// ─── What is on it, and in what order to shut it down ───────────────────────
/**
* Pure: everything that needs `key` (directly or through others), in the order to shut it down — what nothing else needs first, then what that needed, and so on, so
* nothing is switched off while something that depends on it is still running. Each comes with its step (1 = first). `looped` is set if the order could not be settled
* because some of them need each other.
*/
export function shutdownOrder(edges: Map<string, string[]>, key: string): { order: { key: string; step: number }[]; looped: boolean } {
const reverse = new Map<string, string[]>();
for (const [from, tos] of edges) for (const to of tos) reverse.set(to, [...(reverse.get(to) ?? []), from]);
const set = new Set<string>();
const queue = [key];
while (queue.length > 0) {
for (const d of reverse.get(queue.shift()!) ?? []) {
if (d === key || set.has(d)) continue;
set.add(d);
queue.push(d);
}
}
// A node's step is one more than the latest step of anything in the set that needs it.
const step = new Map<string, number>();
let looped = false;
const visiting = new Set<string>();
const stepOf = (n: string): number => {
if (step.has(n)) return step.get(n)!;
if (visiting.has(n)) {
looped = true;
return 1;
}
visiting.add(n);
const above = (reverse.get(n) ?? []).filter((d) => set.has(d));
const s = above.length === 0 ? 1 : 1 + Math.max(...above.map(stepOf));
visiting.delete(n);
step.set(n, s);
return s;
};
for (const n of set) stepOf(n);
return { order: [...set].map((k) => ({ key: k, step: step.get(k) ?? 1 })).sort((a, b) => a.step - b.step || a.key.localeCompare(b.key)), looped };
}
// ─── History ────────────────────────────────────────────────────────────────
export interface UpsPoint {
/** ISO. */
t: string;
charge: number | null;
runtimeSeconds: number | null;
loadPercent: number | null;
inputVoltage: number | null;
/** It was on battery at some time in this stretch. */
onBattery: boolean;
}
type ReadingRow = { at: string; charge: number | null; runtimeSeconds: number | null; loadPercent: number | null; inputVoltage: number | null; onBattery: boolean };
/** Pure: readings into at most `maxPoints` equal slices of [from, to] (each the average of what it holds), and the stretches spent on battery. */
export function bucketUps(rows: ReadingRow[], from: number, to: number, maxPoints: number): { points: UpsPoint[]; onBatteryPeriods: { from: string; to: string; ongoing: boolean }[] } {
const size = Math.max((to - from) / maxPoints, 1);
const buckets = new Map<number, { t: number; n: number; sums: Record<"charge" | "runtimeSeconds" | "loadPercent" | "inputVoltage", { sum: number; n: number }>; onBattery: boolean }>();
for (const r of rows) {
const t = Date.parse(r.at);
if (t < from || t > to) continue;
const key = Math.min(maxPoints - 1, Math.floor((t - from) / size));
const b = buckets.get(key) ?? { t: 0, n: 0, sums: { charge: { sum: 0, n: 0 }, runtimeSeconds: { sum: 0, n: 0 }, loadPercent: { sum: 0, n: 0 }, inputVoltage: { sum: 0, n: 0 } }, onBattery: false };
b.t += t;
b.n += 1;
b.onBattery ||= r.onBattery;
for (const k of ["charge", "runtimeSeconds", "loadPercent", "inputVoltage"] as const) {
const v = r[k];
if (v !== null) {
b.sums[k].sum += v;
b.sums[k].n += 1;
}
}
buckets.set(key, b);
}
const avg = (x: { sum: number; n: number }) => (x.n === 0 ? null : Math.round((x.sum / x.n) * 10) / 10);
const points = [...buckets.entries()].sort((a, b) => a[0] - b[0]).map(([, b]) => ({ t: new Date(b.t / b.n).toISOString(), charge: avg(b.sums.charge), runtimeSeconds: avg(b.sums.runtimeSeconds), loadPercent: avg(b.sums.loadPercent), inputVoltage: avg(b.sums.inputVoltage), onBattery: b.onBattery }));
// A stretch on battery runs from the first reading that says so to the first that says it is back.
const periods: { from: string; to: string; ongoing: boolean }[] = [];
let start: string | null = null;
let last: string | null = null;
for (const r of [...rows].sort((a, b) => a.at.localeCompare(b.at))) {
if (r.onBattery) {
start ??= r.at;
last = r.at;
} else if (start !== null) {
periods.push({ from: start, to: r.at, ongoing: false });
start = null;
}
}
// One still going has no end yet: it runs to the last reading that says it is on battery, and says so.
if (start !== null && last !== null) periods.push({ from: start, to: last, ongoing: true });
return { points, onBatteryPeriods: periods };
}
let timer: ReturnType<typeof setInterval> | null = null;
export function initUpsSync(): void {
if (timer) return;
const pass = () => void syncAllUps().catch((err) => console.error("[ups] pass failed:", err));
timer = setInterval(pass, SYNC_INTERVAL_MS);
setTimeout(pass, 9_000);
console.log(`[ups] reading UPSs every ${SYNC_INTERVAL_MS / 1000} seconds`);
}