A UPS in the map: read over NUT or SNMP, alerts, shutdown order
New UPS integration (NUT or SNMP v2c, read only). Monitoring -> UPS shows charge, runtime, load and mains voltage with history and the times mains power was lost. Mains lost and a nearly empty battery are critical alerts; battery to be replaced and a heavy load are warnings. The UPS is a node in the dependency map and counts as down while on battery, so servers that go offline with it are folded under its alert; the page lists what is on it in the order to shut it down. /metrics series and docs added. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3cb3e4988f
commit
232a5661ab
38 files changed
+8049
-16
No files matched your search
@@ -72,6 +72,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
|
||||
unifi: { label: "UniFi", link: "/network/devices" },
|
||||
npm: { label: "Nginx Proxy Manager", link: "/proxy-hosts" },
|
||||
homeassistant: { label: "Home Assistant", link: "/home-assistant" },
|
||||
ups: { label: "UPS", link: "/ups" },
|
||||
secrets: { label: "Secrets", link: "/secrets" },
|
||||
domains: { label: "Domains", link: "/dns?view=domains" },
|
||||
endpoint: { label: "Endpoint", link: "/endpoints" },
|
||||
@@ -100,6 +101,8 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
|
||||
if (m) return { type: "synology", id: Number(m[1]), category: "disk" };
|
||||
m = /^automation:(semaphore|gitea):(\d+):/.exec(key);
|
||||
if (m) return { type: m[1], id: Number(m[2]), category: "automation" };
|
||||
m = /^ups-(?:battery|low|replace|load):(\d+)$/.exec(key);
|
||||
if (m) return { type: "ups", id: Number(m[1]), category: "monitoring" };
|
||||
return { type: "server", id: null, category: "disk" };
|
||||
}
|
||||
|
||||
@@ -213,22 +216,23 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
const offlineMatch = /^offline:(server|device):(\d+)$/.exec(c.key);
|
||||
const offlineNode = offlineMatch ? `${offlineMatch[1]}:${offlineMatch[2]}` : undefined;
|
||||
const parentOf = offlineNode ? cover.cause.get(offlineNode) : undefined;
|
||||
const upsOnBattery = /^ups-battery:(\d+)$/.exec(c.key);
|
||||
push({
|
||||
causedBy: parentOf !== undefined ? { name: cover.graph.names.get(parentOf) ?? parentOf, alertId: alertIdOfNode(parentOf, cover.graph) ?? undefined } : undefined,
|
||||
affects: offlineNode ? cover.taken.get(offlineNode)?.sort((a, b) => a.localeCompare(b)) : undefined,
|
||||
affects: offlineNode ? cover.taken.get(offlineNode)?.sort((a, b) => a.localeCompare(b)) : upsOnBattery ? cover.taken.get(`integration:${upsOnBattery[1]}`)?.sort((a, b) => a.localeCompare(b)) : undefined,
|
||||
category: origin.category,
|
||||
severity: c.severity ?? "warning",
|
||||
type: origin.type,
|
||||
message: c.message,
|
||||
key: c.key,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : undefined,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : undefined,
|
||||
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
|
||||
});
|
||||
}
|
||||
// Integrations the check couldn't read this time. (A whole-integration entry covers its nodes, so skip those.)
|
||||
if (held.has("server:rules")) cannotCheck("Alert rules", "couldn't be judged — see the server log", "Server and storage health");
|
||||
for (const h of held) {
|
||||
const m = /^(proxmox|synology|unifi):(\d+)(?::(.+))?$/.exec(h);
|
||||
const m = /^(proxmox|synology|unifi|ups):(\d+)(?::(.+))?$/.exec(h);
|
||||
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
|
||||
const row = intById.get(Number(m[2]));
|
||||
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
|
||||
|
||||
@@ -36,6 +36,7 @@ export const HISTORY: Record<string, HistoryMeta> = {
|
||||
heartbeat_pings: { time: "at", keep: 14 },
|
||||
storage_samples: { time: "at", keep: 400, note: "one reading an hour of each Proxmox storage and Synology volume" },
|
||||
ha_readings: { time: "at", keep: 35, note: "the watched Home Assistant numbers" },
|
||||
ups_readings: { time: "at", keep: 35, note: "a UPS's charge, load and mains voltage" },
|
||||
alert_incidents: { time: "resolved_at", keep: 400, where: "resolved_at IS NOT NULL", note: "trimmed 400 days after an incident resolved" },
|
||||
webhook_deliveries: { time: "created_at", keep: 14, note: "the last 100 per webhook, and none older than two weeks" },
|
||||
audit_log: { time: "created_at", keep: "log", note: "who did what" },
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
* device each is connected through, as a UniFi controller says), plus the dependencies written here. Servers, network devices and integrations are the things
|
||||
* that can be down and be needed. (DNS providers raise no alerts of their own, so there is
|
||||
* nothing to fold.) An integration is judged per kind of service, as its health is: it is folded only if every enabled instance of that
|
||||
* kind is explained.
|
||||
* kind is explained. A UPS is down while it runs on battery (services/upsSync.ts).
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
@@ -15,6 +15,7 @@ import { dependencies, integrations, networkDevices, networkServerLinks, servers
|
||||
import { getInternalFlag } from "./settingsStore.js";
|
||||
import { listActiveWindows } from "./maintenance.js";
|
||||
import { loadParentRows, resolveParents, type ParentLink } from "./serverParents.js";
|
||||
import { upsOnBatteryIds } from "./upsSync.js";
|
||||
|
||||
/** Where the health monitor keeps the problems it is announcing; read here so a service's alert can tell a server it needs is down. */
|
||||
export const HEALTH_STATE_FLAG = "healthActiveConditions";
|
||||
@@ -180,6 +181,8 @@ export function alertIdOfNode(key: string, graph: Graph): string | null {
|
||||
if (m[1] === "server") return `offline:offline:server:${m[2]}`;
|
||||
if (m[1] === "device") return `offline:offline:device:${m[2]}`;
|
||||
const row = graph.intRows.find((r) => r.id === Number(m[2]));
|
||||
// A UPS is "down" for the map while it runs on battery, and its alert is the one about that.
|
||||
if (row?.type === "ups") return `monitoring:ups-battery:${row.id}`;
|
||||
return row ? `integration:failing:${row.type}` : null;
|
||||
}
|
||||
|
||||
@@ -229,6 +232,8 @@ export async function dependencyCover(offlineServerIds: Iterable<number>, failin
|
||||
for (const id of offlineDeviceIds) down.add(nodeKey("device", id));
|
||||
const failing = new Set(failingTypes);
|
||||
for (const [type, ids] of graph.instances) if (failing.has(type)) for (const id of ids) down.add(nodeKey("integration", id));
|
||||
// A UPS running on battery counts as down: whatever it carries is about to follow it, and is folded under its alert if it does.
|
||||
for (const id of await upsOnBatteryIds()) if (graph.names.has(nodeKey("integration", id))) down.add(nodeKey("integration", id));
|
||||
const cause = coveredNodes(down, graph.edges);
|
||||
const taken = new Map<string, string[]>();
|
||||
for (const [child, parent] of cause) taken.set(parent, [...(taken.get(parent) ?? []), graph.names.get(child) ?? child]);
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, networkDevices, serverPorts, servers } from "../db/schema.js";
|
||||
import { integrations, networkDevices, serverPorts, servers, upsState } from "../db/schema.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
|
||||
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
|
||||
@@ -17,6 +17,7 @@ import { parseJson, type AgentPort } from "./agentPorts.js";
|
||||
import { diskLimit, parseThresholds, type ServerThresholds } from "./serverThresholds.js";
|
||||
import { computeRuleFindings, findingKey, listRules, type RuleFinding } from "./alertRules.js";
|
||||
import { recordStorageSamples } from "./storageHistory.js";
|
||||
import { isUpsFresh, upsProblems, upsSyncStatuses, type UpsStateRow } from "./upsSync.js";
|
||||
|
||||
const STATE_FLAG = HEALTH_STATE_FLAG;
|
||||
/** After a restart the agents haven't had a chance to report yet (they run every 15 min), so server-derived conditions are held rather than judged. */
|
||||
@@ -63,11 +64,18 @@ export interface UnifiSnapshot {
|
||||
integrationName: string;
|
||||
devices: { id: number; name: string; kind: string; model: string | null; state: string; offlineSince: string | null }[];
|
||||
}
|
||||
/** One UPS as last read, only offered while that reading is recent enough to be believed. */
|
||||
export interface UpsSnapshot {
|
||||
integrationId: number;
|
||||
integrationName: string;
|
||||
state: UpsStateRow;
|
||||
}
|
||||
export interface HealthSnapshot {
|
||||
servers: ServerSnapshot[];
|
||||
proxmox: ProxmoxSnapshot[];
|
||||
synology: SynologySnapshot[];
|
||||
unifi?: UnifiSnapshot[];
|
||||
ups?: UpsSnapshot[];
|
||||
/** What the administrator's own alert rules find on the servers (services/alertRules.ts). */
|
||||
rules?: RuleFinding[];
|
||||
}
|
||||
@@ -183,6 +191,11 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
}
|
||||
}
|
||||
|
||||
// A UPS that has lost mains power, is nearly empty, wants a new battery or is overloaded.
|
||||
for (const u of snapshot.ups ?? []) {
|
||||
for (const p of upsProblems(u.integrationId, u.integrationName, u.state, new Date(now))) out.push({ key: p.key, source: `ups:${u.integrationId}`, message: p.message, severity: p.severity });
|
||||
}
|
||||
|
||||
for (const px of snapshot.proxmox) {
|
||||
const seenShared = new Set<string>();
|
||||
for (const node of px.nodes) {
|
||||
@@ -395,6 +408,21 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
}
|
||||
}
|
||||
|
||||
// UPSs are read by their own pass every minute; what is judged here is what that pass stored, and only while it is recent.
|
||||
const upsRows = rows.filter((r) => r.type === "ups");
|
||||
if (upsRows.length > 0) {
|
||||
const statuses = await upsSyncStatuses();
|
||||
const nowMs = Date.now();
|
||||
for (const row of upsRows) {
|
||||
const [state] = await db.select().from(upsState).where(eq(upsState.integrationId, row.id)).limit(1);
|
||||
if (!state || !isUpsFresh(statuses[row.id], nowMs)) {
|
||||
held.add(`ups:${row.id}`);
|
||||
continue;
|
||||
}
|
||||
(snapshot.ups ??= []).push({ integrationId: row.id, integrationName: row.name, state });
|
||||
}
|
||||
}
|
||||
|
||||
for (const row of rows) {
|
||||
if (row.type !== "proxmox" && row.type !== "synology") continue;
|
||||
try {
|
||||
@@ -494,7 +522,8 @@ export async function runHealthCheck(now: number = Date.now()): Promise<{ added:
|
||||
// A host's message says what else it takes with it.
|
||||
const added = announced.map((c) => {
|
||||
const m = /^offline:(server|device):(\d+)$/.exec(c.key);
|
||||
const children = m ? cover.taken.get(`${m[1]}:${m[2]}`) : undefined;
|
||||
const ups = /^ups-battery:(\d+)$/.exec(c.key);
|
||||
const children = m ? cover.taken.get(`${m[1]}:${m[2]}`) : ups ? cover.taken.get(`integration:${ups[1]}`) : undefined;
|
||||
return children?.length ? { ...c, message: `${c.message} — also takes down ${alsoList(children)}` } : c;
|
||||
});
|
||||
|
||||
|
||||
@@ -37,6 +37,8 @@ export function subjectOfConditionKey(key: string): string | null {
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^automation:(?:semaphore|gitea):(\d+):/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^ups-(?:battery|low|replace|load):(\d+)$/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
@@ -3,8 +3,10 @@
|
||||
* severity, integrations, maintenance, things that expire. It reads what the app already knows — nothing here calls out to a server or an
|
||||
* integration on its own, so a scrape every fifteen seconds costs a few database reads (the alerts are the app's own minute-old result).
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers } from "../db/schema.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState } from "../db/schema.js";
|
||||
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
|
||||
import { latestTests } from "./restoreTests.js";
|
||||
import { heartbeatState, parseStamp } from "./heartbeats.js";
|
||||
import { env } from "../env.js";
|
||||
@@ -56,6 +58,25 @@ class Exposition {
|
||||
}
|
||||
}
|
||||
|
||||
/** Each UPS, for /metrics: whether it is on battery, how full the battery is, how long it would last and how loaded it is. */
|
||||
async function addUpsMetrics(out: Exposition, now: Date) {
|
||||
const fresh = new Set<number>();
|
||||
const nowMs = now.getTime();
|
||||
const statuses = await upsSyncStatuses();
|
||||
for (const [id, s] of Object.entries(statuses)) if (isUpsFresh(s, nowMs)) fresh.add(Number(id));
|
||||
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "ups"))).map((r) => [r.id, r]));
|
||||
for (const s of await db.select().from(upsState)) {
|
||||
const row = names.get(s.integrationId);
|
||||
if (!row || !row.enabled || !fresh.has(s.integrationId)) continue;
|
||||
const labels = { name: row.name };
|
||||
out.add("homelab_ups_on_battery", "gauge", "1 if the UPS has lost mains power and runs on its battery, else 0.", s.onBattery ? 1 : 0, labels);
|
||||
if (s.charge !== null) out.add("homelab_ups_battery_charge_percent", "gauge", "How full the UPS's battery is.", s.charge, labels);
|
||||
if (s.runtimeSeconds !== null) out.add("homelab_ups_runtime_seconds", "gauge", "How long the UPS's battery would last at the present load.", s.runtimeSeconds, labels);
|
||||
if (s.loadPercent !== null) out.add("homelab_ups_load_percent", "gauge", "How much of its capacity the UPS's load uses.", s.loadPercent, labels);
|
||||
if (s.inputVoltage !== null) out.add("homelab_ups_input_volts", "gauge", "The mains voltage the UPS sees.", s.inputVoltage, labels);
|
||||
}
|
||||
}
|
||||
|
||||
/** Endpoint checks and restore tests, for /metrics. */
|
||||
async function addEndpointMetrics(out: Exposition, now: Date) {
|
||||
for (const c of await db.select().from(endpointChecks)) {
|
||||
@@ -217,6 +238,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
out.add("homelab_scrape_duration_seconds", "gauge", "How long building this answer took.", (Date.now() - started) / 1000);
|
||||
out.add("homelab_scrape_timestamp_seconds", "gauge", "When this answer was built, as a Unix time.", nowSec);
|
||||
await addEndpointMetrics(out, now);
|
||||
await addUpsMetrics(out, now);
|
||||
await addHeartbeatMetrics(out, now);
|
||||
return out.toString();
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
/**
|
||||
* Keeps what each UPS says up to date: every minute the UPS integrations are read (NUT or SNMP — integrations/ups), the state is stored, and a reading is added to the
|
||||
* history (every five minutes, and every minute while it runs on battery). The page, the charts, the alerts and the dependency map are built from what is stored, so
|
||||
* none of them talks to a UPS, and one that can't be reached costs only how fresh they are.
|
||||
*
|
||||
* What makes an alert: mains power lost (the UPS runs on battery), a nearly empty battery, a battery to be replaced, and a load near its capacity. While a UPS is on
|
||||
* battery it counts as *down* in the dependency map, so the servers that go offline with it are folded under its alert instead of each being announced.
|
||||
*/
|
||||
import { and, desc, eq, gte, lt } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, upsReadings, upsState } from "../db/schema.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createUpsAdapter } from "../integrations/ups/adapter.js";
|
||||
import type { UpsReading } from "../integrations/ups/types.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export const SYNC_INTERVAL_MS = 60_000;
|
||||
/** After this long without a good read, what is stored is no longer believed: alerts for it are held rather than judged. */
|
||||
export const STALE_AFTER_MS = 5 * 60_000;
|
||||
export const READING_KEEP_DAYS = 35;
|
||||
const READING_GAP_MS = 5 * 60_000 - 20_000;
|
||||
/** A load this high (percent of capacity) is a warning. */
|
||||
export const OVERLOAD_PERCENT = 90;
|
||||
const STATUS_FLAG = "upsSync";
|
||||
|
||||
export interface UpsSyncStatus {
|
||||
/** The last read that worked (kept when a later one failed). */
|
||||
at: string;
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
async function readStatuses(): Promise<Record<string, UpsSyncStatus>> {
|
||||
try {
|
||||
const raw = await getInternalFlag(STATUS_FLAG);
|
||||
return raw ? (JSON.parse(raw) as Record<string, UpsSyncStatus>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export async function upsSyncStatuses(): Promise<Record<number, UpsSyncStatus>> {
|
||||
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
|
||||
}
|
||||
|
||||
export function isUpsFresh(status: UpsSyncStatus | undefined, nowMs: number): boolean {
|
||||
return !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
|
||||
}
|
||||
|
||||
export type UpsStateRow = typeof upsState.$inferSelect;
|
||||
|
||||
/** Writes down one read: the state (with when mains power was lost, kept until it is back) and, when it is time, a point in the history. */
|
||||
export async function storeReading(integrationId: number, r: UpsReading, now: Date = new Date()): Promise<void> {
|
||||
const nowIso = now.toISOString();
|
||||
const [before] = await db.select().from(upsState).where(eq(upsState.integrationId, integrationId)).limit(1);
|
||||
const onBatterySince = r.onBattery ? (before?.onBattery && before.onBatterySince ? before.onBatterySince : nowIso) : null;
|
||||
const values = {
|
||||
model: r.model,
|
||||
manufacturer: r.manufacturer,
|
||||
statusText: r.statusText,
|
||||
onBattery: r.onBattery,
|
||||
lowBattery: r.lowBattery,
|
||||
replaceBattery: r.replaceBattery,
|
||||
overload: r.overload,
|
||||
charge: r.charge,
|
||||
runtimeSeconds: r.runtimeSeconds,
|
||||
loadPercent: r.loadPercent,
|
||||
inputVoltage: r.inputVoltage,
|
||||
outputVoltage: r.outputVoltage,
|
||||
onBatterySince,
|
||||
updatedAt: nowIso,
|
||||
};
|
||||
await db.insert(upsState).values({ integrationId, ...values }).onConflictDoUpdate({ target: upsState.integrationId, set: values });
|
||||
|
||||
// Every minute while on battery (that is when it matters), and when it changes; otherwise every five.
|
||||
const [newest] = await db.select({ at: upsReadings.at }).from(upsReadings).where(eq(upsReadings.integrationId, integrationId)).orderBy(desc(upsReadings.at)).limit(1);
|
||||
const changed = !before || before.onBattery !== r.onBattery;
|
||||
if (changed || r.onBattery || !newest || now.getTime() - Date.parse(newest.at) >= READING_GAP_MS) {
|
||||
await db.insert(upsReadings).values({ integrationId, at: nowIso, charge: r.charge, runtimeSeconds: r.runtimeSeconds, loadPercent: r.loadPercent, inputVoltage: r.inputVoltage, onBattery: r.onBattery });
|
||||
}
|
||||
await db.delete(upsReadings).where(and(eq(upsReadings.integrationId, integrationId), lt(upsReadings.at, new Date(now.getTime() - READING_KEEP_DAYS * 86_400_000).toISOString())));
|
||||
}
|
||||
|
||||
/** Reads one UPS and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
|
||||
export async function syncUps(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<UpsReading>): Promise<UpsSyncStatus> {
|
||||
let status: UpsSyncStatus;
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
if (!loaded || loaded.integration.type !== "ups") throw new Error("not a UPS integration");
|
||||
const reading = await (read ? read(loaded.config) : createUpsAdapter(loaded.config as never).read());
|
||||
await storeReading(integrationId, reading, now);
|
||||
status = { at: now.toISOString(), ok: true, error: null };
|
||||
} catch (err) {
|
||||
const previous = (await readStatuses())[String(integrationId)];
|
||||
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
console.error(`[ups] couldn't read integration ${integrationId}:`, status.error);
|
||||
}
|
||||
try {
|
||||
const all = await readStatuses();
|
||||
all[String(integrationId)] = status;
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
} catch (err) {
|
||||
console.error("[ups] couldn't record the result:", err);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
export async function syncAllUps(now: Date = new Date()): Promise<Record<number, UpsSyncStatus>> {
|
||||
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "ups"), eq(integrations.enabled, true)));
|
||||
const out: Record<number, UpsSyncStatus> = {};
|
||||
for (const r of rows) out[r.id] = await syncUps(r.id, now);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Forgets how a UPS's last read went when it is removed (its state and history go with its row). */
|
||||
export async function forgetUps(integrationId: number): Promise<void> {
|
||||
const all = await readStatuses();
|
||||
if (String(integrationId) in all) {
|
||||
delete all[String(integrationId)];
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
}
|
||||
}
|
||||
|
||||
/** The ids of the UPSs that are running on battery, as last read and believed: they count as down in the dependency map. */
|
||||
export async function upsOnBatteryIds(now: Date = new Date()): Promise<number[]> {
|
||||
try {
|
||||
const rows = await db
|
||||
.select({ id: upsState.integrationId })
|
||||
.from(upsState)
|
||||
.innerJoin(integrations, eq(integrations.id, upsState.integrationId))
|
||||
.where(and(eq(upsState.onBattery, true), eq(integrations.enabled, true), gte(upsState.updatedAt, new Date(now.getTime() - STALE_AFTER_MS).toISOString())));
|
||||
return rows.map((r) => r.id);
|
||||
} catch (err) {
|
||||
console.error("[ups] couldn't tell which are on battery:", err);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// ─── What is wrong ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface UpsProblem {
|
||||
key: string;
|
||||
kind: "battery" | "low" | "replace" | "load";
|
||||
severity: "critical" | "warning";
|
||||
message: string;
|
||||
}
|
||||
|
||||
const minutesWords = (seconds: number) => {
|
||||
const m = Math.max(1, Math.round(seconds / 60));
|
||||
return m < 120 ? `${m} min` : `${Math.round(m / 60)} h`;
|
||||
};
|
||||
const since = (iso: string, now: Date) => {
|
||||
const m = Math.max(0, Math.floor((now.getTime() - Date.parse(iso)) / 60_000));
|
||||
return m < 1 ? "less than a minute" : m < 120 ? `${m} min` : `${Math.round(m / 60)} h`;
|
||||
};
|
||||
|
||||
/** Pure: what is wrong with one UPS, from what is stored for it. */
|
||||
export function upsProblems(id: number, name: string, s: Pick<UpsStateRow, "onBattery" | "lowBattery" | "replaceBattery" | "overload" | "charge" | "runtimeSeconds" | "loadPercent" | "onBatterySince">, now: Date): UpsProblem[] {
|
||||
const out: UpsProblem[] = [];
|
||||
const left = s.runtimeSeconds !== null ? `${minutesWords(s.runtimeSeconds)} left` : null;
|
||||
const charge = s.charge !== null ? `${Math.round(s.charge)} %` : null;
|
||||
if (s.onBattery) {
|
||||
const detail = [s.onBatterySince ? `for ${since(s.onBatterySince, now)}` : null, left, charge].filter(Boolean).join(", ");
|
||||
out.push({ key: `ups-battery:${id}`, kind: "battery", severity: "critical", message: `${name}: mains power has failed — running on battery${detail ? ` (${detail})` : ""}` });
|
||||
}
|
||||
if (s.lowBattery) {
|
||||
out.push({ key: `ups-low:${id}`, kind: "low", severity: "critical", message: `${name}: the battery is nearly empty${[charge, left].filter(Boolean).length ? ` (${[charge, left].filter(Boolean).join(", ")})` : ""} — shut down what it carries` });
|
||||
}
|
||||
if (s.replaceBattery) out.push({ key: `ups-replace:${id}`, kind: "replace", severity: "warning", message: `${name}: the UPS asks for its battery to be replaced` });
|
||||
if (s.overload || (s.loadPercent !== null && s.loadPercent >= OVERLOAD_PERCENT)) {
|
||||
out.push({ key: `ups-load:${id}`, kind: "load", severity: "warning", message: `${name} is carrying ${s.loadPercent !== null ? `${Math.round(s.loadPercent)} % of` : "more than"} what it can` });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ─── What is on it, and in what order to shut it down ───────────────────────
|
||||
|
||||
/**
|
||||
* Pure: everything that needs `key` (directly or through others), in the order to shut it down — what nothing else needs first, then what that needed, and so on, so
|
||||
* nothing is switched off while something that depends on it is still running. Each comes with its step (1 = first). `looped` is set if the order could not be settled
|
||||
* because some of them need each other.
|
||||
*/
|
||||
export function shutdownOrder(edges: Map<string, string[]>, key: string): { order: { key: string; step: number }[]; looped: boolean } {
|
||||
const reverse = new Map<string, string[]>();
|
||||
for (const [from, tos] of edges) for (const to of tos) reverse.set(to, [...(reverse.get(to) ?? []), from]);
|
||||
const set = new Set<string>();
|
||||
const queue = [key];
|
||||
while (queue.length > 0) {
|
||||
for (const d of reverse.get(queue.shift()!) ?? []) {
|
||||
if (d === key || set.has(d)) continue;
|
||||
set.add(d);
|
||||
queue.push(d);
|
||||
}
|
||||
}
|
||||
// A node's step is one more than the latest step of anything in the set that needs it.
|
||||
const step = new Map<string, number>();
|
||||
let looped = false;
|
||||
const visiting = new Set<string>();
|
||||
const stepOf = (n: string): number => {
|
||||
if (step.has(n)) return step.get(n)!;
|
||||
if (visiting.has(n)) {
|
||||
looped = true;
|
||||
return 1;
|
||||
}
|
||||
visiting.add(n);
|
||||
const above = (reverse.get(n) ?? []).filter((d) => set.has(d));
|
||||
const s = above.length === 0 ? 1 : 1 + Math.max(...above.map(stepOf));
|
||||
visiting.delete(n);
|
||||
step.set(n, s);
|
||||
return s;
|
||||
};
|
||||
for (const n of set) stepOf(n);
|
||||
return { order: [...set].map((k) => ({ key: k, step: step.get(k) ?? 1 })).sort((a, b) => a.step - b.step || a.key.localeCompare(b.key)), looped };
|
||||
}
|
||||
|
||||
// ─── History ────────────────────────────────────────────────────────────────
|
||||
|
||||
export interface UpsPoint {
|
||||
/** ISO. */
|
||||
t: string;
|
||||
charge: number | null;
|
||||
runtimeSeconds: number | null;
|
||||
loadPercent: number | null;
|
||||
inputVoltage: number | null;
|
||||
/** It was on battery at some time in this stretch. */
|
||||
onBattery: boolean;
|
||||
}
|
||||
|
||||
type ReadingRow = { at: string; charge: number | null; runtimeSeconds: number | null; loadPercent: number | null; inputVoltage: number | null; onBattery: boolean };
|
||||
|
||||
/** Pure: readings into at most `maxPoints` equal slices of [from, to] (each the average of what it holds), and the stretches spent on battery. */
|
||||
export function bucketUps(rows: ReadingRow[], from: number, to: number, maxPoints: number): { points: UpsPoint[]; onBatteryPeriods: { from: string; to: string; ongoing: boolean }[] } {
|
||||
const size = Math.max((to - from) / maxPoints, 1);
|
||||
const buckets = new Map<number, { t: number; n: number; sums: Record<"charge" | "runtimeSeconds" | "loadPercent" | "inputVoltage", { sum: number; n: number }>; onBattery: boolean }>();
|
||||
for (const r of rows) {
|
||||
const t = Date.parse(r.at);
|
||||
if (t < from || t > to) continue;
|
||||
const key = Math.min(maxPoints - 1, Math.floor((t - from) / size));
|
||||
const b = buckets.get(key) ?? { t: 0, n: 0, sums: { charge: { sum: 0, n: 0 }, runtimeSeconds: { sum: 0, n: 0 }, loadPercent: { sum: 0, n: 0 }, inputVoltage: { sum: 0, n: 0 } }, onBattery: false };
|
||||
b.t += t;
|
||||
b.n += 1;
|
||||
b.onBattery ||= r.onBattery;
|
||||
for (const k of ["charge", "runtimeSeconds", "loadPercent", "inputVoltage"] as const) {
|
||||
const v = r[k];
|
||||
if (v !== null) {
|
||||
b.sums[k].sum += v;
|
||||
b.sums[k].n += 1;
|
||||
}
|
||||
}
|
||||
buckets.set(key, b);
|
||||
}
|
||||
const avg = (x: { sum: number; n: number }) => (x.n === 0 ? null : Math.round((x.sum / x.n) * 10) / 10);
|
||||
const points = [...buckets.entries()].sort((a, b) => a[0] - b[0]).map(([, b]) => ({ t: new Date(b.t / b.n).toISOString(), charge: avg(b.sums.charge), runtimeSeconds: avg(b.sums.runtimeSeconds), loadPercent: avg(b.sums.loadPercent), inputVoltage: avg(b.sums.inputVoltage), onBattery: b.onBattery }));
|
||||
// A stretch on battery runs from the first reading that says so to the first that says it is back.
|
||||
const periods: { from: string; to: string; ongoing: boolean }[] = [];
|
||||
let start: string | null = null;
|
||||
let last: string | null = null;
|
||||
for (const r of [...rows].sort((a, b) => a.at.localeCompare(b.at))) {
|
||||
if (r.onBattery) {
|
||||
start ??= r.at;
|
||||
last = r.at;
|
||||
} else if (start !== null) {
|
||||
periods.push({ from: start, to: r.at, ongoing: false });
|
||||
start = null;
|
||||
}
|
||||
}
|
||||
// One still going has no end yet: it runs to the last reading that says it is on battery, and says so.
|
||||
if (start !== null && last !== null) periods.push({ from: start, to: last, ongoing: true });
|
||||
return { points, onBatteryPeriods: periods };
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initUpsSync(): void {
|
||||
if (timer) return;
|
||||
const pass = () => void syncAllUps().catch((err) => console.error("[ups] pass failed:", err));
|
||||
timer = setInterval(pass, SYNC_INTERVAL_MS);
|
||||
setTimeout(pass, 9_000);
|
||||
console.log(`[ups] reading UPSs every ${SYNC_INTERVAL_MS / 1000} seconds`);
|
||||
}
|
||||
Reference in new issue
Block a user