Jellyfin: who is watching, and don't update a machine mid-film

New Jellyfin integration (read-only, every 2 minutes): version, restart wanted,
streams and transcoding, failed scheduled tasks, item counts, failed sign-ins.
Monitoring -> Jellyfin page with a peak-streams history, three alerts, /metrics.

When Jellyfin runs on a managed server, an update run asks first if people are
streaming (409 "watching" unless startAnyway), and a patch plan waits up to two
hours for them before updating that server.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-24 22:01:18 +02:00
1 parent 2f082ca5c2
commit 0146eb8bca
38 files changed
+8136 -13

No files matched your search

+27
View File
@@ -0,0 +1,27 @@
CREATE TABLE `jellyfin_readings` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`integration_id` integer NOT NULL,
`at` text NOT NULL,
`streams` integer NOT NULL,
`transcodes` integer NOT NULL,
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE INDEX `jellyfin_readings_idx` ON `jellyfin_readings` (`integration_id`,`at`);--> statement-breakpoint
CREATE TABLE `jellyfin_state` (
`integration_id` integer PRIMARY KEY NOT NULL,
`server_name` text,
`version` text,
`os` text,
`pending_restart` integer DEFAULT false NOT NULL,
`update_available` integer DEFAULT false NOT NULL,
`streams` text DEFAULT '[]' NOT NULL,
`tasks` text DEFAULT '[]' NOT NULL,
`movies` integer,
`series` integer,
`episodes` integer,
`songs` integer,
`failed_logins_24h` integer,
`updated_at` text NOT NULL,
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
);
File diff suppressed because it is too large. Load diff
+7
View File
@@ -400,6 +400,13 @@
"when": 1792869961357,
"tag": "0056_dizzy_carnage",
"breakpoints": true
},
{
"idx": 57,
"version": "6",
"when": 1792870755473,
"tag": "0057_lame_cloak",
"breakpoints": true
}
]
}
+2
View File
@@ -58,6 +58,7 @@ import { homeAssistantRouter } from "./routes/homeAssistant.js";
import { upsRouter } from "./routes/ups.js";
import { awayRouter } from "./routes/away.js";
import { duplicatiReportRouter, duplicatiRouter } from "./routes/duplicati.js";
import { jellyfinRouter } from "./routes/jellyfin.js";
import { serviceLevelsRouter } from "./routes/serviceLevels.js";
import { publicApiRouter } from "./routes/publicApi.js";
import { generatorRouter } from "./routes/generator.js";
@@ -193,6 +194,7 @@ export function createApp() {
app.use("/api/ups", upsRouter);
app.use("/api/away", awayRouter);
app.use("/api/duplicati", duplicatiRouter);
app.use("/api/jellyfin", jellyfinRouter);
app.use("/api/service-levels", serviceLevelsRouter);
app.use("/api/todos", todosRouter);
app.use("/api/notes", notesRouter);
+39
View File
@@ -553,6 +553,7 @@ export const integrationTypes = [
"npm",
"homeassistant",
"ups",
"jellyfin",
] as const;
export type IntegrationType = (typeof integrationTypes)[number];
@@ -1159,6 +1160,44 @@ export const haReadings = sqliteTable(
(t) => [index("ha_readings_lookup_idx").on(t.integrationId, t.entityId, t.at)],
);
// ─── Jellyfin ───────────────────────────────────────────────────────────────
// How a Jellyfin server (an integration of type "jellyfin") is right now, one row each, replaced at every read: its version, whether it wants a restart, who is streaming, how its
// scheduled tasks last ended, and how many sign-ins failed lately. Read by services/jellyfinSync.ts.
export const jellyfinState = sqliteTable("jellyfin_state", {
integrationId: integer("integration_id")
.primaryKey()
.references(() => integrations.id, { onDelete: "cascade" }),
serverName: text("server_name"),
version: text("version"),
os: text("os"),
pendingRestart: integer("pending_restart", { mode: "boolean" }).notNull().default(false),
updateAvailable: integer("update_available", { mode: "boolean" }).notNull().default(false),
streams: text("streams").notNull().default("[]"), // JSON: who is streaming what
tasks: text("tasks").notNull().default("[]"), // JSON: the scheduled tasks and how they last ended
movies: integer("movies"),
series: integer("series"),
episodes: integer("episodes"),
songs: integer("songs"),
failedLogins24h: integer("failed_logins_24h"),
updatedAt: text("updated_at").notNull(), // ISO
});
// How many were streaming (and how many of those were being transcoded) at a moment, for the chart: one every five minutes, and at once when it changes. Kept 35 days.
export const jellyfinReadings = sqliteTable(
"jellyfin_readings",
{
id: integer("id").primaryKey({ autoIncrement: true }),
integrationId: integer("integration_id")
.notNull()
.references(() => integrations.id, { onDelete: "cascade" }),
at: text("at").notNull(), // ISO
streams: integer("streams").notNull(),
transcodes: integer("transcodes").notNull(),
},
(t) => [index("jellyfin_readings_idx").on(t.integrationId, t.at)],
);
// ─── Duplicati ──────────────────────────────────────────────────────────────
// A Duplicati that reports to this app: one row per machine running it, with the secret token its reports carry in their address (only the hash is kept). Duplicati pushes a
+2
View File
@@ -29,6 +29,7 @@ import { initUnifiSync } from "./services/unifiSync.js";
import { initHomeAssistantSync } from "./services/homeAssistantSync.js";
import { initUpsSync } from "./services/upsSync.js";
import { initAwayMonitor } from "./services/awayMonitor.js";
import { initJellyfinSync } from "./services/jellyfinSync.js";
import { backfillMetricRollups } from "./services/serverMetrics.js";
import { backfillEndpointHourly } from "./services/endpointChecks.js";
import { initSelfBackupScheduler } from "./services/selfBackupScheduler.js";
@@ -79,6 +80,7 @@ initUnifiSync();
initHomeAssistantSync();
initUpsSync();
initAwayMonitor();
initJellyfinSync();
// The hourly and daily figures are built from what is already kept, once, the first time after they were added (in the background: nothing waits for it).
void Promise.all([backfillMetricRollups(), backfillEndpointHourly()]).catch((err) => console.error("[history] couldn't build the hourly and daily figures:", err));
+5
View File
@@ -99,6 +99,11 @@ export const INTEGRATION_FIELDS: Partial<Record<IntegrationType, IntegrationFiel
{ key: "token", label: "Long-lived access token", secret: true, type: "password", placeholder: "made on your profile page in Home Assistant" },
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
],
jellyfin: [
{ key: "url", label: "Jellyfin URL", secret: false, placeholder: "http://jellyfin.example.lan:8096" },
{ key: "apiKey", label: "API key", secret: true, type: "password", placeholder: "made under Dashboard → API Keys in Jellyfin" },
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
],
ups: [
{ key: "host", label: "Host name or address", secret: false, placeholder: "the machine running NUT (upsd), or the UPS's network card" },
{ key: "snmp", label: "Read it over SNMP (version 2c) instead of NUT", secret: false, type: "checkbox" },
+169
View File
@@ -0,0 +1,169 @@
/**
* Jellyfin adapter. Config: url, apiKey (made in Jellyfin under Dashboard → API Keys); optional: insecure.
*
* Reading only: the server's own information (version, whether it wants a restart or an update), who is streaming right now and whether it is being transcoded, the scheduled tasks
* (a library scan that failed), how many items there are and how many sign-ins failed lately. Nothing is sent to Jellyfin but requests for these.
*
* Endpoints and field names are from Jellyfin's published API (`/System/Info`, `/Sessions`, `/ScheduledTasks`, `/Items/Counts`, `/System/ActivityLog/Entries`) and have not
* been checked against a live server here; every field is read defensively, and one that is missing is just left out.
*/
import { requestJson } from "../httpJson.js";
import { withDiagLogging } from "../../services/diagLog.js";
export interface JellyfinConfig {
url: string;
apiKey: string;
insecure?: boolean;
}
export interface JellyfinStream {
user: string;
client: string | null;
device: string | null;
/** What is being watched or listened to, with the series it is of. */
title: string;
/** Converted on the fly, which is what loads the server. */
transcoding: boolean;
paused: boolean;
}
export interface JellyfinTask {
name: string;
category: string | null;
state: string;
/** Completed, Failed, Cancelled or Aborted; null when it has never run. */
lastStatus: string | null;
lastEndedAt: string | null;
error: string | null;
}
export interface JellyfinRead {
name: string | null;
version: string | null;
os: string | null;
pendingRestart: boolean;
updateAvailable: boolean;
streams: JellyfinStream[];
tasks: JellyfinTask[];
counts: { movies: number | null; series: number | null; episodes: number | null; songs: number | null };
/** Failed sign-ins in the last day; null when the activity log could not be read. */
failedLogins24h: number | null;
}
export interface JellyfinAdapter {
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
read(now?: Date): Promise<JellyfinRead>;
}
const obj = (v: unknown): Record<string, unknown> => (v !== null && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {});
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() !== "" ? v.trim() : null);
const num = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null);
const iso = (v: unknown): string | null => {
const s = str(v);
return s && Number.isFinite(Date.parse(s)) ? new Date(s).toISOString() : null;
};
/** Pure: what /System/Info says. */
export function parseSystemInfo(raw: unknown): Pick<JellyfinRead, "name" | "version" | "os" | "pendingRestart" | "updateAvailable"> {
const o = obj(raw);
return { name: str(o.ServerName), version: str(o.Version), os: str(o.OperatingSystemDisplayName) ?? str(o.OperatingSystem), pendingRestart: o.HasPendingRestart === true, updateAvailable: o.HasUpdateAvailable === true };
}
/** Pure: the sessions that are playing something. A session with nothing playing is just somebody signed in. */
export function parseSessions(raw: unknown): JellyfinStream[] {
const out: JellyfinStream[] = [];
for (const item of Array.isArray(raw) ? raw : []) {
const s = obj(item);
const playing = obj(s.NowPlayingItem);
const name = str(playing.Name);
if (!name) continue;
const series = str(playing.SeriesName);
const state = obj(s.PlayState);
out.push({
user: str(s.UserName) ?? "someone",
client: str(s.Client),
device: str(s.DeviceName),
title: series ? `${series} — ${name}` : name,
transcoding: str(state.PlayMethod)?.toLowerCase() === "transcode" || (s.TranscodingInfo !== undefined && s.TranscodingInfo !== null && Object.keys(obj(s.TranscodingInfo)).length > 0),
paused: state.IsPaused === true,
});
}
return out;
}
/** Pure: the scheduled tasks with how they last ended. */
export function parseTasks(raw: unknown): JellyfinTask[] {
const out: JellyfinTask[] = [];
for (const item of Array.isArray(raw) ? raw : []) {
const t = obj(item);
const name = str(t.Name);
if (!name || t.IsHidden === true) continue;
const last = obj(t.LastExecutionResult);
out.push({ name, category: str(t.Category), state: str(t.State) ?? "Idle", lastStatus: str(last.Status), lastEndedAt: iso(last.EndTimeUtc), error: str(last.ErrorMessage) });
}
return out;
}
/** Pure: how many of the activity log's entries from the last day are failed sign-ins. */
export function countFailedLogins(raw: unknown, now: Date): number | null {
const items = obj(raw).Items;
if (!Array.isArray(items)) return null;
const since = now.getTime() - 86_400_000;
return items.filter((i) => {
const e = obj(i);
const at = Date.parse(String(e.Date ?? ""));
return str(e.Type) === "AuthenticationFailed" && (!Number.isFinite(at) || at >= since);
}).length;
}
export function createJellyfinAdapter(config: JellyfinConfig): JellyfinAdapter {
const base = String(config.url ?? "").replace(/\/+$/, "");
const insecure = config.insecure === true;
const headers = { "X-Emby-Token": config.apiKey, Authorization: `MediaBrowser Token="${config.apiKey}"` };
async function get(path: string): Promise<unknown> {
const res = await requestJson(`${base}${path}`, { headers, insecure });
if (res.status === 401 || res.status === 403) throw new Error(`Jellyfin did not accept the API key (HTTP ${res.status})`);
if (res.status === 404) throw new Error("Jellyfin has no API at this address — check the address and port (8096)");
if (res.status < 200 || res.status >= 300) throw new Error(`Jellyfin error: HTTP ${res.status}`);
if (res.json === null) throw new Error("Jellyfin answered with something that isn't JSON — is the address its own?");
return res.json;
}
async function read(now: Date = new Date()): Promise<JellyfinRead> {
const info = parseSystemInfo(await get("/System/Info"));
// The rest is worth having but not worth failing for: a Jellyfin that answers about itself and not about its sessions still says it is up.
const soft = async <T>(path: string, parse: (raw: unknown) => T, fallback: T): Promise<T> => {
try {
return parse(await get(path));
} catch {
return fallback;
}
};
const [streams, tasks, countsRaw, failed] = await Promise.all([
soft("/Sessions?ActiveWithinSeconds=300", parseSessions, [] as JellyfinStream[]),
soft("/ScheduledTasks?isHidden=false", parseTasks, [] as JellyfinTask[]),
soft("/Items/Counts", obj, {} as Record<string, unknown>),
soft(`/System/ActivityLog/Entries?startIndex=0&limit=100&minDate=${encodeURIComponent(new Date(now.getTime() - 86_400_000).toISOString())}`, (r) => countFailedLogins(r, now), null as number | null),
]);
return {
...info,
streams,
tasks,
counts: { movies: num(countsRaw.MovieCount), series: num(countsRaw.SeriesCount), episodes: num(countsRaw.EpisodeCount), songs: num(countsRaw.SongCount) },
failedLogins24h: failed,
};
}
async function ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }> {
const start = Date.now();
try {
await get("/System/Info");
return { ok: true, latencyMs: Date.now() - start };
} catch (err) {
return { ok: false, error: err instanceof Error ? err.message : String(err) };
}
}
return withDiagLogging("jellyfin", { ping, read });
}
+3
View File
@@ -14,6 +14,7 @@ import { createUnifiAdapter } from "./unifi/adapter.js";
import { createNpmAdapter } from "./npm/adapter.js";
import { createHomeAssistantAdapter } from "./homeassistant/adapter.js";
import { createUpsAdapter } from "./ups/adapter.js";
import { createJellyfinAdapter } from "./jellyfin/adapter.js";
export interface PingableAdapter {
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
@@ -56,6 +57,8 @@ export function createIntegrationAdapter(type: IntegrationType, config: Integrat
return createHomeAssistantAdapter(config as any);
case "ups":
return createUpsAdapter(config as any);
case "jellyfin":
return createJellyfinAdapter(config as any);
default:
throw new Error(`Integration type "${type}" is not implemented yet`);
}
+4
View File
@@ -29,6 +29,7 @@ import { getSourceStatus } from "../services/integrationHealthMonitor.js";
import { removeIntegrationData, syncUnifiIntegration } from "../services/unifiSync.js";
import { removeHomeAssistantData, syncHomeAssistant } from "../services/homeAssistantSync.js";
import { forgetUps, syncUps } from "../services/upsSync.js";
import { forgetJellyfin, syncJellyfin } from "../services/jellyfinSync.js";
import { invalidateAlertsCache } from "../services/alerts.js";
import { getSettings, updateSettings } from "../services/settingsStore.js";
import { asyncHandler } from "../utils/asyncHandler.js";
@@ -131,6 +132,7 @@ integrationsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res)
if (type === "unifi") void syncUnifiIntegration(created.id).then(() => invalidateAlertsCache());
if (type === "homeassistant") void syncHomeAssistant(created.id).then(() => invalidateAlertsCache());
if (type === "ups") void syncUps(created.id).then(() => invalidateAlertsCache());
if (type === "jellyfin") void syncJellyfin(created.id).then(() => invalidateAlertsCache());
res.status(201).json({
integration: {
@@ -234,6 +236,7 @@ integrationsRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req,
if (updated.type === "unifi" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncUnifiIntegration(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "homeassistant" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncHomeAssistant(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "ups" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncUps(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "jellyfin" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncJellyfin(updated.id).then(() => invalidateAlertsCache());
res.json({
integration: {
@@ -313,6 +316,7 @@ integrationsRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req,
if (existing.type === "unifi") await removeIntegrationData(id);
if (existing.type === "homeassistant") await removeHomeAssistantData(id);
if (existing.type === "ups") await forgetUps(id);
if (existing.type === "jellyfin") await forgetJellyfin(id);
await db.delete(integrations).where(eq(integrations.id, id));
if (existing.credentialId) {
await db.delete(integrationCredentials).where(eq(integrationCredentials.id, existing.credentialId));
+77
View File
@@ -0,0 +1,77 @@
import { Router } from "express";
import { and, asc, eq, gte } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, jellyfinReadings, jellyfinState, servers } from "../db/schema.js";
import { requireAuth } from "../auth/middleware.js";
import { hostOfAddress } from "../services/kumaTargets.js";
import { bucketPeaks, isJellyfinFresh, jellyfinProblems, jellyfinSyncStatuses, READING_KEEP_DAYS, streamsOf, tasksOf, watchingOnServer, watchingWords } from "../services/jellyfinSync.js";
import { matchServerForTarget } from "../services/uptimeKumaMatch.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const jellyfinRouter = Router();
jellyfinRouter.use(requireAuth);
const DAY_MS = 86_400_000;
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
/** Every Jellyfin, how it is and who is streaming from it, from what the last read stored. Everyone signed in can see it, like the other pages made of stored readings. */
jellyfinRouter.get("/", asyncHandler(async (_req, res) => {
const [rows, states, statuses, serverRows] = await Promise.all([
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled, baseUrl: integrations.baseUrl }).from(integrations).where(eq(integrations.type, "jellyfin")).orderBy(integrations.name),
db.select().from(jellyfinState),
jellyfinSyncStatuses(),
db.select({ id: servers.id, name: servers.name, hostname: servers.hostname, ipAddresses: servers.ipAddresses }).from(servers),
]);
const matchable = serverRows.map((s) => {
let ips: string[] = [];
try {
const v = JSON.parse(s.ipAddresses ?? "[]");
ips = Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : [];
} catch {
// none
}
return { id: s.id, name: s.name, hostname: s.hostname, ips };
});
const byId = new Map(states.map((s) => [s.integrationId, s]));
const now = Date.now();
res.json({
jellyfins: rows.map((r) => {
const state = byId.get(r.id) ?? null;
const status = statuses[r.id] ?? null;
const fresh = isJellyfinFresh(status ?? undefined, now);
const machine = matchServerForTarget(hostOfAddress(r.baseUrl), matchable);
return {
id: r.id,
name: r.name,
enabled: r.enabled,
runsOn: machine ? { id: machine.id, name: machine.name } : null,
status,
fresh,
state: state ? { serverName: state.serverName, version: state.version, os: state.os, pendingRestart: state.pendingRestart, updateAvailable: state.updateAvailable, streams: streamsOf(state), tasks: tasksOf(state), counts: { movies: state.movies, series: state.series, episodes: state.episodes, songs: state.songs }, failedLogins24h: state.failedLogins24h, updatedAt: state.updatedAt } : null,
problems: state && fresh ? jellyfinProblems(r.id, r.name, state) : [],
};
}),
});
}));
/** Who is streaming from the Jellyfin that runs on a server, for the warning before it is restarted. */
jellyfinRouter.get("/watching", asyncHandler(async (req, res) => {
const serverId = idOf(String(req.query.serverId ?? ""));
if (!Number.isInteger(serverId)) return res.status(400).json({ error: "invalid_id" });
const watching = await watchingOnServer(serverId);
res.json({ watching, words: watchingWords(watching) });
}));
/** How many were streaming over time, as peaks (readings are kept 35 days). */
jellyfinRouter.get("/:id/history", asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [row] = await db.select({ id: integrations.id, name: integrations.name, type: integrations.type }).from(integrations).where(eq(integrations.id, id)).limit(1);
if (!row || row.type !== "jellyfin") return res.status(404).json({ error: "not_found" });
const asked = Number(req.query.days);
const days = Number.isFinite(asked) ? Math.min(READING_KEEP_DAYS, Math.max(1, Math.round(asked))) : 7;
const to = Date.now();
const from = to - days * DAY_MS;
const rows = await db.select().from(jellyfinReadings).where(and(eq(jellyfinReadings.integrationId, id), gte(jellyfinReadings.at, new Date(from).toISOString()))).orderBy(asc(jellyfinReadings.at));
res.json({ jellyfin: { id: row.id, name: row.name }, days, points: bucketPeaks(rows, from, to, 200) });
}));
+2
View File
@@ -18,6 +18,8 @@ const startSchema = z.object({
templateId: z.number().int().positive(),
limit: z.string().trim().regex(/^[A-Za-z0-9][A-Za-z0-9._,:*\-]{0,199}$/, "Use the host name as it is in the Semaphore inventory, like web01"),
windowMinutes: z.number().int().min(15).max(480).default(90),
// Somebody is streaming from the machine and the person has seen the warning.
startAnyway: z.boolean().optional(),
});
const shape = (r: Awaited<ReturnType<typeof listUpdateRuns>>[number]) => ({
+5 -2
View File
@@ -75,6 +75,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
ups: { label: "UPS", link: "/ups" },
slo: { label: "Service level", link: "/service-levels" },
duplicati: { label: "Duplicati", link: "/duplicati" },
jellyfin: { label: "Jellyfin", link: "/jellyfin" },
secrets: { label: "Secrets", link: "/secrets" },
domains: { label: "Domains", link: "/dns?view=domains" },
endpoint: { label: "Endpoint", link: "/endpoints" },
@@ -113,6 +114,8 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
m = /^duplicati-(?:failed|stale):(\d+)/.exec(key);
if (m) return { type: "duplicati", id: Number(m[1]), category: "backup" };
m = /^jellyfin-(?:restart|task|logins):(\d+)/.exec(key);
if (m) return { type: "jellyfin", id: Number(m[1]), category: key.startsWith("jellyfin-task") ? "automation" : "monitoring" };
return { type: "server", id: null, category: "disk" };
}
@@ -235,7 +238,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
type: origin.type,
message: c.message,
key: c.key,
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : undefined,
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : undefined,
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
});
}
@@ -245,7 +248,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health");
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
for (const h of held) {
const m = /^(proxmox|synology|unifi|ups|dockhand):(\d+)(?::(.+))?$/.exec(h);
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin):(\d+)(?::(.+))?$/.exec(h);
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
const row = intById.get(Number(m[2]));
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
+25 -1
View File
@@ -1,6 +1,6 @@
import { and, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, networkDevices, serverPorts, servers, upsState } from "../db/schema.js";
import { integrations, jellyfinState, networkDevices, serverPorts, servers, upsState } from "../db/schema.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
@@ -20,6 +20,7 @@ import { baselineKey, computeBaselineFindings, type BaselineFinding } from "./ba
import { levelFindings, levelKey, type LevelFinding } from "./serviceLevels.js";
import { createDockhandAdapter } from "../integrations/dockhand/adapter.js";
import { currentProblems as duplicatiProblemsNow, type DuplicatiProblem } from "./duplicati.js";
import { isJellyfinFresh, jellyfinProblems, jellyfinSyncStatuses, type JellyfinProblem } from "./jellyfinSync.js";
import { evaluateContainers, loadContainerMemory, saveContainerMemory, type ContainerMemory, type DockerSnapshot } from "./containerHealth.js";
import { recordStorageSamples } from "./storageHistory.js";
import { isUpsFresh, upsProblems, upsSyncStatuses, type UpsStateRow } from "./upsSync.js";
@@ -87,6 +88,8 @@ export interface HealthSnapshot {
baselines?: BaselineFinding[];
/** Service-level targets whose error budget for the month is mostly or wholly used (services/serviceLevels.ts). */
levels?: LevelFinding[];
/** What is wrong with each Jellyfin, from what was stored, only while that reading is recent (services/jellyfinSync.ts). */
jellyfin?: { integrationId: number; problems: JellyfinProblem[] }[];
/** Duplicati backups that failed or have not succeeded in time (services/duplicati.ts). */
duplicati?: DuplicatiProblem[];
/** Each Docker integration's containers as last read, with what was remembered of them (services/containerHealth.ts). */
@@ -198,6 +201,9 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
}
}
// A Jellyfin that wants a restart, a scheduled task that failed, a burst of failed sign-ins.
for (const j of snapshot.jellyfin ?? []) for (const p of j.problems) out.push({ key: p.key, source: `jellyfin:${j.integrationId}`, message: p.message, severity: p.severity });
// Duplicati backups that failed, or have not succeeded for longer than they should.
for (const p of snapshot.duplicati ?? []) out.push({ key: p.key, source: "duplicati", message: p.message, severity: p.severity });
@@ -428,6 +434,24 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
}
}
// Jellyfin: read by its own pass every couple of minutes; what is judged is what that pass stored, and only while it is recent.
try {
const rowsJ = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "jellyfin"), eq(integrations.enabled, true)));
if (rowsJ.length > 0) {
const statuses = await jellyfinSyncStatuses();
for (const r of rowsJ) {
const [state] = await db.select().from(jellyfinState).where(eq(jellyfinState.integrationId, r.id)).limit(1);
if (!state || !isJellyfinFresh(statuses[r.id], Date.now())) {
held.add(`jellyfin:${r.id}`);
continue;
}
(snapshot.jellyfin ??= []).push({ integrationId: r.id, problems: jellyfinProblems(r.id, r.name, state) });
}
}
} catch (err) {
console.error("[health] couldn't judge Jellyfin:", err instanceof Error ? err.message : err);
}
// Duplicati: what its reports say, read from what is stored (nothing is asked of Duplicati). A failure here costs only this check.
try {
snapshot.duplicati = await duplicatiProblemsNow(Date.now());
+229
View File
@@ -0,0 +1,229 @@
/**
* Keeps what Jellyfin says up to date: every two minutes each Jellyfin integration is read (its version, who is streaming, how its scheduled tasks ended, how many sign-ins
* failed), the state is stored, and the number streaming is added to a history. Pages, charts and alerts are built from what is stored, so none of them talks to Jellyfin.
*
* Two things come of it besides a page. **Alerts**: a server that wants a restart, a scheduled task (a library scan) that failed, a burst of failed sign-ins. And **who is
* watching**: before an update run or a patch plan restarts the machine Jellyfin runs on, the app can say that people are streaming from it — and a patch plan waits for them to
* finish, for a while, instead of cutting a film off.
*/
import { and, desc, eq, lt } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, jellyfinReadings, jellyfinState, servers } from "../db/schema.js";
import { createJellyfinAdapter, type JellyfinRead, type JellyfinStream, type JellyfinTask } from "../integrations/jellyfin/adapter.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { hostOfAddress } from "./kumaTargets.js";
import { matchServerForTarget } from "./uptimeKumaMatch.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
export const SYNC_INTERVAL_MS = 2 * 60_000;
/** After this long without a good read, what is stored is no longer believed. */
export const STALE_AFTER_MS = 10 * 60_000;
export const READING_KEEP_DAYS = 35;
const READING_GAP_MS = 5 * 60_000 - 20_000;
/** This many failed sign-ins in a day is a warning. */
export const FAILED_LOGINS_WARN = 10;
const STATUS_FLAG = "jellyfinSync";
export interface JellyfinSyncStatus {
/** The last read that worked (kept when a later one failed). */
at: string;
ok: boolean;
error: string | null;
}
async function readStatuses(): Promise<Record<string, JellyfinSyncStatus>> {
try {
const raw = await getInternalFlag(STATUS_FLAG);
return raw ? (JSON.parse(raw) as Record<string, JellyfinSyncStatus>) : {};
} catch {
return {};
}
}
export async function jellyfinSyncStatuses(): Promise<Record<number, JellyfinSyncStatus>> {
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
}
export const isJellyfinFresh = (status: JellyfinSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
export type StateRow = typeof jellyfinState.$inferSelect;
const parse = <T>(text: string): T[] => {
try {
const v = JSON.parse(text);
return Array.isArray(v) ? (v as T[]) : [];
} catch {
return [];
}
};
export const streamsOf = (row: Pick<StateRow, "streams">) => parse<JellyfinStream>(row.streams);
export const tasksOf = (row: Pick<StateRow, "tasks">) => parse<JellyfinTask>(row.tasks);
/** Writes down one read: the state, and a point in the history when it is time (or when the number streaming has changed). */
export async function storeRead(integrationId: number, r: JellyfinRead, now: Date = new Date()): Promise<void> {
const nowIso = now.toISOString();
const values = {
serverName: r.name,
version: r.version,
os: r.os,
pendingRestart: r.pendingRestart,
updateAvailable: r.updateAvailable,
streams: JSON.stringify(r.streams),
tasks: JSON.stringify(r.tasks),
movies: r.counts.movies,
series: r.counts.series,
episodes: r.counts.episodes,
songs: r.counts.songs,
failedLogins24h: r.failedLogins24h,
updatedAt: nowIso,
};
await db.insert(jellyfinState).values({ integrationId, ...values }).onConflictDoUpdate({ target: jellyfinState.integrationId, set: values });
const transcodes = r.streams.filter((s) => s.transcoding).length;
const [newest] = await db.select({ at: jellyfinReadings.at, streams: jellyfinReadings.streams, transcodes: jellyfinReadings.transcodes }).from(jellyfinReadings).where(eq(jellyfinReadings.integrationId, integrationId)).orderBy(desc(jellyfinReadings.at)).limit(1);
const due = !newest || newest.streams !== r.streams.length || newest.transcodes !== transcodes || now.getTime() - Date.parse(newest.at) >= READING_GAP_MS;
if (due) {
await db.insert(jellyfinReadings).values({ integrationId, at: nowIso, streams: r.streams.length, transcodes });
}
await db.delete(jellyfinReadings).where(and(eq(jellyfinReadings.integrationId, integrationId), lt(jellyfinReadings.at, new Date(now.getTime() - READING_KEEP_DAYS * 86_400_000).toISOString())));
}
/** Reads one Jellyfin and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
export async function syncJellyfin(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<JellyfinRead>): Promise<JellyfinSyncStatus> {
let status: JellyfinSyncStatus;
try {
const loaded = await loadIntegrationConfig(integrationId);
if (!loaded || loaded.integration.type !== "jellyfin") throw new Error("not a Jellyfin integration");
const result = await (read ? read(loaded.config) : createJellyfinAdapter(loaded.config as never).read(now));
await storeRead(integrationId, result, now);
status = { at: now.toISOString(), ok: true, error: null };
} catch (err) {
const previous = (await readStatuses())[String(integrationId)];
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
console.error(`[jellyfin] couldn't read integration ${integrationId}:`, status.error);
}
try {
const all = await readStatuses();
all[String(integrationId)] = status;
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
} catch (err) {
console.error("[jellyfin] couldn't record the result:", err);
}
return status;
}
export async function syncAllJellyfin(now: Date = new Date()): Promise<Record<number, JellyfinSyncStatus>> {
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "jellyfin"), eq(integrations.enabled, true)));
const out: Record<number, JellyfinSyncStatus> = {};
for (const r of rows) out[r.id] = await syncJellyfin(r.id, now);
return out;
}
export async function forgetJellyfin(integrationId: number): Promise<void> {
const all = await readStatuses();
if (String(integrationId) in all) {
delete all[String(integrationId)];
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
}
}
// ─── What is wrong ──────────────────────────────────────────────────────────
export interface JellyfinProblem {
key: string;
message: string;
severity: "warning";
}
/** Pure: what is wrong with one Jellyfin, from what is stored for it. */
export function jellyfinProblems(id: number, name: string, s: Pick<StateRow, "pendingRestart" | "tasks" | "failedLogins24h">): JellyfinProblem[] {
const out: JellyfinProblem[] = [];
if (s.pendingRestart) out.push({ key: `jellyfin-restart:${id}`, severity: "warning", message: `${name} is waiting to be restarted — a plugin or an update needs it` });
for (const t of tasksOf(s)) {
if (t.lastStatus === "Failed") out.push({ key: `jellyfin-task:${id}:${t.name}`, severity: "warning", message: `${name}: the scheduled task “${t.name}” failed${t.error ? ` — ${t.error}` : ""}` });
}
if (s.failedLogins24h !== null && s.failedLogins24h >= FAILED_LOGINS_WARN) out.push({ key: `jellyfin-logins:${id}`, severity: "warning", message: `${name}: ${s.failedLogins24h} sign-ins failed in the last day` });
return out;
}
// ─── Who is watching ────────────────────────────────────────────────────────
export interface Watching {
integrationId: number;
integrationName: string;
streams: JellyfinStream[];
}
/** Pure: a sentence about who is watching, for a warning or a waiting message: "2 people are streaming from Jellyfin (Ann: Dune, Bo: The Office — S1E2)". */
export function watchingWords(w: Watching[]): string {
const streams = w.flatMap((x) => x.streams);
if (streams.length === 0) return "";
const names = w.map((x) => x.integrationName);
const who = streams.slice(0, 3).map((s) => `${s.user}: ${s.title}`).join(", ");
const rest = streams.length > 3 ? ` and ${streams.length - 3} more` : "";
const transcoding = streams.filter((s) => s.transcoding).length;
return `${streams.length === 1 ? "1 person is" : `${streams.length} people are`} streaming from ${[...new Set(names)].join(" and ")} (${who}${rest})${transcoding > 0 ? `, ${transcoding} being transcoded` : ""}`;
}
/** The Jellyfins that run on this server — the one whose address is the server's — and who is streaming from each, as last read and believed. Only those with somebody streaming are listed. */
export async function watchingOnServer(serverId: number, now: Date = new Date()): Promise<Watching[]> {
const rows = await db.select({ id: integrations.id, name: integrations.name, baseUrl: integrations.baseUrl }).from(integrations).where(and(eq(integrations.type, "jellyfin"), eq(integrations.enabled, true)));
if (rows.length === 0) return [];
const statuses = await jellyfinSyncStatuses();
const serverRows = await db.select({ id: servers.id, name: servers.name, hostname: servers.hostname, ipAddresses: servers.ipAddresses }).from(servers);
const matchable = serverRows.map((s) => {
let ips: string[] = [];
try {
const v = JSON.parse(s.ipAddresses ?? "[]");
ips = Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : [];
} catch {
// no addresses
}
return { id: s.id, name: s.name, hostname: s.hostname, ips };
});
const out: Watching[] = [];
for (const r of rows) {
if (!isJellyfinFresh(statuses[r.id], now.getTime())) continue;
if (matchServerForTarget(hostOfAddress(r.baseUrl), matchable)?.id !== serverId) continue;
const [state] = await db.select().from(jellyfinState).where(eq(jellyfinState.integrationId, r.id)).limit(1);
const streams = state ? streamsOf(state) : [];
if (streams.length > 0) out.push({ integrationId: r.id, integrationName: r.name, streams });
}
return out;
}
export interface PeakPoint {
/** ISO. */
t: string;
/** The most that were streaming, and the most transcoded, in this stretch. */
streams: number;
transcodes: number;
}
/** Pure: readings into at most `maxPoints` equal slices of [from, to], each as its peak — an average would hide the evening somebody watched. */
export function bucketPeaks(rows: { at: string; streams: number; transcodes: number }[], from: number, to: number, maxPoints: number): PeakPoint[] {
if (rows.length === 0 || to <= from) return [];
const size = Math.max((to - from) / maxPoints, 1);
const buckets = new Map<number, { tSum: number; n: number; streams: number; transcodes: number }>();
for (const r of rows) {
const t = Date.parse(r.at);
if (t < from || t > to) continue;
const key = Math.min(maxPoints - 1, Math.floor((t - from) / size));
const b = buckets.get(key) ?? { tSum: 0, n: 0, streams: 0, transcodes: 0 };
b.tSum += t;
b.n += 1;
b.streams = Math.max(b.streams, r.streams);
b.transcodes = Math.max(b.transcodes, r.transcodes);
buckets.set(key, b);
}
return [...buckets.entries()].sort((a, b) => a[0] - b[0]).map(([, b]) => ({ t: new Date(b.tSum / b.n).toISOString(), streams: b.streams, transcodes: b.transcodes }));
}
let timer: ReturnType<typeof setInterval> | null = null;
export function initJellyfinSync(): void {
if (timer) return;
const pass = () => void syncAllJellyfin().catch((err) => console.error("[jellyfin] pass failed:", err));
timer = setInterval(pass, SYNC_INTERVAL_MS);
setTimeout(pass, 11_000);
console.log(`[jellyfin] reading Jellyfin every ${SYNC_INTERVAL_MS / 60000} minutes`);
}
+2
View File
@@ -31,6 +31,8 @@ export function subjectOfConditionKey(key: string): string | null {
if (m) return `server:${m[1]}`;
m = /^container-(?:restarting|unhealthy|stopped|created|host):(\d+):/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^jellyfin-(?:restart|task|logins):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^duplicati-(?:failed|stale):\d+:server:(\d+)$/.exec(key);
if (m) return `server:${m[1]}`;
m = /^(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:(\d+)$/.exec(key);
+18 -1
View File
@@ -5,10 +5,11 @@
*/
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState } from "../db/schema.js";
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState } from "../db/schema.js";
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
import { levelStatuses } from "./serviceLevels.js";
import { backupState, listAll as listDuplicati } from "./duplicati.js";
import { isJellyfinFresh, jellyfinSyncStatuses, streamsOf } from "./jellyfinSync.js";
import { latestTests } from "./restoreTests.js";
import { heartbeatState, parseStamp } from "./heartbeats.js";
import { env } from "../env.js";
@@ -60,6 +61,21 @@ class Exposition {
}
}
/** Each Jellyfin, for /metrics: how many are streaming, how many of those are transcoded, and whether it wants a restart. Only while the reading is recent. */
async function addJellyfinMetrics(out: Exposition, now: Date) {
const statuses = await jellyfinSyncStatuses();
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "jellyfin"))).map((r) => [r.id, r]));
for (const s of await db.select().from(jellyfinState)) {
const row = names.get(s.integrationId);
if (!row || !row.enabled || !isJellyfinFresh(statuses[s.integrationId], now.getTime())) continue;
const labels = { name: row.name };
const streams = streamsOf(s);
out.add("homelab_jellyfin_streams", "gauge", "How many are streaming from Jellyfin right now.", streams.length, labels);
out.add("homelab_jellyfin_transcodes", "gauge", "How many of those streams Jellyfin is transcoding.", streams.filter((x) => x.transcoding).length, labels);
out.add("homelab_jellyfin_pending_restart", "gauge", "1 if Jellyfin is waiting to be restarted, else 0.", s.pendingRestart ? 1 : 0, labels);
}
}
/** Each Duplicati backup, for /metrics: whether it is in order, how long since it last succeeded, and how big what it backs up is. */
async function addDuplicatiMetrics(out: Exposition, now: Date) {
const { sources, backups } = await listDuplicati();
@@ -267,6 +283,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
await addUpsMetrics(out, now);
await addServiceLevelMetrics(out, now);
await addDuplicatiMetrics(out, now);
await addJellyfinMetrics(out, now);
await addHeartbeatMetrics(out, now);
return out.toString();
}
+21 -1
View File
@@ -17,6 +17,7 @@ import { invalidateAlertsCache } from "./alerts.js";
import { recordAudit } from "./audit.js";
import { impactOf } from "./dependencies.js";
import { startOrExtendWindow } from "./maintenance.js";
import { watchingOnServer, watchingWords } from "./jellyfinSync.js";
import { occurrencesBetween, type Occurrence } from "./maintenanceSchedules.js";
import { notifyPatchPlanFinished } from "./notify.js";
import { parseTags } from "./serverTags.js";
@@ -36,8 +37,13 @@ export interface PlanItem {
message: string | null;
/** Windows made for what depends on this server, which end with it. */
windowIds: number[];
/** When its turn came and somebody was streaming from it, so it was left a while (ISO). */
waitingSince?: string | null;
}
/** How long a server is left, when it is its turn and people are streaming from it, before it is updated anyway. */
export const WATCHING_WAIT_MS = 2 * 3_600_000;
/** How late a plan may still start, if the app was not running at its time. */
export const START_GRACE_MS = 30 * 60_000;
export const POLL_INTERVAL_MS = 20_000;
@@ -217,6 +223,19 @@ async function advanceRun(run: PlanRunRow, now: Date): Promise<void> {
}
}
const actor = `Patch plan "${plan.name}"`;
// People streaming from it (Jellyfin on this machine): its turn waits, up to two hours, rather than cutting a film off.
const watching = await watchingOnServer(server.id, now);
let waited = "";
if (watching.length > 0) {
const since = next.waitingSince ? Date.parse(next.waitingSince) : now.getTime();
if (!next.waitingSince) next.waitingSince = now.toISOString();
if (now.getTime() - since < WATCHING_WAIT_MS) {
next.message = `Waiting: ${watchingWords(watching)}.`;
await saveRun(run, items);
return; // looked at again at the next pass
}
waited = `Went ahead after waiting ${Math.round((now.getTime() - since) / 3_600_000)} h; ${watchingWords(watching)}. `;
}
try {
const ur = await startUpdateRun(
{
@@ -227,10 +246,11 @@ async function advanceRun(run: PlanRunRow, now: Date): Promise<void> {
limit: plan.limitFrom === "hostname" && server.hostname ? server.hostname : server.name,
windowMinutes: plan.windowMinutes,
actor,
startAnyway: true,
},
now,
);
Object.assign(next, { status: "running", updateRunId: ur.id, message: "The update run has started.", windowIds: await silenceDependents(plan, server.id, server.name, actor, now) });
Object.assign(next, { status: "running", updateRunId: ur.id, message: `${waited}The update run has started.`, windowIds: await silenceDependents(plan, server.id, server.name, actor, now) });
await saveRun(run, items);
return; // one at a time: this one is followed until it is over
} catch (err) {
+9
View File
@@ -15,6 +15,7 @@ import { createSemaphoreAdapter, type SemaphoreAdapter, type SemaphoreTask } fro
import { invalidateAlertsCache } from "./alerts.js";
import { recordAudit } from "./audit.js";
import { startOrExtendWindow } from "./maintenance.js";
import { watchingOnServer, watchingWords } from "./jellyfinSync.js";
import { notifyUpdateRunFinished } from "./notify.js";
import { parsePendingUpdates } from "./pendingUpdates.js";
@@ -68,6 +69,8 @@ export interface StartInput {
/** How long the maintenance window lasts at most; it ends sooner when the run does. */
windowMinutes: number;
actor: string;
/** Go ahead although somebody is streaming from the machine (a person who has seen the warning, or a plan that has waited long enough). */
startAnyway?: boolean;
}
export async function startUpdateRun(input: StartInput, now: Date = new Date()): Promise<UpdateRunRow> {
@@ -82,6 +85,12 @@ export async function startUpdateRun(input: StartInput, now: Date = new Date()):
.limit(1);
if (busy) throw new UpdateRunError(409, "already_running", `An update run for ${server.name} is already under way.`);
// Jellyfin on this machine, with people watching: the run may restart it. Said, and the person asked again, rather than cutting a film off.
if (!input.startAnyway) {
const watching = await watchingOnServer(server.id, now);
if (watching.length > 0) throw new UpdateRunError(409, "watching", `${watchingWords(watching)}. Wait until they have finished, or start it anyway.`);
}
const adapter = await semaphoreFor(input.integrationId);
if (!adapter) throw new UpdateRunError(400, "no_semaphore", "That Semaphore integration doesn't exist or isn't switched on.");
+337
View File
@@ -0,0 +1,337 @@
/**
* Jellyfin: reading what its API says, what is stored and when a history point is added, what is wrong with it, who is watching (and the warning before an update run
* restarts the machine), the health pass, the page's routes and /metrics.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { createServer as createHttpServer, type Server } from "node:http";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-jellyfin-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, integrations, jellyfinReadings, jellyfinState } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const adapter = await import("../src/integrations/jellyfin/adapter.js");
const jf = await import("../src/services/jellyfinSync.js");
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { startUpdateRun, UpdateRunError } = await import("../src/services/updateRuns.js");
const { jellyfinRouter } = await import("../src/routes/jellyfin.js");
const { buildMetrics } = await import("../src/services/metrics.js");
const { setInternalFlag } = await import("../src/services/settingsStore.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const MIN = 60_000;
const DAY = 86_400_000;
const SESSIONS = [
{ UserName: "Ann", Client: "Jellyfin Web", DeviceName: "Firefox", NowPlayingItem: { Name: "Dune" }, PlayState: { PlayMethod: "DirectPlay", IsPaused: false } },
{ UserName: "Bo", Client: "Android TV", DeviceName: "Shield", NowPlayingItem: { Name: "Pilot", SeriesName: "The Office" }, PlayState: { PlayMethod: "Transcode", IsPaused: true }, TranscodingInfo: { VideoCodec: "h264" } },
{ UserName: "Cy", Client: "Jellyfin Web", DeviceName: "Chrome" }, // signed in, playing nothing
];
const read = (extra: Partial<import("../src/integrations/jellyfin/adapter.js").JellyfinRead> = {}) => ({
name: "Den", version: "10.10.3", os: "Linux", pendingRestart: false, updateAvailable: false, streams: [] as import("../src/integrations/jellyfin/adapter.js").JellyfinStream[],
tasks: [] as import("../src/integrations/jellyfin/adapter.js").JellyfinTask[], counts: { movies: 120, series: 30, episodes: 900, songs: null }, failedLogins24h: 0, ...extra,
});
const stream = (user: string, extra: Partial<import("../src/integrations/jellyfin/adapter.js").JellyfinStream> = {}) => ({ user, client: "Web", device: "Firefox", title: "Dune", transcoding: false, paused: false, ...extra });
describe("what Jellyfin's API says", () => {
test("the server's own information", () => {
const i = adapter.parseSystemInfo({ ServerName: "Den", Version: "10.10.3", OperatingSystemDisplayName: "Debian", HasPendingRestart: true, HasUpdateAvailable: false });
assert.deepEqual(i, { name: "Den", version: "10.10.3", os: "Debian", pendingRestart: true, updateAvailable: false });
assert.deepEqual(adapter.parseSystemInfo(null), { name: null, version: null, os: null, pendingRestart: false, updateAvailable: false });
});
test("only the sessions playing something are streams, with the series in the title and transcoding told from the play method", () => {
const s = adapter.parseSessions(SESSIONS);
assert.deepEqual(s.map((x) => [x.user, x.title, x.transcoding, x.paused]), [["Ann", "Dune", false, false], ["Bo", "The Office — Pilot", true, true]]);
assert.deepEqual(adapter.parseSessions("nonsense"), []);
});
test("the scheduled tasks, hidden ones left out, with how they last ended", () => {
const t = adapter.parseTasks([
{ Name: "Scan media library", Category: "Library", State: "Idle", LastExecutionResult: { Status: "Failed", EndTimeUtc: "2026-10-20T03:00:00Z", ErrorMessage: "disk full" } },
{ Name: "Hidden", IsHidden: true },
{ Name: "Never run" },
]);
assert.equal(t.length, 2);
assert.deepEqual([t[0].lastStatus, t[0].error, t[0].lastEndedAt], ["Failed", "disk full", "2026-10-20T03:00:00.000Z"]);
assert.equal(t[1].lastStatus, null);
});
test("failed sign-ins in the last day, from the activity log", () => {
const now = new Date("2026-10-20T12:00:00Z");
const raw = { Items: [{ Type: "AuthenticationFailed", Date: "2026-10-20T11:00:00Z" }, { Type: "AuthenticationFailed", Date: "2026-10-18T11:00:00Z" }, { Type: "UserLoggedIn", Date: "2026-10-20T11:30:00Z" }] };
assert.equal(adapter.countFailedLogins(raw, now), 1);
assert.equal(adapter.countFailedLogins({}, now), null);
});
describe("the adapter against a Jellyfin that answers", () => {
let server: Server;
let url = "";
const seen: { path: string; token: string | undefined }[] = [];
before(async () => {
server = createHttpServer((req, res) => {
seen.push({ path: req.url ?? "", token: req.headers["x-emby-token"] as string | undefined });
const send = (code: number, body: unknown) => {
res.writeHead(code, { "content-type": "application/json" });
res.end(JSON.stringify(body));
};
if (req.headers["x-emby-token"] !== "key") return send(401, {});
const path = (req.url ?? "").split("?")[0];
if (path === "/System/Info") return send(200, { ServerName: "Den", Version: "10.10.3", HasPendingRestart: false });
if (path === "/Sessions") return send(200, SESSIONS);
if (path === "/ScheduledTasks") return send(500, {}); // the rest is worth having, not worth failing for
if (path === "/Items/Counts") return send(200, { MovieCount: 120, SeriesCount: 30 });
send(404, {});
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
url = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
});
after(() => server.close());
test("it reads, sends the key, and a part that cannot be read does not fail the whole", async () => {
const r = await adapter.createJellyfinAdapter({ url: `${url}/`, apiKey: "key" }).read(new Date());
assert.equal(r.name, "Den");
assert.equal(r.streams.length, 2);
assert.deepEqual(r.tasks, [], "its tasks could not be read");
assert.deepEqual([r.counts.movies, r.counts.songs, r.failedLogins24h], [120, null, null]);
assert.ok(seen.every((s) => s.token === "key"));
});
test("a refused key and a dead address are said plainly", async () => {
const bad = await adapter.createJellyfinAdapter({ url, apiKey: "nope" }).ping();
assert.equal(bad.ok, false);
assert.match(bad.error ?? "", /did not accept the API key/);
const good = await adapter.createJellyfinAdapter({ url, apiKey: "key" }).ping();
assert.equal(good.ok, true);
const dead = await adapter.createJellyfinAdapter({ url: "http://127.0.0.1:1", apiKey: "key" }).ping();
assert.equal(dead.ok, false);
});
});
});
describe("what is stored", () => {
let id = 0;
before(async () => {
id = (await db.insert(integrations).values({ type: "jellyfin", name: "Den", baseUrl: "http://10.9.9.8:8096", config: JSON.stringify({ url: "http://10.9.9.8:8096", apiKey: "key" }) }).returning({ id: integrations.id }))[0].id;
});
const points = async () => (await db.select().from(jellyfinReadings).where(eq(jellyfinReadings.integrationId, id))).sort((a, b) => a.at.localeCompare(b.at));
test("the state is replaced at each read; a history point is added when it is time or when the number streaming changes", async () => {
const t0 = new Date(Date.now() - 60 * MIN);
await jf.storeRead(id, read({ streams: [stream("Ann")] }), t0);
await jf.storeRead(id, read({ streams: [stream("Ann")] }), new Date(t0.getTime() + 1 * MIN));
assert.equal((await points()).length, 1, "nothing changed, and it is too soon");
await jf.storeRead(id, read({ streams: [stream("Ann"), stream("Bo", { transcoding: true })] }), new Date(t0.getTime() + 2 * MIN));
assert.deepEqual((await points()).map((p) => [p.streams, p.transcodes]), [[1, 0], [2, 1]], "a change is a point at once");
await jf.storeRead(id, read({ streams: [stream("Ann"), stream("Bo", { transcoding: true })] }), new Date(t0.getTime() + 8 * MIN));
assert.equal((await points()).length, 3, "and so is the passing of five minutes");
const [state] = await db.select().from(jellyfinState).where(eq(jellyfinState.integrationId, id));
assert.equal(jf.streamsOf(state).length, 2);
assert.equal(state.movies, 120);
});
test("readings older than 35 days are dropped", async () => {
await db.insert(jellyfinReadings).values({ integrationId: id, at: new Date(Date.now() - 40 * DAY).toISOString(), streams: 5, transcodes: 0 });
await jf.storeRead(id, read(), new Date());
assert.ok((await points()).every((p) => Date.parse(p.at) > Date.now() - 36 * DAY));
});
test("a read that failed is said, and what was stored is left alone", async () => {
const ok = await jf.syncJellyfin(id, new Date(), async () => read({ name: "Den", version: "10.10.4" }));
assert.equal(ok.ok, true);
const failed = await jf.syncJellyfin(id, new Date(), async () => {
throw new Error("connection refused");
});
assert.deepEqual([failed.ok, failed.error], [false, "connection refused"]);
assert.equal(failed.at, ok.at, "the time of the last good read is kept");
const [state] = await db.select().from(jellyfinState).where(eq(jellyfinState.integrationId, id));
assert.equal(state.version, "10.10.4");
assert.equal((await jf.jellyfinSyncStatuses())[id].ok, false);
assert.equal(jf.isJellyfinFresh((await jf.jellyfinSyncStatuses())[id], Date.now()), false, "a failed read is not believed");
});
test("only a good read recent enough is fresh", () => {
const now = Date.now();
assert.equal(jf.isJellyfinFresh({ at: new Date(now - 3 * MIN).toISOString(), ok: true, error: null }, now), true);
assert.equal(jf.isJellyfinFresh({ at: new Date(now - 11 * MIN).toISOString(), ok: true, error: null }, now), false);
assert.equal(jf.isJellyfinFresh(undefined, now), false);
});
});
describe("what is wrong, and the history in peaks", () => {
const row = (extra: Partial<Parameters<typeof jf.jellyfinProblems>[2]> = {}) => ({ pendingRestart: false, tasks: "[]", failedLogins24h: 0, ...extra });
test("a restart that is wanted, a task that failed, a burst of failed sign-ins", () => {
assert.deepEqual(jf.jellyfinProblems(1, "Den", row()), []);
const p = jf.jellyfinProblems(
1,
"Den",
row({ pendingRestart: true, failedLogins24h: jf.FAILED_LOGINS_WARN, tasks: JSON.stringify([{ name: "Scan", category: null, state: "Idle", lastStatus: "Failed", lastEndedAt: null, error: "disk full" }, { name: "Clean", category: null, state: "Idle", lastStatus: "Completed", lastEndedAt: null, error: null }]) }),
);
assert.deepEqual(p.map((x) => x.key), ["jellyfin-restart:1", "jellyfin-task:1:Scan", "jellyfin-logins:1"]);
assert.match(p[1].message, /“Scan” failed — disk full/);
assert.equal(jf.jellyfinProblems(1, "Den", row({ failedLogins24h: jf.FAILED_LOGINS_WARN - 1 })).length, 0);
});
test("history is bucketed to the peak of each stretch", () => {
const from = Date.parse("2026-10-20T00:00:00Z");
const to = from + 4 * 3_600_000;
const rows = [
{ at: new Date(from + 5 * MIN).toISOString(), streams: 1, transcodes: 0 },
{ at: new Date(from + 20 * MIN).toISOString(), streams: 3, transcodes: 2 },
{ at: new Date(from + 3 * 3_600_000 + 10 * MIN).toISOString(), streams: 2, transcodes: 0 },
{ at: new Date(from - MIN).toISOString(), streams: 9, transcodes: 9 }, // before the range
];
const b = jf.bucketPeaks(rows, from, to, 4);
assert.deepEqual(b.map((p) => [p.streams, p.transcodes]), [[3, 2], [2, 0]], "an average would have hidden the evening");
assert.deepEqual(jf.bucketPeaks([], from, to, 4), []);
});
});
describe("who is watching, the warning before an update, the health pass, the page and /metrics", () => {
let jfId = 0;
let serverId = 0;
let otherId = 0;
let app: Server;
let base = "";
const fresh = () => setInternalFlag("jellyfinSync", JSON.stringify({ [jfId]: { at: new Date().toISOString(), ok: true, error: null } }));
before(async () => {
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
jfId = (await db.insert(integrations).values({ type: "jellyfin", name: "Cinema", baseUrl: "http://10.9.9.9:8096", config: JSON.stringify({ url: "http://10.9.9.9:8096", apiKey: "key" }) }).returning({ id: integrations.id }))[0].id;
const mk = async (name: string, ip: string) =>
(await db.insert(servers).values({ name, apiTokenHash: name, apiTokenPrefix: name, lastSeenAt: new Date().toISOString(), ipAddresses: JSON.stringify([ip]) }).returning({ id: servers.id }))[0].id;
serverId = await mk("media", "10.9.9.9");
otherId = await mk("other", "10.9.9.10");
await jf.storeRead(jfId, read({ name: "Cinema", pendingRestart: true, streams: [stream("Ann", { title: "Dune" }), stream("Bo", { title: "The Office — Pilot", transcoding: true })] }));
await fresh();
const a = express();
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
next();
});
a.use("/api/jellyfin", jellyfinRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/jellyfin`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("the people streaming from the Jellyfin that runs on a server, in words; none on another, none when the read is old", async () => {
const w = await jf.watchingOnServer(serverId);
assert.equal(w.length, 1);
assert.equal(jf.watchingWords(w), "2 people are streaming from Cinema (Ann: Dune, Bo: The Office — Pilot), 1 being transcoded");
assert.deepEqual(await jf.watchingOnServer(otherId), []);
await setInternalFlag("jellyfinSync", JSON.stringify({ [jfId]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null } }));
assert.deepEqual(await jf.watchingOnServer(serverId), [], "an old reading is not believed");
await fresh();
assert.equal(jf.watchingWords([]), "");
});
test("an update run on that machine is held back with 409 'watching' unless it is told to go ahead", async () => {
const input = { serverId, integrationId: 999_999, projectId: 1, templateId: 1, limit: "media", windowMinutes: 90, actor: "test" };
await assert.rejects(
() => startUpdateRun(input),
(err: unknown) => err instanceof UpdateRunError && err.status === 409 && err.code === "watching" && /2 people are streaming from Cinema/.test(err.message),
);
// Past the warning it carries on to the next check, which is the Semaphore that doesn't exist here.
await assert.rejects(
() => startUpdateRun({ ...input, startAnyway: true }),
(err: unknown) => err instanceof UpdateRunError && err.code === "no_semaphore",
);
// A machine nobody is streaming from is not held back.
await assert.rejects(
() => startUpdateRun({ ...input, serverId: otherId }),
(err: unknown) => err instanceof UpdateRunError && err.code === "no_semaphore",
);
});
test("the health pass judges it from what is stored, and holds it when the last read is old", async () => {
let { snapshot, held } = await collectSnapshot();
assert.deepEqual(snapshot.jellyfin?.map((j) => j.integrationId), [jfId]);
assert.equal(held.has(`jellyfin:${jfId}`), false);
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
const c = list.find((x) => x.key === `jellyfin-restart:${jfId}`)!;
assert.deepEqual([c.severity, c.source], ["warning", `jellyfin:${jfId}`]);
assert.equal(subjectOfConditionKey(c.key), `integration:${jfId}`, "maintenance on it silences its alerts");
await setInternalFlag("jellyfinSync", JSON.stringify({ [jfId]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null } }));
({ snapshot, held } = await collectSnapshot());
assert.deepEqual([snapshot.jellyfin ?? [], held.has(`jellyfin:${jfId}`)], [[], true]);
await fresh();
});
test("the Alerts page shows it, with where to look", async () => {
invalidateAlertsCache();
const report = await getAlerts(true);
const a = report.alerts.find((x) => x.id === `monitoring:jellyfin-restart:${jfId}`)!;
assert.ok(a, "the restart that is wanted is an alert");
assert.deepEqual([a.severity, a.category, a.source, a.link, a.target?.type, a.target?.id], ["warning", "monitoring", "Jellyfin", "/jellyfin", "integration", jfId]);
});
test("anyone signed in sees each Jellyfin, who is streaming and what is wrong", async () => {
const res = await fetch(base);
assert.equal(res.status, 200);
const body = (await res.json()) as { jellyfins: { id: number; fresh: boolean; runsOn: { name: string } | null; state: { streams: unknown[] } | null; problems: { key: string }[] }[] };
const j = body.jellyfins.find((x) => x.id === jfId)!;
assert.equal(j.fresh, true);
assert.equal(j.runsOn?.name, "media");
assert.equal(j.state?.streams.length, 2);
assert.ok(j.problems.some((p) => p.key === `jellyfin-restart:${jfId}`));
});
test("who is watching on a server is asked for by its id, and a wrong id is refused", async () => {
const r = (await (await fetch(`${base}/watching?serverId=${serverId}`)).json()) as { watching: unknown[]; words: string };
assert.equal(r.watching.length, 1);
assert.match(r.words, /2 people are streaming/);
const none = (await (await fetch(`${base}/watching?serverId=${otherId}`)).json()) as { watching: unknown[]; words: string };
assert.deepEqual([none.watching.length, none.words], [0, ""]);
assert.equal((await fetch(`${base}/watching?serverId=abc`)).status, 400);
});
test("the history is the peaks of the last days; a missing Jellyfin is not found", async () => {
const res = await fetch(`${base}/${jfId}/history?days=1`);
assert.equal(res.status, 200);
const body = (await res.json()) as { days: number; points: { streams: number; transcodes: number }[] };
assert.equal(body.days, 1);
assert.ok(body.points.some((p) => p.streams === 2 && p.transcodes === 1));
assert.equal((await (await fetch(`${base}/${jfId}/history?days=400`)).json() as { days: number }).days, jf.READING_KEEP_DAYS, "no further back than is kept");
assert.equal((await fetch(`${base}/999999/history`)).status, 404);
});
test("/metrics has how many are streaming, how many are transcoded and whether a restart is wanted — only while the reading is recent", async () => {
let text = await buildMetrics(new Date());
assert.match(text, /homelab_jellyfin_streams\{name="Cinema"\} 2/);
assert.match(text, /homelab_jellyfin_transcodes\{name="Cinema"\} 1/);
assert.match(text, /homelab_jellyfin_pending_restart\{name="Cinema"\} 1/);
await setInternalFlag("jellyfinSync", JSON.stringify({ [jfId]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null } }));
text = await buildMetrics(new Date());
assert.doesNotMatch(text, /homelab_jellyfin_streams/);
});
});
+36 -2
View File
@@ -20,14 +20,15 @@ process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, patchPlans, patchPlanRuns, updateRuns, maintenanceWindows } = await import("../src/db/schema.js");
const { users, servers, patchPlans, patchPlanRuns, updateRuns, maintenanceWindows, integrations } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { planServers, dueOccurrence, summariseItems, startPlanRun, advancePlanRuns, startDuePlans, cancelPlanRun, parseItems, PlanError } = await import("../src/services/patchPlans.js");
const { advanceUpdateRuns, setSemaphoreForTests } = await import("../src/services/updateRuns.js");
const { listActiveWindows } = await import("../src/services/maintenance.js");
const { agendaFromData } = await import("../src/services/agenda.js");
const { patchPlansRouter } = await import("../src/routes/patchPlans.js");
const { updateSettings } = await import("../src/services/settingsStore.js");
const { updateSettings, setInternalFlag } = await import("../src/services/settingsStore.js");
const { storeRead } = await import("../src/services/jellyfinSync.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
@@ -196,6 +197,39 @@ describe("a run, from start to end", () => {
assert.match(last, /✓ web/);
});
test("a server people are streaming from waits its turn for up to two hours, then is updated anyway", async () => {
await reset({ tags: "[]", serverIds: JSON.stringify([ids.api]) });
await db.update(servers).set({ ipAddresses: JSON.stringify(["10.7.7.7"]) }).where(eq(servers.id, ids.api));
const jfId = (await db.insert(integrations).values({ type: "jellyfin", name: "Cinema", baseUrl: "http://10.7.7.7:8096", config: "{}" }).returning({ id: integrations.id }))[0].id;
const seen = (m: number) => setInternalFlag("jellyfinSync", JSON.stringify({ [jfId]: { at: at(m).toISOString(), ok: true, error: null } }));
try {
await storeRead(jfId, { name: "Cinema", version: null, os: null, pendingRestart: false, updateAvailable: false, streams: [{ user: "Ann", client: null, device: null, title: "Dune", transcoding: false, paused: false }], tasks: [], counts: { movies: null, series: null, episodes: null, songs: null }, failedLogins24h: null }, T0);
const run = await startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 });
await seen(0);
await advancePlanRuns(T0);
let [item] = await itemsOf(run.id);
assert.deepEqual([item.status, fake.runs.length], ["pending", 0], "somebody is watching, so it is not its turn yet");
assert.match(item.message ?? "", /^Waiting: 1 person is streaming from Cinema \(Ann: Dune\)/);
assert.equal(item.waitingSince, T0.toISOString());
await seen(60);
await advancePlanRuns(at(60));
[item] = await itemsOf(run.id);
assert.deepEqual([item.status, fake.runs.length, item.waitingSince], ["pending", 0, T0.toISOString()], "an hour on, still waiting, and the wait is counted from the first time");
await seen(121);
await advancePlanRuns(at(121));
[item] = await itemsOf(run.id);
assert.deepEqual([item.status, fake.runs.map((r) => r.limit)], ["running", ["api"]], "after two hours it goes ahead");
assert.match(item.message ?? "", /^Went ahead after waiting 2 h; 1 person is streaming/);
} finally {
await db.delete(integrations).where(eq(integrations.id, jfId));
await setInternalFlag("jellyfinSync", "{}");
await db.update(servers).set({ ipAddresses: null }).where(eq(servers.id, ids.api));
}
});
test("what depends on a server is silenced while it is updated, and let go when it is done", async () => {
await reset({ tags: "[]", serverIds: JSON.stringify([ids.web]) });
const run = await startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 });