Jellyfin: who is watching, and don't update a machine mid-film
New Jellyfin integration (read-only, every 2 minutes): version, restart wanted, streams and transcoding, failed scheduled tasks, item counts, failed sign-ins. Monitoring -> Jellyfin page with a peak-streams history, three alerts, /metrics. When Jellyfin runs on a managed server, an update run asks first if people are streaming (409 "watching" unless startAnyway), and a patch plan waits up to two hours for them before updating that server. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
2f082ca5c2
commit
0146eb8bca
38 files changed
+8136
-13
No files matched your search
@@ -58,6 +58,7 @@ import { homeAssistantRouter } from "./routes/homeAssistant.js";
|
||||
import { upsRouter } from "./routes/ups.js";
|
||||
import { awayRouter } from "./routes/away.js";
|
||||
import { duplicatiReportRouter, duplicatiRouter } from "./routes/duplicati.js";
|
||||
import { jellyfinRouter } from "./routes/jellyfin.js";
|
||||
import { serviceLevelsRouter } from "./routes/serviceLevels.js";
|
||||
import { publicApiRouter } from "./routes/publicApi.js";
|
||||
import { generatorRouter } from "./routes/generator.js";
|
||||
@@ -193,6 +194,7 @@ export function createApp() {
|
||||
app.use("/api/ups", upsRouter);
|
||||
app.use("/api/away", awayRouter);
|
||||
app.use("/api/duplicati", duplicatiRouter);
|
||||
app.use("/api/jellyfin", jellyfinRouter);
|
||||
app.use("/api/service-levels", serviceLevelsRouter);
|
||||
app.use("/api/todos", todosRouter);
|
||||
app.use("/api/notes", notesRouter);
|
||||
|
||||
@@ -553,6 +553,7 @@ export const integrationTypes = [
|
||||
"npm",
|
||||
"homeassistant",
|
||||
"ups",
|
||||
"jellyfin",
|
||||
] as const;
|
||||
export type IntegrationType = (typeof integrationTypes)[number];
|
||||
|
||||
@@ -1159,6 +1160,44 @@ export const haReadings = sqliteTable(
|
||||
(t) => [index("ha_readings_lookup_idx").on(t.integrationId, t.entityId, t.at)],
|
||||
);
|
||||
|
||||
// ─── Jellyfin ───────────────────────────────────────────────────────────────
|
||||
|
||||
// How a Jellyfin server (an integration of type "jellyfin") is right now, one row each, replaced at every read: its version, whether it wants a restart, who is streaming, how its
|
||||
// scheduled tasks last ended, and how many sign-ins failed lately. Read by services/jellyfinSync.ts.
|
||||
export const jellyfinState = sqliteTable("jellyfin_state", {
|
||||
integrationId: integer("integration_id")
|
||||
.primaryKey()
|
||||
.references(() => integrations.id, { onDelete: "cascade" }),
|
||||
serverName: text("server_name"),
|
||||
version: text("version"),
|
||||
os: text("os"),
|
||||
pendingRestart: integer("pending_restart", { mode: "boolean" }).notNull().default(false),
|
||||
updateAvailable: integer("update_available", { mode: "boolean" }).notNull().default(false),
|
||||
streams: text("streams").notNull().default("[]"), // JSON: who is streaming what
|
||||
tasks: text("tasks").notNull().default("[]"), // JSON: the scheduled tasks and how they last ended
|
||||
movies: integer("movies"),
|
||||
series: integer("series"),
|
||||
episodes: integer("episodes"),
|
||||
songs: integer("songs"),
|
||||
failedLogins24h: integer("failed_logins_24h"),
|
||||
updatedAt: text("updated_at").notNull(), // ISO
|
||||
});
|
||||
|
||||
// How many were streaming (and how many of those were being transcoded) at a moment, for the chart: one every five minutes, and at once when it changes. Kept 35 days.
|
||||
export const jellyfinReadings = sqliteTable(
|
||||
"jellyfin_readings",
|
||||
{
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
integrationId: integer("integration_id")
|
||||
.notNull()
|
||||
.references(() => integrations.id, { onDelete: "cascade" }),
|
||||
at: text("at").notNull(), // ISO
|
||||
streams: integer("streams").notNull(),
|
||||
transcodes: integer("transcodes").notNull(),
|
||||
},
|
||||
(t) => [index("jellyfin_readings_idx").on(t.integrationId, t.at)],
|
||||
);
|
||||
|
||||
// ─── Duplicati ──────────────────────────────────────────────────────────────
|
||||
|
||||
// A Duplicati that reports to this app: one row per machine running it, with the secret token its reports carry in their address (only the hash is kept). Duplicati pushes a
|
||||
|
||||
@@ -29,6 +29,7 @@ import { initUnifiSync } from "./services/unifiSync.js";
|
||||
import { initHomeAssistantSync } from "./services/homeAssistantSync.js";
|
||||
import { initUpsSync } from "./services/upsSync.js";
|
||||
import { initAwayMonitor } from "./services/awayMonitor.js";
|
||||
import { initJellyfinSync } from "./services/jellyfinSync.js";
|
||||
import { backfillMetricRollups } from "./services/serverMetrics.js";
|
||||
import { backfillEndpointHourly } from "./services/endpointChecks.js";
|
||||
import { initSelfBackupScheduler } from "./services/selfBackupScheduler.js";
|
||||
@@ -79,6 +80,7 @@ initUnifiSync();
|
||||
initHomeAssistantSync();
|
||||
initUpsSync();
|
||||
initAwayMonitor();
|
||||
initJellyfinSync();
|
||||
// The hourly and daily figures are built from what is already kept, once, the first time after they were added (in the background: nothing waits for it).
|
||||
void Promise.all([backfillMetricRollups(), backfillEndpointHourly()]).catch((err) => console.error("[history] couldn't build the hourly and daily figures:", err));
|
||||
|
||||
|
||||
@@ -99,6 +99,11 @@ export const INTEGRATION_FIELDS: Partial<Record<IntegrationType, IntegrationFiel
|
||||
{ key: "token", label: "Long-lived access token", secret: true, type: "password", placeholder: "made on your profile page in Home Assistant" },
|
||||
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
|
||||
],
|
||||
jellyfin: [
|
||||
{ key: "url", label: "Jellyfin URL", secret: false, placeholder: "http://jellyfin.example.lan:8096" },
|
||||
{ key: "apiKey", label: "API key", secret: true, type: "password", placeholder: "made under Dashboard → API Keys in Jellyfin" },
|
||||
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
|
||||
],
|
||||
ups: [
|
||||
{ key: "host", label: "Host name or address", secret: false, placeholder: "the machine running NUT (upsd), or the UPS's network card" },
|
||||
{ key: "snmp", label: "Read it over SNMP (version 2c) instead of NUT", secret: false, type: "checkbox" },
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
/**
|
||||
* Jellyfin adapter. Config: url, apiKey (made in Jellyfin under Dashboard → API Keys); optional: insecure.
|
||||
*
|
||||
* Reading only: the server's own information (version, whether it wants a restart or an update), who is streaming right now and whether it is being transcoded, the scheduled tasks
|
||||
* (a library scan that failed), how many items there are and how many sign-ins failed lately. Nothing is sent to Jellyfin but requests for these.
|
||||
*
|
||||
* Endpoints and field names are from Jellyfin's published API (`/System/Info`, `/Sessions`, `/ScheduledTasks`, `/Items/Counts`, `/System/ActivityLog/Entries`) and have not
|
||||
* been checked against a live server here; every field is read defensively, and one that is missing is just left out.
|
||||
*/
|
||||
import { requestJson } from "../httpJson.js";
|
||||
import { withDiagLogging } from "../../services/diagLog.js";
|
||||
|
||||
export interface JellyfinConfig {
|
||||
url: string;
|
||||
apiKey: string;
|
||||
insecure?: boolean;
|
||||
}
|
||||
|
||||
export interface JellyfinStream {
|
||||
user: string;
|
||||
client: string | null;
|
||||
device: string | null;
|
||||
/** What is being watched or listened to, with the series it is of. */
|
||||
title: string;
|
||||
/** Converted on the fly, which is what loads the server. */
|
||||
transcoding: boolean;
|
||||
paused: boolean;
|
||||
}
|
||||
|
||||
export interface JellyfinTask {
|
||||
name: string;
|
||||
category: string | null;
|
||||
state: string;
|
||||
/** Completed, Failed, Cancelled or Aborted; null when it has never run. */
|
||||
lastStatus: string | null;
|
||||
lastEndedAt: string | null;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
export interface JellyfinRead {
|
||||
name: string | null;
|
||||
version: string | null;
|
||||
os: string | null;
|
||||
pendingRestart: boolean;
|
||||
updateAvailable: boolean;
|
||||
streams: JellyfinStream[];
|
||||
tasks: JellyfinTask[];
|
||||
counts: { movies: number | null; series: number | null; episodes: number | null; songs: number | null };
|
||||
/** Failed sign-ins in the last day; null when the activity log could not be read. */
|
||||
failedLogins24h: number | null;
|
||||
}
|
||||
|
||||
export interface JellyfinAdapter {
|
||||
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
|
||||
read(now?: Date): Promise<JellyfinRead>;
|
||||
}
|
||||
|
||||
const obj = (v: unknown): Record<string, unknown> => (v !== null && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {});
|
||||
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() !== "" ? v.trim() : null);
|
||||
const num = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null);
|
||||
const iso = (v: unknown): string | null => {
|
||||
const s = str(v);
|
||||
return s && Number.isFinite(Date.parse(s)) ? new Date(s).toISOString() : null;
|
||||
};
|
||||
|
||||
/** Pure: what /System/Info says. */
|
||||
export function parseSystemInfo(raw: unknown): Pick<JellyfinRead, "name" | "version" | "os" | "pendingRestart" | "updateAvailable"> {
|
||||
const o = obj(raw);
|
||||
return { name: str(o.ServerName), version: str(o.Version), os: str(o.OperatingSystemDisplayName) ?? str(o.OperatingSystem), pendingRestart: o.HasPendingRestart === true, updateAvailable: o.HasUpdateAvailable === true };
|
||||
}
|
||||
|
||||
/** Pure: the sessions that are playing something. A session with nothing playing is just somebody signed in. */
|
||||
export function parseSessions(raw: unknown): JellyfinStream[] {
|
||||
const out: JellyfinStream[] = [];
|
||||
for (const item of Array.isArray(raw) ? raw : []) {
|
||||
const s = obj(item);
|
||||
const playing = obj(s.NowPlayingItem);
|
||||
const name = str(playing.Name);
|
||||
if (!name) continue;
|
||||
const series = str(playing.SeriesName);
|
||||
const state = obj(s.PlayState);
|
||||
out.push({
|
||||
user: str(s.UserName) ?? "someone",
|
||||
client: str(s.Client),
|
||||
device: str(s.DeviceName),
|
||||
title: series ? `${series} — ${name}` : name,
|
||||
transcoding: str(state.PlayMethod)?.toLowerCase() === "transcode" || (s.TranscodingInfo !== undefined && s.TranscodingInfo !== null && Object.keys(obj(s.TranscodingInfo)).length > 0),
|
||||
paused: state.IsPaused === true,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Pure: the scheduled tasks with how they last ended. */
|
||||
export function parseTasks(raw: unknown): JellyfinTask[] {
|
||||
const out: JellyfinTask[] = [];
|
||||
for (const item of Array.isArray(raw) ? raw : []) {
|
||||
const t = obj(item);
|
||||
const name = str(t.Name);
|
||||
if (!name || t.IsHidden === true) continue;
|
||||
const last = obj(t.LastExecutionResult);
|
||||
out.push({ name, category: str(t.Category), state: str(t.State) ?? "Idle", lastStatus: str(last.Status), lastEndedAt: iso(last.EndTimeUtc), error: str(last.ErrorMessage) });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Pure: how many of the activity log's entries from the last day are failed sign-ins. */
|
||||
export function countFailedLogins(raw: unknown, now: Date): number | null {
|
||||
const items = obj(raw).Items;
|
||||
if (!Array.isArray(items)) return null;
|
||||
const since = now.getTime() - 86_400_000;
|
||||
return items.filter((i) => {
|
||||
const e = obj(i);
|
||||
const at = Date.parse(String(e.Date ?? ""));
|
||||
return str(e.Type) === "AuthenticationFailed" && (!Number.isFinite(at) || at >= since);
|
||||
}).length;
|
||||
}
|
||||
|
||||
export function createJellyfinAdapter(config: JellyfinConfig): JellyfinAdapter {
|
||||
const base = String(config.url ?? "").replace(/\/+$/, "");
|
||||
const insecure = config.insecure === true;
|
||||
const headers = { "X-Emby-Token": config.apiKey, Authorization: `MediaBrowser Token="${config.apiKey}"` };
|
||||
|
||||
async function get(path: string): Promise<unknown> {
|
||||
const res = await requestJson(`${base}${path}`, { headers, insecure });
|
||||
if (res.status === 401 || res.status === 403) throw new Error(`Jellyfin did not accept the API key (HTTP ${res.status})`);
|
||||
if (res.status === 404) throw new Error("Jellyfin has no API at this address — check the address and port (8096)");
|
||||
if (res.status < 200 || res.status >= 300) throw new Error(`Jellyfin error: HTTP ${res.status}`);
|
||||
if (res.json === null) throw new Error("Jellyfin answered with something that isn't JSON — is the address its own?");
|
||||
return res.json;
|
||||
}
|
||||
|
||||
async function read(now: Date = new Date()): Promise<JellyfinRead> {
|
||||
const info = parseSystemInfo(await get("/System/Info"));
|
||||
// The rest is worth having but not worth failing for: a Jellyfin that answers about itself and not about its sessions still says it is up.
|
||||
const soft = async <T>(path: string, parse: (raw: unknown) => T, fallback: T): Promise<T> => {
|
||||
try {
|
||||
return parse(await get(path));
|
||||
} catch {
|
||||
return fallback;
|
||||
}
|
||||
};
|
||||
const [streams, tasks, countsRaw, failed] = await Promise.all([
|
||||
soft("/Sessions?ActiveWithinSeconds=300", parseSessions, [] as JellyfinStream[]),
|
||||
soft("/ScheduledTasks?isHidden=false", parseTasks, [] as JellyfinTask[]),
|
||||
soft("/Items/Counts", obj, {} as Record<string, unknown>),
|
||||
soft(`/System/ActivityLog/Entries?startIndex=0&limit=100&minDate=${encodeURIComponent(new Date(now.getTime() - 86_400_000).toISOString())}`, (r) => countFailedLogins(r, now), null as number | null),
|
||||
]);
|
||||
return {
|
||||
...info,
|
||||
streams,
|
||||
tasks,
|
||||
counts: { movies: num(countsRaw.MovieCount), series: num(countsRaw.SeriesCount), episodes: num(countsRaw.EpisodeCount), songs: num(countsRaw.SongCount) },
|
||||
failedLogins24h: failed,
|
||||
};
|
||||
}
|
||||
|
||||
async function ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }> {
|
||||
const start = Date.now();
|
||||
try {
|
||||
await get("/System/Info");
|
||||
return { ok: true, latencyMs: Date.now() - start };
|
||||
} catch (err) {
|
||||
return { ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
}
|
||||
}
|
||||
|
||||
return withDiagLogging("jellyfin", { ping, read });
|
||||
}
|
||||
@@ -14,6 +14,7 @@ import { createUnifiAdapter } from "./unifi/adapter.js";
|
||||
import { createNpmAdapter } from "./npm/adapter.js";
|
||||
import { createHomeAssistantAdapter } from "./homeassistant/adapter.js";
|
||||
import { createUpsAdapter } from "./ups/adapter.js";
|
||||
import { createJellyfinAdapter } from "./jellyfin/adapter.js";
|
||||
|
||||
export interface PingableAdapter {
|
||||
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
|
||||
@@ -56,6 +57,8 @@ export function createIntegrationAdapter(type: IntegrationType, config: Integrat
|
||||
return createHomeAssistantAdapter(config as any);
|
||||
case "ups":
|
||||
return createUpsAdapter(config as any);
|
||||
case "jellyfin":
|
||||
return createJellyfinAdapter(config as any);
|
||||
default:
|
||||
throw new Error(`Integration type "${type}" is not implemented yet`);
|
||||
}
|
||||
|
||||
@@ -29,6 +29,7 @@ import { getSourceStatus } from "../services/integrationHealthMonitor.js";
|
||||
import { removeIntegrationData, syncUnifiIntegration } from "../services/unifiSync.js";
|
||||
import { removeHomeAssistantData, syncHomeAssistant } from "../services/homeAssistantSync.js";
|
||||
import { forgetUps, syncUps } from "../services/upsSync.js";
|
||||
import { forgetJellyfin, syncJellyfin } from "../services/jellyfinSync.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { getSettings, updateSettings } from "../services/settingsStore.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
@@ -131,6 +132,7 @@ integrationsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res)
|
||||
if (type === "unifi") void syncUnifiIntegration(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "homeassistant") void syncHomeAssistant(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "ups") void syncUps(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "jellyfin") void syncJellyfin(created.id).then(() => invalidateAlertsCache());
|
||||
|
||||
res.status(201).json({
|
||||
integration: {
|
||||
@@ -234,6 +236,7 @@ integrationsRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req,
|
||||
if (updated.type === "unifi" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncUnifiIntegration(updated.id).then(() => invalidateAlertsCache());
|
||||
if (updated.type === "homeassistant" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncHomeAssistant(updated.id).then(() => invalidateAlertsCache());
|
||||
if (updated.type === "ups" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncUps(updated.id).then(() => invalidateAlertsCache());
|
||||
if (updated.type === "jellyfin" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncJellyfin(updated.id).then(() => invalidateAlertsCache());
|
||||
|
||||
res.json({
|
||||
integration: {
|
||||
@@ -313,6 +316,7 @@ integrationsRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req,
|
||||
if (existing.type === "unifi") await removeIntegrationData(id);
|
||||
if (existing.type === "homeassistant") await removeHomeAssistantData(id);
|
||||
if (existing.type === "ups") await forgetUps(id);
|
||||
if (existing.type === "jellyfin") await forgetJellyfin(id);
|
||||
await db.delete(integrations).where(eq(integrations.id, id));
|
||||
if (existing.credentialId) {
|
||||
await db.delete(integrationCredentials).where(eq(integrationCredentials.id, existing.credentialId));
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
import { Router } from "express";
|
||||
import { and, asc, eq, gte } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, jellyfinReadings, jellyfinState, servers } from "../db/schema.js";
|
||||
import { requireAuth } from "../auth/middleware.js";
|
||||
import { hostOfAddress } from "../services/kumaTargets.js";
|
||||
import { bucketPeaks, isJellyfinFresh, jellyfinProblems, jellyfinSyncStatuses, READING_KEEP_DAYS, streamsOf, tasksOf, watchingOnServer, watchingWords } from "../services/jellyfinSync.js";
|
||||
import { matchServerForTarget } from "../services/uptimeKumaMatch.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const jellyfinRouter = Router();
|
||||
jellyfinRouter.use(requireAuth);
|
||||
|
||||
const DAY_MS = 86_400_000;
|
||||
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
|
||||
|
||||
/** Every Jellyfin, how it is and who is streaming from it, from what the last read stored. Everyone signed in can see it, like the other pages made of stored readings. */
|
||||
jellyfinRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const [rows, states, statuses, serverRows] = await Promise.all([
|
||||
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled, baseUrl: integrations.baseUrl }).from(integrations).where(eq(integrations.type, "jellyfin")).orderBy(integrations.name),
|
||||
db.select().from(jellyfinState),
|
||||
jellyfinSyncStatuses(),
|
||||
db.select({ id: servers.id, name: servers.name, hostname: servers.hostname, ipAddresses: servers.ipAddresses }).from(servers),
|
||||
]);
|
||||
const matchable = serverRows.map((s) => {
|
||||
let ips: string[] = [];
|
||||
try {
|
||||
const v = JSON.parse(s.ipAddresses ?? "[]");
|
||||
ips = Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : [];
|
||||
} catch {
|
||||
// none
|
||||
}
|
||||
return { id: s.id, name: s.name, hostname: s.hostname, ips };
|
||||
});
|
||||
const byId = new Map(states.map((s) => [s.integrationId, s]));
|
||||
const now = Date.now();
|
||||
res.json({
|
||||
jellyfins: rows.map((r) => {
|
||||
const state = byId.get(r.id) ?? null;
|
||||
const status = statuses[r.id] ?? null;
|
||||
const fresh = isJellyfinFresh(status ?? undefined, now);
|
||||
const machine = matchServerForTarget(hostOfAddress(r.baseUrl), matchable);
|
||||
return {
|
||||
id: r.id,
|
||||
name: r.name,
|
||||
enabled: r.enabled,
|
||||
runsOn: machine ? { id: machine.id, name: machine.name } : null,
|
||||
status,
|
||||
fresh,
|
||||
state: state ? { serverName: state.serverName, version: state.version, os: state.os, pendingRestart: state.pendingRestart, updateAvailable: state.updateAvailable, streams: streamsOf(state), tasks: tasksOf(state), counts: { movies: state.movies, series: state.series, episodes: state.episodes, songs: state.songs }, failedLogins24h: state.failedLogins24h, updatedAt: state.updatedAt } : null,
|
||||
problems: state && fresh ? jellyfinProblems(r.id, r.name, state) : [],
|
||||
};
|
||||
}),
|
||||
});
|
||||
}));
|
||||
|
||||
/** Who is streaming from the Jellyfin that runs on a server, for the warning before it is restarted. */
|
||||
jellyfinRouter.get("/watching", asyncHandler(async (req, res) => {
|
||||
const serverId = idOf(String(req.query.serverId ?? ""));
|
||||
if (!Number.isInteger(serverId)) return res.status(400).json({ error: "invalid_id" });
|
||||
const watching = await watchingOnServer(serverId);
|
||||
res.json({ watching, words: watchingWords(watching) });
|
||||
}));
|
||||
|
||||
/** How many were streaming over time, as peaks (readings are kept 35 days). */
|
||||
jellyfinRouter.get("/:id/history", asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [row] = await db.select({ id: integrations.id, name: integrations.name, type: integrations.type }).from(integrations).where(eq(integrations.id, id)).limit(1);
|
||||
if (!row || row.type !== "jellyfin") return res.status(404).json({ error: "not_found" });
|
||||
const asked = Number(req.query.days);
|
||||
const days = Number.isFinite(asked) ? Math.min(READING_KEEP_DAYS, Math.max(1, Math.round(asked))) : 7;
|
||||
const to = Date.now();
|
||||
const from = to - days * DAY_MS;
|
||||
const rows = await db.select().from(jellyfinReadings).where(and(eq(jellyfinReadings.integrationId, id), gte(jellyfinReadings.at, new Date(from).toISOString()))).orderBy(asc(jellyfinReadings.at));
|
||||
res.json({ jellyfin: { id: row.id, name: row.name }, days, points: bucketPeaks(rows, from, to, 200) });
|
||||
}));
|
||||
@@ -18,6 +18,8 @@ const startSchema = z.object({
|
||||
templateId: z.number().int().positive(),
|
||||
limit: z.string().trim().regex(/^[A-Za-z0-9][A-Za-z0-9._,:*\-]{0,199}$/, "Use the host name as it is in the Semaphore inventory, like web01"),
|
||||
windowMinutes: z.number().int().min(15).max(480).default(90),
|
||||
// Somebody is streaming from the machine and the person has seen the warning.
|
||||
startAnyway: z.boolean().optional(),
|
||||
});
|
||||
|
||||
const shape = (r: Awaited<ReturnType<typeof listUpdateRuns>>[number]) => ({
|
||||
|
||||
@@ -75,6 +75,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
|
||||
ups: { label: "UPS", link: "/ups" },
|
||||
slo: { label: "Service level", link: "/service-levels" },
|
||||
duplicati: { label: "Duplicati", link: "/duplicati" },
|
||||
jellyfin: { label: "Jellyfin", link: "/jellyfin" },
|
||||
secrets: { label: "Secrets", link: "/secrets" },
|
||||
domains: { label: "Domains", link: "/dns?view=domains" },
|
||||
endpoint: { label: "Endpoint", link: "/endpoints" },
|
||||
@@ -113,6 +114,8 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
|
||||
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
|
||||
m = /^duplicati-(?:failed|stale):(\d+)/.exec(key);
|
||||
if (m) return { type: "duplicati", id: Number(m[1]), category: "backup" };
|
||||
m = /^jellyfin-(?:restart|task|logins):(\d+)/.exec(key);
|
||||
if (m) return { type: "jellyfin", id: Number(m[1]), category: key.startsWith("jellyfin-task") ? "automation" : "monitoring" };
|
||||
return { type: "server", id: null, category: "disk" };
|
||||
}
|
||||
|
||||
@@ -235,7 +238,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
type: origin.type,
|
||||
message: c.message,
|
||||
key: c.key,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : undefined,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : undefined,
|
||||
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
|
||||
});
|
||||
}
|
||||
@@ -245,7 +248,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health");
|
||||
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
|
||||
for (const h of held) {
|
||||
const m = /^(proxmox|synology|unifi|ups|dockhand):(\d+)(?::(.+))?$/.exec(h);
|
||||
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin):(\d+)(?::(.+))?$/.exec(h);
|
||||
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
|
||||
const row = intById.get(Number(m[2]));
|
||||
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, networkDevices, serverPorts, servers, upsState } from "../db/schema.js";
|
||||
import { integrations, jellyfinState, networkDevices, serverPorts, servers, upsState } from "../db/schema.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
|
||||
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
|
||||
@@ -20,6 +20,7 @@ import { baselineKey, computeBaselineFindings, type BaselineFinding } from "./ba
|
||||
import { levelFindings, levelKey, type LevelFinding } from "./serviceLevels.js";
|
||||
import { createDockhandAdapter } from "../integrations/dockhand/adapter.js";
|
||||
import { currentProblems as duplicatiProblemsNow, type DuplicatiProblem } from "./duplicati.js";
|
||||
import { isJellyfinFresh, jellyfinProblems, jellyfinSyncStatuses, type JellyfinProblem } from "./jellyfinSync.js";
|
||||
import { evaluateContainers, loadContainerMemory, saveContainerMemory, type ContainerMemory, type DockerSnapshot } from "./containerHealth.js";
|
||||
import { recordStorageSamples } from "./storageHistory.js";
|
||||
import { isUpsFresh, upsProblems, upsSyncStatuses, type UpsStateRow } from "./upsSync.js";
|
||||
@@ -87,6 +88,8 @@ export interface HealthSnapshot {
|
||||
baselines?: BaselineFinding[];
|
||||
/** Service-level targets whose error budget for the month is mostly or wholly used (services/serviceLevels.ts). */
|
||||
levels?: LevelFinding[];
|
||||
/** What is wrong with each Jellyfin, from what was stored, only while that reading is recent (services/jellyfinSync.ts). */
|
||||
jellyfin?: { integrationId: number; problems: JellyfinProblem[] }[];
|
||||
/** Duplicati backups that failed or have not succeeded in time (services/duplicati.ts). */
|
||||
duplicati?: DuplicatiProblem[];
|
||||
/** Each Docker integration's containers as last read, with what was remembered of them (services/containerHealth.ts). */
|
||||
@@ -198,6 +201,9 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
}
|
||||
}
|
||||
|
||||
// A Jellyfin that wants a restart, a scheduled task that failed, a burst of failed sign-ins.
|
||||
for (const j of snapshot.jellyfin ?? []) for (const p of j.problems) out.push({ key: p.key, source: `jellyfin:${j.integrationId}`, message: p.message, severity: p.severity });
|
||||
|
||||
// Duplicati backups that failed, or have not succeeded for longer than they should.
|
||||
for (const p of snapshot.duplicati ?? []) out.push({ key: p.key, source: "duplicati", message: p.message, severity: p.severity });
|
||||
|
||||
@@ -428,6 +434,24 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
}
|
||||
}
|
||||
|
||||
// Jellyfin: read by its own pass every couple of minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsJ = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "jellyfin"), eq(integrations.enabled, true)));
|
||||
if (rowsJ.length > 0) {
|
||||
const statuses = await jellyfinSyncStatuses();
|
||||
for (const r of rowsJ) {
|
||||
const [state] = await db.select().from(jellyfinState).where(eq(jellyfinState.integrationId, r.id)).limit(1);
|
||||
if (!state || !isJellyfinFresh(statuses[r.id], Date.now())) {
|
||||
held.add(`jellyfin:${r.id}`);
|
||||
continue;
|
||||
}
|
||||
(snapshot.jellyfin ??= []).push({ integrationId: r.id, problems: jellyfinProblems(r.id, r.name, state) });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't judge Jellyfin:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// Duplicati: what its reports say, read from what is stored (nothing is asked of Duplicati). A failure here costs only this check.
|
||||
try {
|
||||
snapshot.duplicati = await duplicatiProblemsNow(Date.now());
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
/**
|
||||
* Keeps what Jellyfin says up to date: every two minutes each Jellyfin integration is read (its version, who is streaming, how its scheduled tasks ended, how many sign-ins
|
||||
* failed), the state is stored, and the number streaming is added to a history. Pages, charts and alerts are built from what is stored, so none of them talks to Jellyfin.
|
||||
*
|
||||
* Two things come of it besides a page. **Alerts**: a server that wants a restart, a scheduled task (a library scan) that failed, a burst of failed sign-ins. And **who is
|
||||
* watching**: before an update run or a patch plan restarts the machine Jellyfin runs on, the app can say that people are streaming from it — and a patch plan waits for them to
|
||||
* finish, for a while, instead of cutting a film off.
|
||||
*/
|
||||
import { and, desc, eq, lt } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, jellyfinReadings, jellyfinState, servers } from "../db/schema.js";
|
||||
import { createJellyfinAdapter, type JellyfinRead, type JellyfinStream, type JellyfinTask } from "../integrations/jellyfin/adapter.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { hostOfAddress } from "./kumaTargets.js";
|
||||
import { matchServerForTarget } from "./uptimeKumaMatch.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export const SYNC_INTERVAL_MS = 2 * 60_000;
|
||||
/** After this long without a good read, what is stored is no longer believed. */
|
||||
export const STALE_AFTER_MS = 10 * 60_000;
|
||||
export const READING_KEEP_DAYS = 35;
|
||||
const READING_GAP_MS = 5 * 60_000 - 20_000;
|
||||
/** This many failed sign-ins in a day is a warning. */
|
||||
export const FAILED_LOGINS_WARN = 10;
|
||||
const STATUS_FLAG = "jellyfinSync";
|
||||
|
||||
export interface JellyfinSyncStatus {
|
||||
/** The last read that worked (kept when a later one failed). */
|
||||
at: string;
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
async function readStatuses(): Promise<Record<string, JellyfinSyncStatus>> {
|
||||
try {
|
||||
const raw = await getInternalFlag(STATUS_FLAG);
|
||||
return raw ? (JSON.parse(raw) as Record<string, JellyfinSyncStatus>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export async function jellyfinSyncStatuses(): Promise<Record<number, JellyfinSyncStatus>> {
|
||||
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
|
||||
}
|
||||
|
||||
export const isJellyfinFresh = (status: JellyfinSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
|
||||
|
||||
export type StateRow = typeof jellyfinState.$inferSelect;
|
||||
|
||||
const parse = <T>(text: string): T[] => {
|
||||
try {
|
||||
const v = JSON.parse(text);
|
||||
return Array.isArray(v) ? (v as T[]) : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
};
|
||||
export const streamsOf = (row: Pick<StateRow, "streams">) => parse<JellyfinStream>(row.streams);
|
||||
export const tasksOf = (row: Pick<StateRow, "tasks">) => parse<JellyfinTask>(row.tasks);
|
||||
|
||||
/** Writes down one read: the state, and a point in the history when it is time (or when the number streaming has changed). */
|
||||
export async function storeRead(integrationId: number, r: JellyfinRead, now: Date = new Date()): Promise<void> {
|
||||
const nowIso = now.toISOString();
|
||||
const values = {
|
||||
serverName: r.name,
|
||||
version: r.version,
|
||||
os: r.os,
|
||||
pendingRestart: r.pendingRestart,
|
||||
updateAvailable: r.updateAvailable,
|
||||
streams: JSON.stringify(r.streams),
|
||||
tasks: JSON.stringify(r.tasks),
|
||||
movies: r.counts.movies,
|
||||
series: r.counts.series,
|
||||
episodes: r.counts.episodes,
|
||||
songs: r.counts.songs,
|
||||
failedLogins24h: r.failedLogins24h,
|
||||
updatedAt: nowIso,
|
||||
};
|
||||
await db.insert(jellyfinState).values({ integrationId, ...values }).onConflictDoUpdate({ target: jellyfinState.integrationId, set: values });
|
||||
const transcodes = r.streams.filter((s) => s.transcoding).length;
|
||||
const [newest] = await db.select({ at: jellyfinReadings.at, streams: jellyfinReadings.streams, transcodes: jellyfinReadings.transcodes }).from(jellyfinReadings).where(eq(jellyfinReadings.integrationId, integrationId)).orderBy(desc(jellyfinReadings.at)).limit(1);
|
||||
const due = !newest || newest.streams !== r.streams.length || newest.transcodes !== transcodes || now.getTime() - Date.parse(newest.at) >= READING_GAP_MS;
|
||||
if (due) {
|
||||
await db.insert(jellyfinReadings).values({ integrationId, at: nowIso, streams: r.streams.length, transcodes });
|
||||
}
|
||||
await db.delete(jellyfinReadings).where(and(eq(jellyfinReadings.integrationId, integrationId), lt(jellyfinReadings.at, new Date(now.getTime() - READING_KEEP_DAYS * 86_400_000).toISOString())));
|
||||
}
|
||||
|
||||
/** Reads one Jellyfin and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
|
||||
export async function syncJellyfin(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<JellyfinRead>): Promise<JellyfinSyncStatus> {
|
||||
let status: JellyfinSyncStatus;
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
if (!loaded || loaded.integration.type !== "jellyfin") throw new Error("not a Jellyfin integration");
|
||||
const result = await (read ? read(loaded.config) : createJellyfinAdapter(loaded.config as never).read(now));
|
||||
await storeRead(integrationId, result, now);
|
||||
status = { at: now.toISOString(), ok: true, error: null };
|
||||
} catch (err) {
|
||||
const previous = (await readStatuses())[String(integrationId)];
|
||||
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
console.error(`[jellyfin] couldn't read integration ${integrationId}:`, status.error);
|
||||
}
|
||||
try {
|
||||
const all = await readStatuses();
|
||||
all[String(integrationId)] = status;
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
} catch (err) {
|
||||
console.error("[jellyfin] couldn't record the result:", err);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
export async function syncAllJellyfin(now: Date = new Date()): Promise<Record<number, JellyfinSyncStatus>> {
|
||||
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "jellyfin"), eq(integrations.enabled, true)));
|
||||
const out: Record<number, JellyfinSyncStatus> = {};
|
||||
for (const r of rows) out[r.id] = await syncJellyfin(r.id, now);
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function forgetJellyfin(integrationId: number): Promise<void> {
|
||||
const all = await readStatuses();
|
||||
if (String(integrationId) in all) {
|
||||
delete all[String(integrationId)];
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
}
|
||||
}
|
||||
|
||||
// ─── What is wrong ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface JellyfinProblem {
|
||||
key: string;
|
||||
message: string;
|
||||
severity: "warning";
|
||||
}
|
||||
|
||||
/** Pure: what is wrong with one Jellyfin, from what is stored for it. */
|
||||
export function jellyfinProblems(id: number, name: string, s: Pick<StateRow, "pendingRestart" | "tasks" | "failedLogins24h">): JellyfinProblem[] {
|
||||
const out: JellyfinProblem[] = [];
|
||||
if (s.pendingRestart) out.push({ key: `jellyfin-restart:${id}`, severity: "warning", message: `${name} is waiting to be restarted — a plugin or an update needs it` });
|
||||
for (const t of tasksOf(s)) {
|
||||
if (t.lastStatus === "Failed") out.push({ key: `jellyfin-task:${id}:${t.name}`, severity: "warning", message: `${name}: the scheduled task “${t.name}” failed${t.error ? ` — ${t.error}` : ""}` });
|
||||
}
|
||||
if (s.failedLogins24h !== null && s.failedLogins24h >= FAILED_LOGINS_WARN) out.push({ key: `jellyfin-logins:${id}`, severity: "warning", message: `${name}: ${s.failedLogins24h} sign-ins failed in the last day` });
|
||||
return out;
|
||||
}
|
||||
|
||||
// ─── Who is watching ────────────────────────────────────────────────────────
|
||||
|
||||
export interface Watching {
|
||||
integrationId: number;
|
||||
integrationName: string;
|
||||
streams: JellyfinStream[];
|
||||
}
|
||||
|
||||
/** Pure: a sentence about who is watching, for a warning or a waiting message: "2 people are streaming from Jellyfin (Ann: Dune, Bo: The Office — S1E2)". */
|
||||
export function watchingWords(w: Watching[]): string {
|
||||
const streams = w.flatMap((x) => x.streams);
|
||||
if (streams.length === 0) return "";
|
||||
const names = w.map((x) => x.integrationName);
|
||||
const who = streams.slice(0, 3).map((s) => `${s.user}: ${s.title}`).join(", ");
|
||||
const rest = streams.length > 3 ? ` and ${streams.length - 3} more` : "";
|
||||
const transcoding = streams.filter((s) => s.transcoding).length;
|
||||
return `${streams.length === 1 ? "1 person is" : `${streams.length} people are`} streaming from ${[...new Set(names)].join(" and ")} (${who}${rest})${transcoding > 0 ? `, ${transcoding} being transcoded` : ""}`;
|
||||
}
|
||||
|
||||
/** The Jellyfins that run on this server — the one whose address is the server's — and who is streaming from each, as last read and believed. Only those with somebody streaming are listed. */
|
||||
export async function watchingOnServer(serverId: number, now: Date = new Date()): Promise<Watching[]> {
|
||||
const rows = await db.select({ id: integrations.id, name: integrations.name, baseUrl: integrations.baseUrl }).from(integrations).where(and(eq(integrations.type, "jellyfin"), eq(integrations.enabled, true)));
|
||||
if (rows.length === 0) return [];
|
||||
const statuses = await jellyfinSyncStatuses();
|
||||
const serverRows = await db.select({ id: servers.id, name: servers.name, hostname: servers.hostname, ipAddresses: servers.ipAddresses }).from(servers);
|
||||
const matchable = serverRows.map((s) => {
|
||||
let ips: string[] = [];
|
||||
try {
|
||||
const v = JSON.parse(s.ipAddresses ?? "[]");
|
||||
ips = Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : [];
|
||||
} catch {
|
||||
// no addresses
|
||||
}
|
||||
return { id: s.id, name: s.name, hostname: s.hostname, ips };
|
||||
});
|
||||
const out: Watching[] = [];
|
||||
for (const r of rows) {
|
||||
if (!isJellyfinFresh(statuses[r.id], now.getTime())) continue;
|
||||
if (matchServerForTarget(hostOfAddress(r.baseUrl), matchable)?.id !== serverId) continue;
|
||||
const [state] = await db.select().from(jellyfinState).where(eq(jellyfinState.integrationId, r.id)).limit(1);
|
||||
const streams = state ? streamsOf(state) : [];
|
||||
if (streams.length > 0) out.push({ integrationId: r.id, integrationName: r.name, streams });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export interface PeakPoint {
|
||||
/** ISO. */
|
||||
t: string;
|
||||
/** The most that were streaming, and the most transcoded, in this stretch. */
|
||||
streams: number;
|
||||
transcodes: number;
|
||||
}
|
||||
|
||||
/** Pure: readings into at most `maxPoints` equal slices of [from, to], each as its peak — an average would hide the evening somebody watched. */
|
||||
export function bucketPeaks(rows: { at: string; streams: number; transcodes: number }[], from: number, to: number, maxPoints: number): PeakPoint[] {
|
||||
if (rows.length === 0 || to <= from) return [];
|
||||
const size = Math.max((to - from) / maxPoints, 1);
|
||||
const buckets = new Map<number, { tSum: number; n: number; streams: number; transcodes: number }>();
|
||||
for (const r of rows) {
|
||||
const t = Date.parse(r.at);
|
||||
if (t < from || t > to) continue;
|
||||
const key = Math.min(maxPoints - 1, Math.floor((t - from) / size));
|
||||
const b = buckets.get(key) ?? { tSum: 0, n: 0, streams: 0, transcodes: 0 };
|
||||
b.tSum += t;
|
||||
b.n += 1;
|
||||
b.streams = Math.max(b.streams, r.streams);
|
||||
b.transcodes = Math.max(b.transcodes, r.transcodes);
|
||||
buckets.set(key, b);
|
||||
}
|
||||
return [...buckets.entries()].sort((a, b) => a[0] - b[0]).map(([, b]) => ({ t: new Date(b.tSum / b.n).toISOString(), streams: b.streams, transcodes: b.transcodes }));
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initJellyfinSync(): void {
|
||||
if (timer) return;
|
||||
const pass = () => void syncAllJellyfin().catch((err) => console.error("[jellyfin] pass failed:", err));
|
||||
timer = setInterval(pass, SYNC_INTERVAL_MS);
|
||||
setTimeout(pass, 11_000);
|
||||
console.log(`[jellyfin] reading Jellyfin every ${SYNC_INTERVAL_MS / 60000} minutes`);
|
||||
}
|
||||
@@ -31,6 +31,8 @@ export function subjectOfConditionKey(key: string): string | null {
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^container-(?:restarting|unhealthy|stopped|created|host):(\d+):/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^jellyfin-(?:restart|task|logins):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^duplicati-(?:failed|stale):\d+:server:(\d+)$/.exec(key);
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:(\d+)$/.exec(key);
|
||||
|
||||
@@ -5,10 +5,11 @@
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState } from "../db/schema.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState } from "../db/schema.js";
|
||||
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
|
||||
import { levelStatuses } from "./serviceLevels.js";
|
||||
import { backupState, listAll as listDuplicati } from "./duplicati.js";
|
||||
import { isJellyfinFresh, jellyfinSyncStatuses, streamsOf } from "./jellyfinSync.js";
|
||||
import { latestTests } from "./restoreTests.js";
|
||||
import { heartbeatState, parseStamp } from "./heartbeats.js";
|
||||
import { env } from "../env.js";
|
||||
@@ -60,6 +61,21 @@ class Exposition {
|
||||
}
|
||||
}
|
||||
|
||||
/** Each Jellyfin, for /metrics: how many are streaming, how many of those are transcoded, and whether it wants a restart. Only while the reading is recent. */
|
||||
async function addJellyfinMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await jellyfinSyncStatuses();
|
||||
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "jellyfin"))).map((r) => [r.id, r]));
|
||||
for (const s of await db.select().from(jellyfinState)) {
|
||||
const row = names.get(s.integrationId);
|
||||
if (!row || !row.enabled || !isJellyfinFresh(statuses[s.integrationId], now.getTime())) continue;
|
||||
const labels = { name: row.name };
|
||||
const streams = streamsOf(s);
|
||||
out.add("homelab_jellyfin_streams", "gauge", "How many are streaming from Jellyfin right now.", streams.length, labels);
|
||||
out.add("homelab_jellyfin_transcodes", "gauge", "How many of those streams Jellyfin is transcoding.", streams.filter((x) => x.transcoding).length, labels);
|
||||
out.add("homelab_jellyfin_pending_restart", "gauge", "1 if Jellyfin is waiting to be restarted, else 0.", s.pendingRestart ? 1 : 0, labels);
|
||||
}
|
||||
}
|
||||
|
||||
/** Each Duplicati backup, for /metrics: whether it is in order, how long since it last succeeded, and how big what it backs up is. */
|
||||
async function addDuplicatiMetrics(out: Exposition, now: Date) {
|
||||
const { sources, backups } = await listDuplicati();
|
||||
@@ -267,6 +283,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
await addUpsMetrics(out, now);
|
||||
await addServiceLevelMetrics(out, now);
|
||||
await addDuplicatiMetrics(out, now);
|
||||
await addJellyfinMetrics(out, now);
|
||||
await addHeartbeatMetrics(out, now);
|
||||
return out.toString();
|
||||
}
|
||||
@@ -17,6 +17,7 @@ import { invalidateAlertsCache } from "./alerts.js";
|
||||
import { recordAudit } from "./audit.js";
|
||||
import { impactOf } from "./dependencies.js";
|
||||
import { startOrExtendWindow } from "./maintenance.js";
|
||||
import { watchingOnServer, watchingWords } from "./jellyfinSync.js";
|
||||
import { occurrencesBetween, type Occurrence } from "./maintenanceSchedules.js";
|
||||
import { notifyPatchPlanFinished } from "./notify.js";
|
||||
import { parseTags } from "./serverTags.js";
|
||||
@@ -36,8 +37,13 @@ export interface PlanItem {
|
||||
message: string | null;
|
||||
/** Windows made for what depends on this server, which end with it. */
|
||||
windowIds: number[];
|
||||
/** When its turn came and somebody was streaming from it, so it was left a while (ISO). */
|
||||
waitingSince?: string | null;
|
||||
}
|
||||
|
||||
/** How long a server is left, when it is its turn and people are streaming from it, before it is updated anyway. */
|
||||
export const WATCHING_WAIT_MS = 2 * 3_600_000;
|
||||
|
||||
/** How late a plan may still start, if the app was not running at its time. */
|
||||
export const START_GRACE_MS = 30 * 60_000;
|
||||
export const POLL_INTERVAL_MS = 20_000;
|
||||
@@ -217,6 +223,19 @@ async function advanceRun(run: PlanRunRow, now: Date): Promise<void> {
|
||||
}
|
||||
}
|
||||
const actor = `Patch plan "${plan.name}"`;
|
||||
// People streaming from it (Jellyfin on this machine): its turn waits, up to two hours, rather than cutting a film off.
|
||||
const watching = await watchingOnServer(server.id, now);
|
||||
let waited = "";
|
||||
if (watching.length > 0) {
|
||||
const since = next.waitingSince ? Date.parse(next.waitingSince) : now.getTime();
|
||||
if (!next.waitingSince) next.waitingSince = now.toISOString();
|
||||
if (now.getTime() - since < WATCHING_WAIT_MS) {
|
||||
next.message = `Waiting: ${watchingWords(watching)}.`;
|
||||
await saveRun(run, items);
|
||||
return; // looked at again at the next pass
|
||||
}
|
||||
waited = `Went ahead after waiting ${Math.round((now.getTime() - since) / 3_600_000)} h; ${watchingWords(watching)}. `;
|
||||
}
|
||||
try {
|
||||
const ur = await startUpdateRun(
|
||||
{
|
||||
@@ -227,10 +246,11 @@ async function advanceRun(run: PlanRunRow, now: Date): Promise<void> {
|
||||
limit: plan.limitFrom === "hostname" && server.hostname ? server.hostname : server.name,
|
||||
windowMinutes: plan.windowMinutes,
|
||||
actor,
|
||||
startAnyway: true,
|
||||
},
|
||||
now,
|
||||
);
|
||||
Object.assign(next, { status: "running", updateRunId: ur.id, message: "The update run has started.", windowIds: await silenceDependents(plan, server.id, server.name, actor, now) });
|
||||
Object.assign(next, { status: "running", updateRunId: ur.id, message: `${waited}The update run has started.`, windowIds: await silenceDependents(plan, server.id, server.name, actor, now) });
|
||||
await saveRun(run, items);
|
||||
return; // one at a time: this one is followed until it is over
|
||||
} catch (err) {
|
||||
|
||||
@@ -15,6 +15,7 @@ import { createSemaphoreAdapter, type SemaphoreAdapter, type SemaphoreTask } fro
|
||||
import { invalidateAlertsCache } from "./alerts.js";
|
||||
import { recordAudit } from "./audit.js";
|
||||
import { startOrExtendWindow } from "./maintenance.js";
|
||||
import { watchingOnServer, watchingWords } from "./jellyfinSync.js";
|
||||
import { notifyUpdateRunFinished } from "./notify.js";
|
||||
import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
|
||||
@@ -68,6 +69,8 @@ export interface StartInput {
|
||||
/** How long the maintenance window lasts at most; it ends sooner when the run does. */
|
||||
windowMinutes: number;
|
||||
actor: string;
|
||||
/** Go ahead although somebody is streaming from the machine (a person who has seen the warning, or a plan that has waited long enough). */
|
||||
startAnyway?: boolean;
|
||||
}
|
||||
|
||||
export async function startUpdateRun(input: StartInput, now: Date = new Date()): Promise<UpdateRunRow> {
|
||||
@@ -82,6 +85,12 @@ export async function startUpdateRun(input: StartInput, now: Date = new Date()):
|
||||
.limit(1);
|
||||
if (busy) throw new UpdateRunError(409, "already_running", `An update run for ${server.name} is already under way.`);
|
||||
|
||||
// Jellyfin on this machine, with people watching: the run may restart it. Said, and the person asked again, rather than cutting a film off.
|
||||
if (!input.startAnyway) {
|
||||
const watching = await watchingOnServer(server.id, now);
|
||||
if (watching.length > 0) throw new UpdateRunError(409, "watching", `${watchingWords(watching)}. Wait until they have finished, or start it anyway.`);
|
||||
}
|
||||
|
||||
const adapter = await semaphoreFor(input.integrationId);
|
||||
if (!adapter) throw new UpdateRunError(400, "no_semaphore", "That Semaphore integration doesn't exist or isn't switched on.");
|
||||
|
||||
|
||||
Reference in new issue
Block a user