Files
bobbanandClaude Sonnet 5.5 4ff0b44943 Nextcloud, NetBox and n8n integrations
Three read-only integrations, each with a monitoring page, alerts, maintenance
silencing and held-when-unreadable handling.

Nextcloud: database needing upgrade (critical), maintenance mode, and the data
disk running low; users, files and app updates with an admin app password.
NetBox: no background worker running, counts, and a list of the servers this
app manages that have no address in NetBox (a list, not an alert).
n8n: an active workflow whose last run failed, clearing when it next succeeds.

Disk messages for Immich, Paperless and Nextcloud now use the same units as
the pages that show them.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-24 23:13:13 +02:00

254 lines
15 KiB
TypeScript

/**
* Immich: reading what its API says (version, storage, statistics, job queues in either shape), what is stored, what is wrong, the health pass, the alerts, the page's route and /metrics.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { createServer as createHttpServer, type Server } from "node:http";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-immich-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, integrations, immichState } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const adapter = await import("../src/integrations/immich/adapter.js");
const im = await import("../src/services/immichSync.js");
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { immichRouter } = await import("../src/routes/immich.js");
const { buildMetrics } = await import("../src/services/metrics.js");
const { setInternalFlag } = await import("../src/services/settingsStore.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const MIN = 60_000;
const NOW = new Date("2026-10-20T12:00:00Z");
type Read = import("../src/integrations/immich/adapter.js").ImmichRead;
const queue = (name: string, failed = 0, extra: Partial<import("../src/integrations/immich/adapter.js").ImmichQueue> = {}) => ({ name, active: 0, waiting: 0, failed, paused: false, ...extra });
const read = (extra: Partial<Read> = {}): Read => ({ version: "v1.120.2", latestVersion: "v1.120.2", updateAvailable: false, disk: { size: 4e12, used: 1e12, percent: 25 }, photos: 52000, videos: 1800, usageBytes: 7e11, queues: [queue("thumbnailGeneration")], ...extra });
describe("what Immich's API says", () => {
test("a version can be a string or a major, minor and patch; and newer is told by number, not by text", () => {
assert.equal(adapter.parseVersion({ version: "v1.120.2" }), "v1.120.2");
assert.equal(adapter.parseVersion({ major: 1, minor: 120, patch: 2 }), "v1.120.2");
assert.equal(adapter.parseVersion({}), null);
assert.equal(adapter.newerVersion("v1.120.10", "v1.120.9"), true, "10 is later than 9");
assert.equal(adapter.newerVersion("1.120.2", "v1.120.2"), false);
assert.equal(adapter.newerVersion("v2.0.0", "v1.999.9"), true);
assert.equal(adapter.newerVersion(null, "v1.0.0"), false);
assert.equal(adapter.newerVersion("nightly", "v1.0.0"), false);
});
test("the disk under the library", () => {
assert.deepEqual(adapter.parseStorage({ diskSizeRaw: 4e12, diskUseRaw: 1e12, diskUsagePercentage: 25 }), { size: 4e12, used: 1e12, percent: 25 });
assert.equal(adapter.parseStorage({ diskSizeRaw: 1000, diskUseRaw: 333 })!.percent, 33.3, "worked out when Immich does not say");
assert.equal(adapter.parseStorage({}), null);
});
test("job queues: as an object keyed by name (/jobs) and as a list (/queues)", () => {
const asObject = adapter.parseQueues({ thumbnailGeneration: { jobCounts: { active: 1, waiting: 5, failed: 2 }, queueStatus: { isPaused: false } }, library: { jobCounts: { failed: 0 }, queueStatus: { isPaused: true } } })!;
assert.deepEqual(asObject.map((q) => [q.name, q.active, q.waiting, q.failed, q.paused]), [["thumbnailGeneration", 1, 5, 2, false], ["library", 0, 0, 0, true]]);
const asList = adapter.parseQueues([{ name: "metadataExtraction", isPaused: false, statistics: { failed: 7, waiting: 1 } }])!;
assert.deepEqual(asList.map((q) => [q.name, q.failed, q.waiting]), [["metadataExtraction", 7, 1]]);
assert.equal(adapter.parseQueues({}), null);
assert.equal(adapter.parseQueues("x"), null);
});
describe("the adapter against an Immich that answers", () => {
let server: Server;
let url = "";
let newStyle = false;
const seen: string[] = [];
before(async () => {
server = createHttpServer((req, res) => {
const path = (req.url ?? "").split("?")[0];
seen.push(`${req.headers["x-api-key"]} ${path}`);
const send = (code: number, body: unknown) => {
res.writeHead(code, { "content-type": "application/json" });
res.end(JSON.stringify(body));
};
if (req.headers["x-api-key"] !== "key") return send(401, {});
if (path === "/api/server/about") return send(200, { version: "v1.120.2" });
if (path === "/api/server/storage") return send(200, { diskSizeRaw: 4e12, diskUseRaw: 3.8e12, diskUsagePercentage: 95 });
if (path === "/api/server/statistics") return send(200, { photos: 100, videos: 5, usage: 123456 });
if (path === "/api/server/version-check") return send(200, { releaseVersion: "v1.121.0" });
if (path === "/api/jobs") return newStyle ? send(404, {}) : send(200, { thumbnailGeneration: { jobCounts: { failed: 4 } } });
if (path === "/api/queues") return newStyle ? send(200, [{ name: "thumbnailGeneration", statistics: { failed: 9 } }]) : send(404, {});
send(404, {});
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
url = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
});
after(() => server.close());
test("it reads everything, sends the key, and says a newer version is out", async () => {
const r = await adapter.createImmichAdapter({ url: `${url}/`, apiKey: "key" }).read();
assert.deepEqual([r.version, r.latestVersion, r.updateAvailable, r.photos, r.videos, r.usageBytes], ["v1.120.2", "v1.121.0", true, 100, 5, 123456]);
assert.deepEqual(r.disk, { size: 4e12, used: 3.8e12, percent: 95 });
assert.deepEqual(r.queues?.map((q) => [q.name, q.failed]), [["thumbnailGeneration", 4]]);
assert.ok(seen.every((s) => s.startsWith("key ")));
});
test("a newer Immich with /queues in place of /jobs is read all the same", async () => {
newStyle = true;
const r = await adapter.createImmichAdapter({ url, apiKey: "key" }).read();
assert.deepEqual(r.queues?.map((q) => [q.name, q.failed]), [["thumbnailGeneration", 9]]);
});
test("a refused key and a dead address are said plainly", async () => {
const bad = await adapter.createImmichAdapter({ url, apiKey: "nope" }).ping();
assert.equal(bad.ok, false);
assert.match(bad.error ?? "", /did not accept the API key/);
assert.equal((await adapter.createImmichAdapter({ url, apiKey: "key" }).ping()).ok, true);
assert.equal((await adapter.createImmichAdapter({ url: "http://127.0.0.1:1", apiKey: "key" }).ping()).ok, false);
});
});
});
describe("what is stored", () => {
let id = 0;
before(async () => {
id = (await db.insert(integrations).values({ type: "immich", name: "Photos", baseUrl: "http://10.9.9.3:2283", config: JSON.stringify({ url: "http://10.9.9.3:2283", apiKey: "key" }) }).returning({ id: integrations.id }))[0].id;
});
const state = async () => (await db.select().from(immichState).where(eq(immichState.integrationId, id)))[0];
test("the state is replaced at each read; queues that could not be read stay null, not empty", async () => {
await im.storeRead(id, read({ queues: [queue("a", 3)] }), NOW);
assert.deepEqual(im.queuesOf(await state())?.map((q) => q.name), ["a"]);
await im.storeRead(id, read({ queues: null, disk: null, photos: null }), NOW);
const s = await state();
assert.deepEqual([im.queuesOf(s), s.diskPercent, s.photos], [null, null, null]);
});
test("a read that failed is said, and what was stored is left alone", async () => {
const ok = await im.syncImmich(id, new Date(), async () => read({ version: "v1.121.0" }));
assert.equal(ok.ok, true);
const failed = await im.syncImmich(id, new Date(), async () => {
throw new Error("connection refused");
});
assert.deepEqual([failed.ok, failed.error], [false, "connection refused"]);
assert.equal(failed.at, ok.at);
assert.equal((await state()).version, "v1.121.0");
assert.equal(im.isImmichFresh((await im.immichSyncStatuses())[id], Date.now()), false);
});
});
describe("what is wrong", () => {
const row = (extra: Partial<Parameters<typeof im.immichProblems>[2]> = {}) => ({ diskSize: 4e12, diskUsed: 1e12, diskPercent: 25 as number | null, queues: JSON.stringify([queue("thumbnailGeneration")]) as string | null, ...extra });
const keys = (r: ReturnType<typeof row>) => im.immichProblems(1, "Photos", r).map((p) => `${p.key}=${p.severity}`);
test("nothing wrong says nothing; what could not be read is not a problem", () => {
assert.deepEqual(keys(row()), []);
assert.deepEqual(keys(row({ diskPercent: null, queues: null })), []);
});
test("the disk: a warning at 90 %, critical at 95 %", () => {
assert.deepEqual(keys(row({ diskPercent: 89.9 })), []);
assert.deepEqual(keys(row({ diskPercent: 90 })), ["immich-disk:1=warning"]);
assert.deepEqual(keys(row({ diskPercent: 95 })), ["immich-disk:1=critical"]);
assert.match(im.immichProblems(1, "Photos", row({ diskPercent: 96, diskUsed: 3.84e12 }))[0].message, /96 % full \(3576 GB of 3725 GB\)/);
});
test("failed jobs are one alert when there are ten or more together, naming the worst queues", () => {
const q = (a: number, b: number) => JSON.stringify([queue("thumbnailGeneration", a), queue("metadataExtraction", b), queue("library", 0)]);
assert.deepEqual(keys(row({ queues: q(5, 4) })), [], "nine is not yet");
const p = im.immichProblems(1, "Photos", row({ queues: q(8, 4) }));
assert.deepEqual(p.map((x) => [x.key, x.severity]), [["immich-jobs:1", "warning"]]);
assert.match(p[0].message, /12 background jobs have failed \(thumbnailGeneration 8, metadataExtraction 4\)/);
});
});
describe("the health pass, the page and /metrics", () => {
let imId = 0;
let app: Server;
let base = "";
const fresh = () => setInternalFlag("immichSync", JSON.stringify({ [imId]: { at: new Date().toISOString(), ok: true, error: null } }));
const stale = () => setInternalFlag("immichSync", JSON.stringify({ [imId]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null } }));
before(async () => {
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
imId = (await db.insert(integrations).values({ type: "immich", name: "Family photos", baseUrl: "http://10.9.9.2:2283", config: JSON.stringify({ url: "http://10.9.9.2:2283", apiKey: "key" }) }).returning({ id: integrations.id }))[0].id;
await im.storeRead(imId, read({ disk: { size: 4e12, used: 3.9e12, percent: 97.5 }, queues: [queue("thumbnailGeneration", 14, { waiting: 3 }), queue("library")] }), new Date());
await fresh();
const a = express();
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
next();
});
a.use("/api/immich", immichRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/immich`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("the health pass judges it from what is stored, and holds it when the last read is old", async () => {
let { snapshot, held } = await collectSnapshot();
assert.deepEqual(snapshot.immich?.map((i) => i.integrationId), [imId]);
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
const c = list.find((x) => x.key === `immich-disk:${imId}`)!;
assert.deepEqual([c.severity, c.source], ["critical", `immich:${imId}`]);
assert.equal(subjectOfConditionKey(c.key), `integration:${imId}`, "maintenance on it silences its alerts");
assert.equal(subjectOfConditionKey(`immich-jobs:${imId}`), `integration:${imId}`);
assert.ok(list.some((x) => x.key === `immich-jobs:${imId}`));
await stale();
({ snapshot, held } = await collectSnapshot());
assert.deepEqual([snapshot.immich ?? [], held.has(`immich:${imId}`)], [[], true]);
await fresh();
});
test("the Alerts page shows them: the disk under Disk, the jobs under Services, with where to look", async () => {
invalidateAlertsCache();
const report = await getAlerts(true);
const disk = report.alerts.find((x) => x.id === `disk:immich-disk:${imId}`)!;
assert.deepEqual([disk.severity, disk.category, disk.source, disk.link, disk.target?.type, disk.target?.id], ["critical", "disk", "Immich", "/immich", "integration", imId]);
assert.equal(report.alerts.find((x) => x.id === `services:immich-jobs:${imId}`)?.category, "services");
});
test("anyone signed in sees each Immich, the counts, the disk and the queues; an old reading says nothing is wrong", async () => {
const body = (await (await fetch(base)).json()) as { immiches: { id: number; fresh: boolean; state: { photos: number; disk: { percent: number } | null; queues: { name: string; failed: number }[] } | null; problems: unknown[] }[] };
const i = body.immiches.find((x) => x.id === imId)!;
assert.equal(i.fresh, true);
assert.deepEqual([i.state?.photos, i.state?.disk?.percent], [52000, 97.5]);
assert.deepEqual(i.state?.queues.map((q) => [q.name, q.failed]), [["thumbnailGeneration", 14], ["library", 0]]);
assert.equal(i.problems.length, 2);
await stale();
const old = (await (await fetch(base)).json()) as typeof body;
assert.deepEqual(old.immiches.find((x) => x.id === imId)!.problems, []);
await fresh();
});
test("/metrics has the counts, what the library takes, the disk and the failed jobs — only while the reading is recent", async () => {
let text = await buildMetrics(new Date());
assert.match(text, /homelab_immich_photos\{name="Family photos"\} 52000/);
assert.match(text, /homelab_immich_library_bytes\{name="Family photos"\} 7e\+11|homelab_immich_library_bytes\{name="Family photos"\} 700000000000/);
assert.match(text, /homelab_immich_disk_used_percent\{name="Family photos"\} 97\.5/);
assert.match(text, /homelab_immich_failed_jobs\{name="Family photos"\} 14/);
await stale();
text = await buildMetrics(new Date());
assert.doesNotMatch(text, /homelab_immich/);
});
});