From 7a5b623444c5ecb56e3c57c0821463ca45a98a2d Mon Sep 17 00:00:00 2001 From: Bobban Rydh Date: Sun, 18 Oct 2026 01:25:09 +0200 Subject: [PATCH] Stale-backup alerts, per-server alert limits, and update runs from the Updates page - Backups: a guest whose newest backup is older than its limit is an alert (backup kind), in the daily backup message, the digest and /metrics (homelab_backup_guests, ..._last_age_seconds) - Per-server thresholds (servers.thresholds, migration 0030): offline minutes, disk percent, forecast window, backup age, and per-disk limits or "leave alone"; applied everywhere the global ones were; saving clears the alerts cache - Update runs (update_runs, migration 0031): Update... on the Updates page puts the server under maintenance, runs a Semaphore playbook limited to it, waits for the host to report, and reports what is left and whether a restart is due; poller survives restarts - Notification toggle for update-run results; tests and docs Co-Authored-By: Claude Sonnet 5.5 --- API.md | 4 +- DATABASE.md | 29 +- NOTIFICATIONS.md | 2 + README.md | 34 +- ROLES.md | 4 +- server/drizzle/0030_icy_snowbird.sql | 1 + server/drizzle/0031_sad_lizard.sql | 28 + server/drizzle/meta/0030_snapshot.json | 2557 +++++++++++++++ server/drizzle/meta/0031_snapshot.json | 2754 +++++++++++++++++ server/drizzle/meta/_journal.json | 14 + server/src/app.ts | 2 + server/src/db/schema.ts | 39 + server/src/index.ts | 2 + server/src/routes/servers.ts | 29 +- server/src/routes/settings.ts | 3 +- server/src/routes/updateRuns.ts | 78 + server/src/services/alerts.ts | 30 +- server/src/services/backupCoverage.ts | 93 +- server/src/services/healthMonitor.ts | 11 +- server/src/services/metrics.ts | 13 +- server/src/services/notify.ts | 17 + server/src/services/proxmoxBackupScheduler.ts | 10 +- server/src/services/publicWall.ts | 2 +- server/src/services/serverActivity.ts | 13 + server/src/services/serverMetrics.ts | 11 +- server/src/services/serverSummary.ts | 5 +- server/src/services/serverThresholds.ts | 124 + server/src/services/settingsStore.ts | 7 +- server/src/services/topology.ts | 3 +- server/src/services/updateRuns.ts | 271 ++ server/src/services/weeklyDigest.ts | 14 +- server/test/serverThresholds.test.ts | 236 ++ server/test/staleBackups.test.ts | 185 ++ server/test/updateRuns.test.ts | 390 +++ web/src/App.tsx | 2 +- web/src/api/client.ts | 56 + web/src/components/ServerThresholds.tsx | 206 ++ web/src/components/UpdateRunDrawer.tsx | 157 + web/src/components/UpdateRunsCard.tsx | 127 + web/src/pages/ServerDetail.tsx | 2 + web/src/pages/Updates.tsx | 38 +- .../pages/settings/NotificationSettings.tsx | 15 +- 42 files changed, 7561 insertions(+), 57 deletions(-) create mode 100644 server/drizzle/0030_icy_snowbird.sql create mode 100644 server/drizzle/0031_sad_lizard.sql create mode 100644 server/drizzle/meta/0030_snapshot.json create mode 100644 server/drizzle/meta/0031_snapshot.json create mode 100644 server/src/routes/updateRuns.ts create mode 100644 server/src/services/serverThresholds.ts create mode 100644 server/src/services/updateRuns.ts create mode 100644 server/test/serverThresholds.test.ts create mode 100644 server/test/staleBackups.test.ts create mode 100644 server/test/updateRuns.test.ts create mode 100644 web/src/components/ServerThresholds.tsx create mode 100644 web/src/components/UpdateRunDrawer.tsx create mode 100644 web/src/components/UpdateRunsCard.tsx diff --git a/API.md b/API.md index 8838ca9..b816e9b 100644 --- a/API.md +++ b/API.md @@ -39,7 +39,7 @@ All gauges. Labels are escaped as Prometheus requires. | Series | Labels | Meaning | |---|---|---| | `homelab_info` | `environment` | Always 1; says which installation this is. | -| `homelab_server_up` | `server` | 1 if the agent has reported within the offline threshold (Settings → Notifications), else 0. Servers that never reported are left out. | +| `homelab_server_up` | `server` | 1 if the agent has reported within the offline threshold (Settings → Notifications, or the server's own limit if it has one), else 0. Servers that never reported are left out. | | `homelab_server_last_seen_timestamp_seconds` | `server` | When the agent last reported. | | `homelab_server_cpu_load_percent` | `server` | CPU load as the agent reports it. | | `homelab_server_memory_used_percent` | `server` | Memory in use. | @@ -62,6 +62,8 @@ All gauges. Labels are escaped as Prometheus requires. | `homelab_maintenance_active_windows` | | Maintenance windows running now. | | `homelab_secret_expiry_days` | `name` | Days until a secret expires (negative once it has). | | `homelab_inventory_power_watts` | | Typical draw of everything in the inventory. | +| `homelab_backup_guests` | `status` | Proxmox guests by how well they are backed up: `ok`, `stale`, `failed`, `none`, `unknown`. From what the last backup check read; a scrape never goes to Proxmox or Backup Server itself, and the series is left out until something has been read. | +| `homelab_backup_last_age_seconds` | `guest`, `vmid`, `integration` | How long ago a guest's newest backup was. | | `homelab_scrape_duration_seconds` | | How long building the answer took. | | `homelab_scrape_timestamp_seconds` | | When the answer was built. | diff --git a/DATABASE.md b/DATABASE.md index 6a7c5a3..73092f2 100644 --- a/DATABASE.md +++ b/DATABASE.md @@ -23,8 +23,8 @@ what the app actually creates. |---|---| | **File** | `DATABASE_PATH`, default `../data/homelab-manager.sqlite` (relative to `server/`; `/app/data/...` in Docker). The folder is created if missing. | | **Foreign keys** | Switched on for every connection (`PRAGMA foreign_keys = ON`), so the `ON DELETE` rules below are enforced. | -| **Migrations** | SQL files in [`server/drizzle/`](server/drizzle) (`0000` … `0029`), applied automatically when the server starts. Which ones have run is recorded in the table `__drizzle_migrations`. | -| **Tables** | 30 application tables, plus drizzle's own `__drizzle_migrations`. | +| **Migrations** | SQL files in [`server/drizzle/`](server/drizzle) (`0000` … `0031`), applied automatically when the server starts. Which ones have run is recorded in the table `__drizzle_migrations`. | +| **Tables** | 31 application tables, plus drizzle's own `__drizzle_migrations`. | **Not in the database:** @@ -101,7 +101,7 @@ Who changed what. Written by the app on every change (see | `id` | integer PK | | | `actor_user_id` | integer → `users.id`, **set null** on delete | Null for automatic actions, and for entries whose user was deleted. | | `actor_label` | text | The user's name/email as it was at the time (or `system`), so an entry still reads correctly after the account is gone. | -| `category` | text NOT NULL | Free text. In use: `server` (including the health check's own `went_offline`, `back_online`, `disk_full` and `disk_ok`, and `agent_updated` when a server's agent reports a higher version than before, written as `system` and shown on a server's Activity card), `integration` (including the health check's `went_failing` / `recovered` for a kind of service, with `target_type` `integration_type` and the service type as `target_id`, which the dashboard summary and the uptime ribbons read), `dns`, `settings` (including `agent_auto_update`), `ipam`, `secret`, `domain`, `tag`, `task`, `session`, `network`, `maintenance`, `consistency`, `user`, `privacy`, `diag_log`, `alert_ticket`, `share_link` (a link made or removed — never the secret), `runbook`, `inventory`, `metrics_token` and `webhook` (made, changed or removed — never a secret or an address). `user` also records `role_from_groups` when a sign-in changed someone's role. | +| `category` | text NOT NULL | Free text. In use: `server` (including the health check's own `went_offline`, `back_online`, `disk_full` and `disk_ok`, and `agent_updated` when a server's agent reports a higher version than before, written as `system` and shown on a server's Activity card), `integration` (including the health check's `went_failing` / `recovered` for a kind of service, with `target_type` `integration_type` and the service type as `target_id`, which the dashboard summary and the uptime ribbons read), `dns`, `settings` (including `agent_auto_update`), `ipam`, `secret`, `domain`, `tag`, `task`, `session`, `network`, `maintenance`, `consistency`, `user`, `privacy`, `diag_log`, `alert_ticket`, `share_link` (a link made or removed — never the secret), `runbook`, `update_run` (`start`, `finished`, `failed`, `cancelled` — shown on the server's Activity card), `inventory`, `metrics_token` and `webhook` (made, changed or removed — never a secret or an address). `user` also records `role_from_groups` when a sign-in changed someone's role. | | `action` | text NOT NULL | `create`, `update`, `delete`, `start`, `stop`, … — free text. | | `target_type`, `target_id` | text | What it happened to. `target_id` is text so it can hold any kind of ID; there's no foreign key. | | `detail` | text | JSON, free-form context (what changed, names, counts). Secrets are never put here. | @@ -219,6 +219,7 @@ what it last reported. | `no_parent` | bool NOT NULL, default 0 | An admin said this server runs on nothing: no link, and no guess from its Proxmox node. Choosing a server by hand, or going back to automatic, clears it. | | `pending_updates` | text | JSON `{"manager":"apt","count":5,"security":2,"rebootRequired":false,"reportedAt":"…"}` from the agent; null until it has said. `security` is null when the package manager can't tell. | | `updates_pending_since` | text | ISO; when updates were first seen waiting (and none were before). Null when none are waiting. | +| `thresholds` | text | This server's own alert limits as JSON: `{"serverOfflineMinutes":…,"diskUsagePercent":…,"diskForecastDays":…,"backupStaleHours":…,"mounts":{"/data":{"percent":95,"ignore":true}}}`. Anything left out follows Settings; null when it follows Settings entirely. Ranges are checked on the way in (offline 15–10080 min, disk 50–99 %, forecast 0–90 days, backup 0–2160 h; 0 means off). | | `host_health` | text | JSON `{"failedUnits":[{unit,description}],"smart":[{device,model,healthy,temperatureC,wearPercent}],"zfs":[{name,health,capacityPercent}],"reportedAt":"…"}`. Each part is present only if the agent reported it, and a report that leaves one out leaves it as it was. Null until an agent says anything. Judged by the health check and shown on the server's page. | | `agent_version` | integer | Which version of the agent script runs on it, as the agent says in each report. Null until it says; an agent that predates the number counts as version 1. Compared with the scripts the app serves to say which agents are behind. | @@ -489,6 +490,26 @@ checks see it. A row is one *occurrence*: when an alert clears and later returns | `ticketed_at` | text | ISO; when the ticket was opened. | | `cleared_at` | text | ISO; null while the alert is still active. Rows cleared more than 180 days ago are deleted. | +#### `update_runs` + +A playbook run in Semaphore for one server, wrapped in a maintenance window and followed until the server reports again. + +| Column | Type | Notes | +|---|---|---| +| `id` | integer PK | | +| `server_id` | integer NOT NULL | → `servers.id`, deleted with the server. | +| `integration_id`, `project_id`, `template_id` | integer NOT NULL | Which Semaphore, project and template. `integration_id` is a plain reference, **not a foreign key**, so history stays readable when an integration is removed. | +| `template_name`, `host_limit` | text NOT NULL | The template's name and ansible's `--limit` it was run with (never empty: an update run is for one server). | +| `task_id`, `task_status` | integer, text | Semaphore's run number and its own word for how it is going. | +| `window_id`, `window_created` | integer, bool | The maintenance window put round it, and whether this run made it (a window that was already there is extended and left to run out). | +| `status` | text NOT NULL | `running`, `waiting_for_host`, `done`, `failed` or `cancelled`. | +| `message` | text | What it says now, in words. | +| `started_by` | text | | +| `started_at`, `task_ended_at`, `host_reported_at`, `finished_at` | text | ISO. | +| `updates_before`, `updates_after` | integer | The waiting count before, and from the report that came after. | +| `reboot_required` | bool | From that report. | +| `poll_errors` | integer | Consecutive times Semaphore didn't answer (ten ends the run). | + #### `runbooks` Notes about what to do when something is wrong, attached to a server, integration or DNS provider. @@ -605,7 +626,7 @@ that were never changed aren't stored; the app fills in defaults when reading. | `agents` | `autoUpdate` (false): whether agents that are behind replace themselves. | | `inventory` | `currency` (a short name, may be blank) and `pricePerKwh` (null) — what the inventory's running cost is worked out with. | | `digest` | The weekly digest: `enabled` (false), `day` (0 = Sunday … 6; default 1, Monday) and `time` (`HH:MM`, default `08:00`), in the notification time zone. | -| `healthChecks` | `serverOfflineMinutes` (60), `diskUsagePercent` (90), `domainWarnDays` (30), `diskForecastDays` (14; 0 = off — how far ahead a filling disk is flagged on the Alerts page). | +| `healthChecks` | `serverOfflineMinutes` (60), `diskUsagePercent` (90), `domainWarnDays` (30), `diskForecastDays` (14; 0 = off — how far ahead a filling disk is flagged on the Alerts page), `backupStaleHours` (48; 0 = off — how old a Proxmox guest's newest backup may be). A server can override each of these (see `servers.thresholds`). | | `logRetention` | `enabled`, `retentionDays` (90), `intervalHours` (24). | | `display` | `dateFormat`, `timeFormat`, `pageSize`. | | `providerColors`, `integrationColors` | Badge colours, by provider / integration type. | diff --git a/NOTIFICATIONS.md b/NOTIFICATIONS.md index 68ec5d7..6112976 100644 --- a/NOTIFICATIONS.md +++ b/NOTIFICATIONS.md @@ -53,6 +53,8 @@ until the next scheduled time to catch up. | Domain expiry | *Domain registration expiring or expired (daily reminder)* | Any tracked domain registration is expired or within the warning window; every domain is re-looked-up against its registry first. A domain whose registry lookup has been failing for 3+ days is also mentioned, separately, as "may be stale." Registries that don't publish an expiry (e.g. `.de`, `.eu`) are tracked but never trigger this. | Settings → Notifications → Health checks → **Domain expiry warning** (default 30 days) | | Tailscale key expiry | *Tailscale key expiry reminder* | Any device's node key (across every enabled Tailscale integration) is within 30 days of expiring, or already expired. Devices with key expiry disabled are skipped. | Fixed at 30 days | | Docker image updates | *Docker image update available* | Any container (across every enabled Dockhand integration) has an image update available, per Dockhand's own cached update-check results — this doesn't trigger a fresh registry lookup, just reads what the Docker page itself would show. | Not configurable (reflects Dockhand's own check interval) | +| Proxmox backups — gone stale | *(same toggle as the Proxmox backup check)* | Once a day with the other backup messages: a guest whose newest backup (on Proxmox or a Backup Server) is older than the limit. Guests under maintenance (on the server or the Proxmox integration) are left out. | Settings → Notifications → **A guest's backup is too old after** (default 48 h; 0 off); a server can have its own limit on its page | +| Update run finished | *An update run you started finished, or didn't* | An update run started from the Updates page ended: what is still waiting and whether a restart is due, or why it failed. Sent at once (not held for the daily check). | Settings → Notifications → What to send | | Proxmox backups | *Proxmox backup failed or a guest has no coverage* | Two independent conditions, both under this one toggle: (1) a node's most recent `vzdump` backup task (across every enabled Proxmox integration) didn't succeed; (2) a VM/LXC isn't covered by any enabled backup job at all. | Not configurable | | Proxmox Backup Server verification | *Proxmox Backup Server snapshot failed verification* | Across every enabled PBS integration: any datastore has at least one stored snapshot whose verification state is "failed", or a datastore couldn't be read at all (e.g. a permissions problem). This is distinct from the Proxmox check above — PVE only knows a backup *ran*, PBS is the only place that knows whether the stored data still verifies. | Not configurable | diff --git a/README.md b/README.md index 2327870..de225a6 100644 --- a/README.md +++ b/README.md @@ -447,8 +447,20 @@ installed or refreshed — that one question is asked outside the agent's sandbo hours, cached under `%ProgramData%\HomelabManager`). A server whose agent hasn't said — an older agent, or another package manager — shows "not reported", never "up to date"; run its install command again to update its agent (how, and how to -tell it worked: [AGENT_UPDATE.md](AGENT_UPDATE.md)). It only looks: to apply -updates, run a playbook. +tell it worked: [AGENT_UPDATE.md](AGENT_UPDATE.md)). + +**Update runs** (Updates → Update…, operators). Apply a server's waiting updates in +one go: pick the Semaphore playbook that does it, and Homelab Manager puts the server +under maintenance (so nothing alerts while it works and restarts), runs the playbook +limited to that one server, follows it, waits for the server's agent to report again, +and then says how many updates are still waiting and whether a restart is due before +ending the window it made. The runs are listed on the Updates page and in the server's +history, survive a restart of this app, and end as *failed* (with the window ended, so +the alerts can speak) if the playbook fails, Semaphore stops answering, the server +doesn't report again within 30 minutes, or it takes more than three hours. "Stop +following" ends the window and stops watching; it can't stop a playbook already running +in Semaphore. A message goes to your channels when a run ends (a toggle under *What to +send*). It never restarts the server itself — the playbook does, if it is written to. **Host health and automatic agent updates.** The Linux agent (version 4) also reports services that have failed, each disk's SMART verdict, temperature and wear, and ZFS pool state. They show on a server's page @@ -476,7 +488,23 @@ backup task that names the guest), and did that go well — Recent backup, Stale than 1, 2, 3, 7 or 14 days, your choice), Failed, No backup, or Unknown (a job covers it but nothing says when it last ran, typical when backups go somewhere other than a Backup Server — never guessed at). Worst first, searchable, with CSV, and the weekly -digest lists what needs a look. +digest lists what needs a look. A guest whose newest backup is older than **Settings → +Notifications → "A guest's backup is too old after"** (48 hours by default; 0 is off) is +also an alert of the Backups kind — a warning, critical at three times the limit — with a +runbook link like any other, sent once a day with the other backup messages (under the +Proxmox backup toggle) and counted in `/metrics` (`homelab_backup_guests`, +`homelab_backup_last_age_seconds`). A server that is a guest can have its own limit +(a weekly backup is fine at eight days). Failed backups already raise their own alerts — +the latest failing run on a node, and snapshots that fail verification — so those aren't +repeated per guest. The page's own "Stale after" selector is only a view. + +**Alert limits per server.** On a server's page an administrator can give it limits of +its own — offline after, disk usage percentage, the disk-forecast window, and (for a +Proxmox guest) how old its backup may be — and per disk, either a limit or "no alerts +for this disk". Whatever is left empty follows Settings, so changing the global limit +still reaches every server that hasn't said otherwise. They apply everywhere the global +ones did: the alerts, the notifications, the dashboard, the map, the forecast, the +digest and `/metrics`. **Inventory** (Infrastructure → Inventory) keeps the hardware behind it all: name and kind, maker, model, serial, where it sits (and rack unit), when it was bought and for diff --git a/ROLES.md b/ROLES.md index dbf65f8..89fa9e4 100644 --- a/ROLES.md +++ b/ROLES.md @@ -93,7 +93,9 @@ even further, to admin only: next to an alert); writing, editing and removing one needs operator. - **Inventory**: everyone can see the hardware; adding, editing and removing needs operator; the price of power used for the running cost is admin-only. -- **Backup coverage** and **Updates** are read-only pages for everyone. +- **Backup coverage** is a read-only page for everyone. On **Updates**, everyone sees what is waiting and how update runs + are going; starting an update run (**Update…**) or stopping one needs operator. +- **Alert limits** on a server's page: everyone can see the ones a server has of its own; only an administrator changes them. - **Host health** (failed services, disk health, ZFS pools) is shown to everyone on a server's page; **Update agents automatically** on the Servers page is admin-only. - **Sign-in groups, metrics tokens, event webhooks and share links** are all under diff --git a/server/drizzle/0030_icy_snowbird.sql b/server/drizzle/0030_icy_snowbird.sql new file mode 100644 index 0000000..9c09e1b --- /dev/null +++ b/server/drizzle/0030_icy_snowbird.sql @@ -0,0 +1 @@ +ALTER TABLE `servers` ADD `thresholds` text; \ No newline at end of file diff --git a/server/drizzle/0031_sad_lizard.sql b/server/drizzle/0031_sad_lizard.sql new file mode 100644 index 0000000..ce09c5e --- /dev/null +++ b/server/drizzle/0031_sad_lizard.sql @@ -0,0 +1,28 @@ +CREATE TABLE `update_runs` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `server_id` integer NOT NULL, + `integration_id` integer NOT NULL, + `project_id` integer NOT NULL, + `template_id` integer NOT NULL, + `template_name` text NOT NULL, + `host_limit` text NOT NULL, + `task_id` integer, + `task_status` text, + `window_id` integer, + `window_created` integer DEFAULT false NOT NULL, + `status` text NOT NULL, + `message` text, + `started_by` text, + `started_at` text NOT NULL, + `task_ended_at` text, + `host_reported_at` text, + `finished_at` text, + `updates_before` integer, + `updates_after` integer, + `reboot_required` integer DEFAULT false NOT NULL, + `poll_errors` integer DEFAULT 0 NOT NULL, + FOREIGN KEY (`server_id`) REFERENCES `servers`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE INDEX `update_runs_server_idx` ON `update_runs` (`server_id`,`started_at`);--> statement-breakpoint +CREATE INDEX `update_runs_status_idx` ON `update_runs` (`status`); \ No newline at end of file diff --git a/server/drizzle/meta/0030_snapshot.json b/server/drizzle/meta/0030_snapshot.json new file mode 100644 index 0000000..08957d4 --- /dev/null +++ b/server/drizzle/meta/0030_snapshot.json @@ -0,0 +1,2557 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "a678860b-939e-4044-bc22-148acc221807", + "prevId": "5f6ad4ad-471e-4f48-9dc1-0b504657b1af", + "tables": { + "alert_tickets": { + "name": "alert_tickets", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "alert_key": { + "name": "alert_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "seen_count": { + "name": "seen_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "integration_id": { + "name": "integration_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ticket_number": { + "name": "ticket_number", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "grouped": { + "name": "grouped", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "first_seen_at": { + "name": "first_seen_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ticketed_at": { + "name": "ticketed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cleared_at": { + "name": "cleared_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "alert_tickets_key_idx": { + "name": "alert_tickets_key_idx", + "columns": [ + "alert_key" + ], + "isUnique": false + } + }, + "foreignKeys": { + "alert_tickets_integration_id_integrations_id_fk": { + "name": "alert_tickets_integration_id_integrations_id_fk", + "tableFrom": "alert_tickets", + "tableTo": "integrations", + "columnsFrom": [ + "integration_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "assets": { + "name": "assets", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'other'" + }, + "manufacturer": { + "name": "manufacturer", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "serial": { + "name": "serial", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "purchase_date": { + "name": "purchase_date", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "purchase_price": { + "name": "purchase_price", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "warranty_until": { + "name": "warranty_until", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "location": { + "name": "location", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "rack_unit": { + "name": "rack_unit", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "power_watts": { + "name": "power_watts", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "server_id": { + "name": "server_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "audit_log": { + "name": "audit_log", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "actor_label": { + "name": "actor_label", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "category": { + "name": "category", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "detail": { + "name": "detail", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": { + "audit_log_actor_user_id_users_id_fk": { + "name": "audit_log_actor_user_id_users_id_fk", + "tableFrom": "audit_log", + "tableTo": "users", + "columnsFrom": [ + "actor_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "consistency_ignores": { + "name": "consistency_ignores", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": { + "consistency_ignores_key_unique": { + "name": "consistency_ignores_key_unique", + "columns": [ + "key" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "diag_log": { + "name": "diag_log", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "operation": { + "name": "operation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ok": { + "name": "ok", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "latency_ms": { + "name": "latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "dns_providers": { + "name": "dns_providers", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "provider_type": { + "name": "provider_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "credential_id": { + "name": "credential_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "config": { + "name": "config", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": { + "dns_providers_credential_id_integration_credentials_id_fk": { + "name": "dns_providers_credential_id_integration_credentials_id_fk", + "tableFrom": "dns_providers", + "tableTo": "integration_credentials", + "columnsFrom": [ + "credential_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "dns_records_cache": { + "name": "dns_records_cache", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "provider_id": { + "name": "provider_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "zone_id": { + "name": "zone_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_id": { + "name": "record_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ttl": { + "name": "ttl", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "proxied": { + "name": "proxied", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "dns_records_cache_provider_id_dns_providers_id_fk": { + "name": "dns_records_cache_provider_id_dns_providers_id_fk", + "tableFrom": "dns_records_cache", + "tableTo": "dns_providers", + "columnsFrom": [ + "provider_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "dns_zones_cache": { + "name": "dns_zones_cache", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "provider_id": { + "name": "provider_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "zone_id": { + "name": "zone_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "zone_name": { + "name": "zone_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "synced_at": { + "name": "synced_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "dns_zones_cache_provider_zone_idx": { + "name": "dns_zones_cache_provider_zone_idx", + "columns": [ + "provider_id", + "zone_id" + ], + "isUnique": true + } + }, + "foreignKeys": { + "dns_zones_cache_provider_id_dns_providers_id_fk": { + "name": "dns_zones_cache_provider_id_dns_providers_id_fk", + "tableFrom": "dns_zones_cache", + "tableTo": "dns_providers", + "columnsFrom": [ + "provider_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "domains": { + "name": "domains", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'manual'" + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "registrar": { + "name": "registrar", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lookup_source": { + "name": "lookup_source", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_checked_at": { + "name": "last_checked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_checked_ok_at": { + "name": "last_checked_ok_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_check_error": { + "name": "last_check_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": { + "domains_name_unique": { + "name": "domains_name_unique", + "columns": [ + "name" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "event_webhooks": { + "name": "event_webhooks", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "events": { + "name": "events", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "last_attempt_at": { + "name": "last_attempt_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_result": { + "name": "last_result", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "integration_credentials": { + "name": "integration_credentials", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "encrypted_secret": { + "name": "encrypted_secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "integrations": { + "name": "integrations", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_url": { + "name": "base_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "credential_id": { + "name": "credential_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "config": { + "name": "config", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": { + "integrations_credential_id_integration_credentials_id_fk": { + "name": "integrations_credential_id_integration_credentials_id_fk", + "tableFrom": "integrations", + "tableTo": "integration_credentials", + "columnsFrom": [ + "credential_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ipam_entries": { + "name": "ipam_entries", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "vendor": { + "name": "vendor", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "location": { + "name": "location", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": { + "ipam_entries_ip_address_unique": { + "name": "ipam_entries_ip_address_unique", + "columns": [ + "ip_address" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "maintenance_aliases": { + "name": "maintenance_aliases", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_id": { + "name": "target_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": { + "maintenance_aliases_unique_idx": { + "name": "maintenance_aliases_unique_idx", + "columns": [ + "address", + "target_type", + "target_id" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "maintenance_schedules": { + "name": "maintenance_schedules", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "targets": { + "name": "targets", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "days": { + "name": "days", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "start_time": { + "name": "start_time", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "duration_minutes": { + "name": "duration_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "timezone": { + "name": "timezone", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "maintenance_windows": { + "name": "maintenance_windows", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_id": { + "name": "target_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ends_at": { + "name": "ends_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "metrics_tokens": { + "name": "metrics_tokens", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_prefix": { + "name": "token_prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "last_used_at": { + "name": "last_used_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "metrics_tokens_token_hash_unique": { + "name": "metrics_tokens_token_hash_unique", + "columns": [ + "token_hash" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "notification_queue": { + "name": "notification_queue", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "port_forwards": { + "name": "port_forwards", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "external_port": { + "name": "external_port", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "protocol": { + "name": "protocol", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'tcp'" + }, + "server_id": { + "name": "server_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "destination": { + "name": "destination", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "internal_port": { + "name": "internal_port", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "comment": { + "name": "comment", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": { + "port_forwards_server_id_servers_id_fk": { + "name": "port_forwards_server_id_servers_id_fk", + "tableFrom": "port_forwards", + "tableTo": "servers", + "columnsFrom": [ + "server_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "runbooks": { + "name": "runbooks", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_id": { + "name": "target_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "category": { + "name": "category", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "runbooks_target_idx": { + "name": "runbooks_target_idx", + "columns": [ + "target_type", + "target_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "scheduled_tasks": { + "name": "scheduled_tasks", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "server_id": { + "name": "server_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "schedule_type": { + "name": "schedule_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'agent'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "command": { + "name": "command", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "schedule_expression": { + "name": "schedule_expression", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "next_run_at": { + "name": "next_run_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "raw_metadata": { + "name": "raw_metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_stale": { + "name": "is_stale", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "first_seen_at": { + "name": "first_seen_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": { + "scheduled_tasks_server_id_servers_id_fk": { + "name": "scheduled_tasks_server_id_servers_id_fk", + "tableFrom": "scheduled_tasks", + "tableTo": "servers", + "columnsFrom": [ + "server_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "secrets": { + "name": "secrets", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'generic'" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expiry_date": { + "name": "expiry_date", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "warn_days": { + "name": "warn_days", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 30 + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "check_host": { + "name": "check_host", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "check_port": { + "name": "check_port", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_checked_at": { + "name": "last_checked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_check_error": { + "name": "last_check_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "server_links": { + "name": "server_links", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "server_id": { + "name": "server_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": { + "server_links_server_id_servers_id_fk": { + "name": "server_links_server_id_servers_id_fk", + "tableFrom": "server_links", + "tableTo": "servers", + "columnsFrom": [ + "server_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "server_metrics": { + "name": "server_metrics", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "server_id": { + "name": "server_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metric": { + "name": "metric", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "recorded_at": { + "name": "recorded_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "server_metrics_lookup_idx": { + "name": "server_metrics_lookup_idx", + "columns": [ + "server_id", + "metric", + "recorded_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "server_metrics_server_id_servers_id_fk": { + "name": "server_metrics_server_id_servers_id_fk", + "tableFrom": "server_metrics", + "tableTo": "servers", + "columnsFrom": [ + "server_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "server_ports": { + "name": "server_ports", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "server_id": { + "name": "server_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "port": { + "name": "port", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "protocol": { + "name": "protocol", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'tcp'" + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "comment": { + "name": "comment", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "open": { + "name": "open", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "last_seen_open_at": { + "name": "last_seen_open_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": { + "server_ports_unique": { + "name": "server_ports_unique", + "columns": [ + "server_id", + "port", + "protocol" + ], + "isUnique": true + } + }, + "foreignKeys": { + "server_ports_server_id_servers_id_fk": { + "name": "server_ports_server_id_servers_id_fk", + "tableFrom": "server_ports", + "tableTo": "servers", + "columnsFrom": [ + "server_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "servers": { + "name": "servers", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hostname": { + "name": "hostname", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "os_type": { + "name": "os_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'linux'" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "api_token_hash": { + "name": "api_token_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "api_token_prefix": { + "name": "api_token_prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ip_addresses": { + "name": "ip_addresses", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cpu_model": { + "name": "cpu_model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cpu_cores": { + "name": "cpu_cores", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cpu_load_percent": { + "name": "cpu_load_percent", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mem_total_bytes": { + "name": "mem_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mem_used_bytes": { + "name": "mem_used_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disks": { + "name": "disks", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "listening_ports": { + "name": "listening_ports", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_port_scan": { + "name": "last_port_scan", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tags": { + "name": "tags", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "proxmox_integration_id": { + "name": "proxmox_integration_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "proxmox_node": { + "name": "proxmox_node", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "proxmox_guest_type": { + "name": "proxmox_guest_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "proxmox_vmid": { + "name": "proxmox_vmid", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "hide_proxmox_link": { + "name": "hide_proxmox_link", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "parent_server_id": { + "name": "parent_server_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "no_parent": { + "name": "no_parent", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "pending_updates": { + "name": "pending_updates", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updates_pending_since": { + "name": "updates_pending_since", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "host_health": { + "name": "host_health", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "thresholds": { + "name": "thresholds", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "agent_version": { + "name": "agent_version", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "servers_proxmox_integration_id_integrations_id_fk": { + "name": "servers_proxmox_integration_id_integrations_id_fk", + "tableFrom": "servers", + "tableTo": "integrations", + "columnsFrom": [ + "proxmox_integration_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "settings": { + "name": "settings", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "share_links": { + "name": "share_links", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_prefix": { + "name": "token_prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "detail": { + "name": "detail", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'summary'" + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "share_links_token_hash_unique": { + "name": "share_links_token_hash_unique", + "columns": [ + "token_hash" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "tag_definitions": { + "name": "tag_definitions", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "color": { + "name": "color", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": { + "tag_definitions_name_unique": { + "name": "tag_definitions_name_unique", + "columns": [ + "name" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "oidc_sub": { + "name": "oidc_sub", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'viewer'" + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "last_login_at": { + "name": "last_login_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "favorites": { + "name": "favorites", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_looked_at": { + "name": "last_looked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "groups": { + "name": "groups", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dashboard_layout": { + "name": "dashboard_layout", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "users_oidc_sub_unique": { + "name": "users_oidc_sub_unique", + "columns": [ + "oidc_sub" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/server/drizzle/meta/0031_snapshot.json b/server/drizzle/meta/0031_snapshot.json new file mode 100644 index 0000000..325c04f --- /dev/null +++ b/server/drizzle/meta/0031_snapshot.json @@ -0,0 +1,2754 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "f8e70de6-9a27-4382-b3ff-ebf54dfc76e3", + "prevId": "a678860b-939e-4044-bc22-148acc221807", + "tables": { + "alert_tickets": { + "name": "alert_tickets", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "alert_key": { + "name": "alert_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "seen_count": { + "name": "seen_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "integration_id": { + "name": "integration_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ticket_number": { + "name": "ticket_number", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "grouped": { + "name": "grouped", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "first_seen_at": { + "name": "first_seen_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ticketed_at": { + "name": "ticketed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cleared_at": { + "name": "cleared_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "alert_tickets_key_idx": { + "name": "alert_tickets_key_idx", + "columns": [ + "alert_key" + ], + "isUnique": false + } + }, + "foreignKeys": { + "alert_tickets_integration_id_integrations_id_fk": { + "name": "alert_tickets_integration_id_integrations_id_fk", + "tableFrom": "alert_tickets", + "tableTo": "integrations", + "columnsFrom": [ + "integration_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "assets": { + "name": "assets", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'other'" + }, + "manufacturer": { + "name": "manufacturer", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "serial": { + "name": "serial", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "purchase_date": { + "name": "purchase_date", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "purchase_price": { + "name": "purchase_price", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "warranty_until": { + "name": "warranty_until", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "location": { + "name": "location", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "rack_unit": { + "name": "rack_unit", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "power_watts": { + "name": "power_watts", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "server_id": { + "name": "server_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "audit_log": { + "name": "audit_log", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "actor_label": { + "name": "actor_label", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "category": { + "name": "category", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "detail": { + "name": "detail", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": { + "audit_log_actor_user_id_users_id_fk": { + "name": "audit_log_actor_user_id_users_id_fk", + "tableFrom": "audit_log", + "tableTo": "users", + "columnsFrom": [ + "actor_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "consistency_ignores": { + "name": "consistency_ignores", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": { + "consistency_ignores_key_unique": { + "name": "consistency_ignores_key_unique", + "columns": [ + "key" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "diag_log": { + "name": "diag_log", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "operation": { + "name": "operation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ok": { + "name": "ok", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "latency_ms": { + "name": "latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "dns_providers": { + "name": "dns_providers", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "provider_type": { + "name": "provider_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "credential_id": { + "name": "credential_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "config": { + "name": "config", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": { + "dns_providers_credential_id_integration_credentials_id_fk": { + "name": "dns_providers_credential_id_integration_credentials_id_fk", + "tableFrom": "dns_providers", + "tableTo": "integration_credentials", + "columnsFrom": [ + "credential_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "dns_records_cache": { + "name": "dns_records_cache", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "provider_id": { + "name": "provider_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "zone_id": { + "name": "zone_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_id": { + "name": "record_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ttl": { + "name": "ttl", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "proxied": { + "name": "proxied", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "dns_records_cache_provider_id_dns_providers_id_fk": { + "name": "dns_records_cache_provider_id_dns_providers_id_fk", + "tableFrom": "dns_records_cache", + "tableTo": "dns_providers", + "columnsFrom": [ + "provider_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "dns_zones_cache": { + "name": "dns_zones_cache", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "provider_id": { + "name": "provider_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "zone_id": { + "name": "zone_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "zone_name": { + "name": "zone_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "synced_at": { + "name": "synced_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "dns_zones_cache_provider_zone_idx": { + "name": "dns_zones_cache_provider_zone_idx", + "columns": [ + "provider_id", + "zone_id" + ], + "isUnique": true + } + }, + "foreignKeys": { + "dns_zones_cache_provider_id_dns_providers_id_fk": { + "name": "dns_zones_cache_provider_id_dns_providers_id_fk", + "tableFrom": "dns_zones_cache", + "tableTo": "dns_providers", + "columnsFrom": [ + "provider_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "domains": { + "name": "domains", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'manual'" + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "registrar": { + "name": "registrar", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lookup_source": { + "name": "lookup_source", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_checked_at": { + "name": "last_checked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_checked_ok_at": { + "name": "last_checked_ok_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_check_error": { + "name": "last_check_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": { + "domains_name_unique": { + "name": "domains_name_unique", + "columns": [ + "name" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "event_webhooks": { + "name": "event_webhooks", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "events": { + "name": "events", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "last_attempt_at": { + "name": "last_attempt_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_result": { + "name": "last_result", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "integration_credentials": { + "name": "integration_credentials", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "encrypted_secret": { + "name": "encrypted_secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "integrations": { + "name": "integrations", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_url": { + "name": "base_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "credential_id": { + "name": "credential_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "config": { + "name": "config", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": { + "integrations_credential_id_integration_credentials_id_fk": { + "name": "integrations_credential_id_integration_credentials_id_fk", + "tableFrom": "integrations", + "tableTo": "integration_credentials", + "columnsFrom": [ + "credential_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ipam_entries": { + "name": "ipam_entries", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "vendor": { + "name": "vendor", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "location": { + "name": "location", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": { + "ipam_entries_ip_address_unique": { + "name": "ipam_entries_ip_address_unique", + "columns": [ + "ip_address" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "maintenance_aliases": { + "name": "maintenance_aliases", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_id": { + "name": "target_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": { + "maintenance_aliases_unique_idx": { + "name": "maintenance_aliases_unique_idx", + "columns": [ + "address", + "target_type", + "target_id" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "maintenance_schedules": { + "name": "maintenance_schedules", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "targets": { + "name": "targets", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "days": { + "name": "days", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "start_time": { + "name": "start_time", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "duration_minutes": { + "name": "duration_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "timezone": { + "name": "timezone", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "maintenance_windows": { + "name": "maintenance_windows", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_id": { + "name": "target_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ends_at": { + "name": "ends_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "metrics_tokens": { + "name": "metrics_tokens", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_prefix": { + "name": "token_prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "last_used_at": { + "name": "last_used_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "metrics_tokens_token_hash_unique": { + "name": "metrics_tokens_token_hash_unique", + "columns": [ + "token_hash" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "notification_queue": { + "name": "notification_queue", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "port_forwards": { + "name": "port_forwards", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "external_port": { + "name": "external_port", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "protocol": { + "name": "protocol", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'tcp'" + }, + "server_id": { + "name": "server_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "destination": { + "name": "destination", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "internal_port": { + "name": "internal_port", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "comment": { + "name": "comment", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": { + "port_forwards_server_id_servers_id_fk": { + "name": "port_forwards_server_id_servers_id_fk", + "tableFrom": "port_forwards", + "tableTo": "servers", + "columnsFrom": [ + "server_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "runbooks": { + "name": "runbooks", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_id": { + "name": "target_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "category": { + "name": "category", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "runbooks_target_idx": { + "name": "runbooks_target_idx", + "columns": [ + "target_type", + "target_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "scheduled_tasks": { + "name": "scheduled_tasks", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "server_id": { + "name": "server_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "schedule_type": { + "name": "schedule_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'agent'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "command": { + "name": "command", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "schedule_expression": { + "name": "schedule_expression", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "next_run_at": { + "name": "next_run_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "raw_metadata": { + "name": "raw_metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_stale": { + "name": "is_stale", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "first_seen_at": { + "name": "first_seen_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": { + "scheduled_tasks_server_id_servers_id_fk": { + "name": "scheduled_tasks_server_id_servers_id_fk", + "tableFrom": "scheduled_tasks", + "tableTo": "servers", + "columnsFrom": [ + "server_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "secrets": { + "name": "secrets", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'generic'" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expiry_date": { + "name": "expiry_date", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "warn_days": { + "name": "warn_days", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 30 + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "check_host": { + "name": "check_host", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "check_port": { + "name": "check_port", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_checked_at": { + "name": "last_checked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_check_error": { + "name": "last_check_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "server_links": { + "name": "server_links", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "server_id": { + "name": "server_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": { + "server_links_server_id_servers_id_fk": { + "name": "server_links_server_id_servers_id_fk", + "tableFrom": "server_links", + "tableTo": "servers", + "columnsFrom": [ + "server_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "server_metrics": { + "name": "server_metrics", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "server_id": { + "name": "server_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metric": { + "name": "metric", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "recorded_at": { + "name": "recorded_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "server_metrics_lookup_idx": { + "name": "server_metrics_lookup_idx", + "columns": [ + "server_id", + "metric", + "recorded_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "server_metrics_server_id_servers_id_fk": { + "name": "server_metrics_server_id_servers_id_fk", + "tableFrom": "server_metrics", + "tableTo": "servers", + "columnsFrom": [ + "server_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "server_ports": { + "name": "server_ports", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "server_id": { + "name": "server_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "port": { + "name": "port", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "protocol": { + "name": "protocol", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'tcp'" + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "comment": { + "name": "comment", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "open": { + "name": "open", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "last_seen_open_at": { + "name": "last_seen_open_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": { + "server_ports_unique": { + "name": "server_ports_unique", + "columns": [ + "server_id", + "port", + "protocol" + ], + "isUnique": true + } + }, + "foreignKeys": { + "server_ports_server_id_servers_id_fk": { + "name": "server_ports_server_id_servers_id_fk", + "tableFrom": "server_ports", + "tableTo": "servers", + "columnsFrom": [ + "server_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "servers": { + "name": "servers", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hostname": { + "name": "hostname", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "os_type": { + "name": "os_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'linux'" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "api_token_hash": { + "name": "api_token_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "api_token_prefix": { + "name": "api_token_prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ip_addresses": { + "name": "ip_addresses", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cpu_model": { + "name": "cpu_model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cpu_cores": { + "name": "cpu_cores", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cpu_load_percent": { + "name": "cpu_load_percent", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mem_total_bytes": { + "name": "mem_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mem_used_bytes": { + "name": "mem_used_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disks": { + "name": "disks", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "listening_ports": { + "name": "listening_ports", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_port_scan": { + "name": "last_port_scan", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tags": { + "name": "tags", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "proxmox_integration_id": { + "name": "proxmox_integration_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "proxmox_node": { + "name": "proxmox_node", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "proxmox_guest_type": { + "name": "proxmox_guest_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "proxmox_vmid": { + "name": "proxmox_vmid", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "hide_proxmox_link": { + "name": "hide_proxmox_link", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "parent_server_id": { + "name": "parent_server_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "no_parent": { + "name": "no_parent", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "pending_updates": { + "name": "pending_updates", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updates_pending_since": { + "name": "updates_pending_since", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "host_health": { + "name": "host_health", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "thresholds": { + "name": "thresholds", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "agent_version": { + "name": "agent_version", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "servers_proxmox_integration_id_integrations_id_fk": { + "name": "servers_proxmox_integration_id_integrations_id_fk", + "tableFrom": "servers", + "tableTo": "integrations", + "columnsFrom": [ + "proxmox_integration_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "settings": { + "name": "settings", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "share_links": { + "name": "share_links", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_prefix": { + "name": "token_prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "detail": { + "name": "detail", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'summary'" + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "share_links_token_hash_unique": { + "name": "share_links_token_hash_unique", + "columns": [ + "token_hash" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "tag_definitions": { + "name": "tag_definitions", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "color": { + "name": "color", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": { + "tag_definitions_name_unique": { + "name": "tag_definitions_name_unique", + "columns": [ + "name" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "update_runs": { + "name": "update_runs", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "server_id": { + "name": "server_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "integration_id": { + "name": "integration_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "template_id": { + "name": "template_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "template_name": { + "name": "template_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "host_limit": { + "name": "host_limit", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "task_id": { + "name": "task_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "task_status": { + "name": "task_status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "window_id": { + "name": "window_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "window_created": { + "name": "window_created", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "started_by": { + "name": "started_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "task_ended_at": { + "name": "task_ended_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "host_reported_at": { + "name": "host_reported_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updates_before": { + "name": "updates_before", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updates_after": { + "name": "updates_after", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reboot_required": { + "name": "reboot_required", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "poll_errors": { + "name": "poll_errors", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "update_runs_server_idx": { + "name": "update_runs_server_idx", + "columns": [ + "server_id", + "started_at" + ], + "isUnique": false + }, + "update_runs_status_idx": { + "name": "update_runs_status_idx", + "columns": [ + "status" + ], + "isUnique": false + } + }, + "foreignKeys": { + "update_runs_server_id_servers_id_fk": { + "name": "update_runs_server_id_servers_id_fk", + "tableFrom": "update_runs", + "tableTo": "servers", + "columnsFrom": [ + "server_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "oidc_sub": { + "name": "oidc_sub", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'viewer'" + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "last_login_at": { + "name": "last_login_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "favorites": { + "name": "favorites", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_looked_at": { + "name": "last_looked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "groups": { + "name": "groups", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dashboard_layout": { + "name": "dashboard_layout", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "users_oidc_sub_unique": { + "name": "users_oidc_sub_unique", + "columns": [ + "oidc_sub" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/server/drizzle/meta/_journal.json b/server/drizzle/meta/_journal.json index 2a5d9bd..b18dde2 100644 --- a/server/drizzle/meta/_journal.json +++ b/server/drizzle/meta/_journal.json @@ -211,6 +211,20 @@ "when": 1792273577264, "tag": "0029_past_monster_badoon", "breakpoints": true + }, + { + "idx": 30, + "version": "6", + "when": 1792278520773, + "tag": "0030_icy_snowbird", + "breakpoints": true + }, + { + "idx": 31, + "version": "6", + "when": 1792278718505, + "tag": "0031_sad_lizard", + "breakpoints": true } ] } \ No newline at end of file diff --git a/server/src/app.ts b/server/src/app.ts index 3cc0eeb..a9ffa1d 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -11,6 +11,7 @@ import { meRouter } from "./routes/me.js"; import { uptimeRouter } from "./routes/uptime.js"; import { shareLinksRouter } from "./routes/shareLinks.js"; import { runbooksRouter } from "./routes/runbooks.js"; +import { updateRunsRouter } from "./routes/updateRuns.js"; import { inventoryRouter } from "./routes/inventory.js"; import { backupsRouter } from "./routes/backups.js"; import { metricsRouter, metricsTokensRouter } from "./routes/metrics.js"; @@ -98,6 +99,7 @@ export function createApp() { app.use("/api/uptime", uptimeRouter); app.use("/api/share-links", shareLinksRouter); app.use("/api/runbooks", runbooksRouter); + app.use("/api/update-runs", updateRunsRouter); app.use("/api/inventory", inventoryRouter); app.use("/api/backups", backupsRouter); app.use("/api/metrics-tokens", metricsTokensRouter); diff --git a/server/src/db/schema.ts b/server/src/db/schema.ts index d35e634..6817721 100644 --- a/server/src/db/schema.ts +++ b/server/src/db/schema.ts @@ -300,6 +300,10 @@ export const servers = sqliteTable("servers", { // What the agent says about the machine's own health, as JSON: {failedUnits, smart, zfs, reportedAt} — see services/hostHealth.ts. Null until it says. hostHealth: text("host_health"), + // This server's own alert thresholds, as JSON: {serverOfflineMinutes, diskUsagePercent, diskForecastDays, backupStaleHours, mounts: {"/data": {percent, ignore}}} — see + // services/serverThresholds.ts. Whatever isn't there follows Settings. Null when it follows Settings entirely. + thresholds: text("thresholds"), + // Which version of the agent script is running on it, as the agent says in each report (see agent/*/report-tasks.*). Null until it says: // an agent that predates the number being reported is version 1. agentVersion: integer("agent_version"), @@ -568,6 +572,41 @@ export const runbooks = sqliteTable( (t) => [index("runbooks_target_idx").on(t.targetType, t.targetId)], ); +// ─── Update runs: apply a server's updates and see it through ────────────── + +// A playbook run in Semaphore for one server, wrapped in a maintenance window and followed until the server reports again — see +// services/updateRuns.ts. status: running | waiting_for_host | done | failed | cancelled. +export const updateRuns = sqliteTable( + "update_runs", + { + id: integer("id").primaryKey({ autoIncrement: true }), + serverId: integer("server_id") + .notNull() + .references(() => servers.id, { onDelete: "cascade" }), + integrationId: integer("integration_id").notNull(), // the Semaphore integration; no FK — one removed later leaves the history readable + projectId: integer("project_id").notNull(), + templateId: integer("template_id").notNull(), + templateName: text("template_name").notNull(), + hostLimit: text("host_limit").notNull(), // ansible's --limit + taskId: integer("task_id"), + taskStatus: text("task_status"), // Semaphore's own word for the run + windowId: integer("window_id"), // the maintenance window put round it + windowCreated: integer("window_created", { mode: "boolean" }).notNull().default(false), // false if it only extended one that was already there (which is then left alone) + status: text("status").notNull(), + message: text("message"), + startedBy: text("started_by"), + startedAt: text("started_at").notNull(), // ISO + taskEndedAt: text("task_ended_at"), // ISO — when the playbook finished + hostReportedAt: text("host_reported_at"), // ISO — the report that came after it + finishedAt: text("finished_at"), // ISO + updatesBefore: integer("updates_before"), + updatesAfter: integer("updates_after"), + rebootRequired: integer("reboot_required", { mode: "boolean" }).notNull().default(false), + pollErrors: integer("poll_errors").notNull().default(0), + }, + (t) => [index("update_runs_server_idx").on(t.serverId, t.startedAt), index("update_runs_status_idx").on(t.status)], +); + // ─── Inventory: the hardware behind it all ────────────────────────────────── export const assetKinds = ["server", "storage", "network", "power", "client", "other"] as const; diff --git a/server/src/index.ts b/server/src/index.ts index ee965c3..dd5a7d3 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -16,6 +16,7 @@ import { initHealthScheduler } from "./services/healthScheduler.js"; import { initAlertTicketScheduler } from "./services/alertTicketScheduler.js"; import { initKumaMaintenanceScheduler } from "./services/kumaMaintenanceScheduler.js"; import { initMaintenanceScheduleScheduler } from "./services/maintenanceScheduleScheduler.js"; +import { initUpdateRuns } from "./services/updateRuns.js"; { // Sessions are signed with this secret. Without one, the app would sign them with a value that's in the source code. @@ -48,6 +49,7 @@ await initHealthScheduler(); await initAlertTicketScheduler(); await initKumaMaintenanceScheduler(); await initMaintenanceScheduleScheduler(); +initUpdateRuns(); createApp().listen(env.port, () => { console.log(`homelab-manager listening on port ${env.port}`); diff --git a/server/src/routes/servers.ts b/server/src/routes/servers.ts index 4c4d46a..965921b 100644 --- a/server/src/routes/servers.ts +++ b/server/src/routes/servers.ts @@ -9,6 +9,7 @@ import { recordAudit } from "../services/audit.js"; import { loadParentRows, resolveParents, wouldLoop } from "../services/serverParents.js"; import { parsePendingUpdates } from "../services/pendingUpdates.js"; import { parseHostHealth } from "../services/hostHealth.js"; +import { checkThresholds, parseThresholds } from "../services/serverThresholds.js"; import { latestAgentVersions } from "../services/agentVersions.js"; import { getServerMetrics, RETENTION_DAYS } from "../services/serverMetrics.js"; import { getServerActivity } from "../services/serverActivity.js"; @@ -20,14 +21,15 @@ import { serverPortsRouter } from "./serverPorts.js"; import { InvalidTagError, normalizeTags, parseTags } from "../services/serverTags.js"; import { buildServerSummary } from "../services/serverSummary.js"; import { activeSubjects } from "../services/maintenance.js"; +import { invalidateAlertsCache } from "../services/alerts.js"; import { getSettings, updateSettings } from "../services/settingsStore.js"; export const serversRouter = Router(); /** A server row as the API returns it: no token hash, and tags as an array rather than the stored JSON. */ -function publicServer(row: T) { - const { apiTokenHash: _hash, tags, pendingUpdates, hostHealth, ...rest } = row; - return { ...rest, tags: parseTags(tags), pendingUpdates: parsePendingUpdates(pendingUpdates), hostHealth: parseHostHealth(hostHealth) }; +function publicServer(row: T) { + const { apiTokenHash: _hash, tags, pendingUpdates, hostHealth, thresholds, ...rest } = row; + return { ...rest, tags: parseTags(tags), pendingUpdates: parsePendingUpdates(pendingUpdates), hostHealth: parseHostHealth(hostHealth), thresholds: parseThresholds(thresholds) }; } serversRouter.use(requireAuth); @@ -135,6 +137,8 @@ const updateServerSchema = z parentServerId: z.number().int().nullable().optional(), // True says it runs on nothing, which also stops the Proxmox-node guess. Choosing a server, or null, turns it off again. noParent: z.boolean().optional(), + // This server's own alert thresholds (null goes back to following Settings). Checked on its own below, so a wrong number gets a message that says which. + thresholds: z.unknown().optional(), }) .refine( (data) => { @@ -181,10 +185,20 @@ serversRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req, res) } } - const changes: typeof parsed.data = { ...parsed.data }; + const { thresholds: requestedThresholds, ...fields } = parsed.data; + const changes: typeof fields & { thresholds?: string | null } = { ...fields }; if (changes.noParent === true) changes.parentServerId = null; else if (changes.parentServerId !== undefined) changes.noParent = false; + let thresholdsChange: unknown; + if (requestedThresholds !== undefined) { + const checked = checkThresholds(requestedThresholds); + if (!checked.ok) return res.status(400).json({ error: "invalid_thresholds", message: checked.message }); + changes.thresholds = checked.value ? JSON.stringify(checked.value) : null; + thresholdsChange = checked.value; + } const [updated] = await db.update(servers).set(changes).where(eq(servers.id, id)).returning(); + // A server's own limits (or the server it runs on) change what is an alert, so the Alerts page shouldn't go on showing the old answer for up to a minute. + if (requestedThresholds !== undefined || fields.parentServerId !== undefined || fields.noParent !== undefined) invalidateAlertsCache(); await recordAudit({ actor: req.currentUser!, @@ -192,7 +206,7 @@ serversRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req, res) action: "update", targetType: "server", targetId: id, - detail: { name: updated.name, ...(parsed.data.parentServerId !== undefined ? { parentServerId: parsed.data.parentServerId } : {}), ...(parsed.data.noParent !== undefined ? { noParent: parsed.data.noParent } : {}) }, + detail: { name: updated.name, ...(parsed.data.parentServerId !== undefined ? { parentServerId: parsed.data.parentServerId } : {}), ...(parsed.data.noParent !== undefined ? { noParent: parsed.data.noParent } : {}), ...(requestedThresholds !== undefined ? { thresholds: thresholdsChange } : {}) }, }); const serverOut = publicServer(updated); @@ -203,7 +217,7 @@ serversRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req, res) // (which viewers can't read), so the widget and the alerts always agree about what "offline" and "full" mean. serversRouter.get("/summary", asyncHandler(async (_req, res) => { const rows = await db - .select({ id: servers.id, name: servers.name, osType: servers.osType, lastSeenAt: servers.lastSeenAt, disks: servers.disks }) + .select({ id: servers.id, name: servers.name, osType: servers.osType, lastSeenAt: servers.lastSeenAt, disks: servers.disks, thresholds: servers.thresholds }) .from(servers); const { healthChecks } = await getSettings(); res.json(buildServerSummary(rows, healthChecks, Date.now(), await activeSubjects())); @@ -367,6 +381,7 @@ serversRouter.get("/:id/detail", asyncHandler(async (req, res) => { lastPortScan: _lastPortScan, pendingUpdates: rawUpdates, hostHealth: rawHealth, + thresholds: rawThresholds, ...serverOut } = server; @@ -380,7 +395,7 @@ serversRouter.get("/:id/detail", asyncHandler(async (req, res) => { children: [...links2.entries()].filter(([, l]) => l.parentId === id).map(([childId, l]) => ({ id: childId, name: nameOf.get(childId) ?? "", auto: l.auto })).sort((a, b) => a.name.localeCompare(b.name)), }; - res.json({ server: { ...serverOut, tags: parseTags(rawTags), pendingUpdates: parsePendingUpdates(rawUpdates), hostHealth: parseHostHealth(rawHealth) }, ipAddresses, hardware, dnsMatches, links, relations, agentLatest: latestAgentVersions() }); + res.json({ server: { ...serverOut, tags: parseTags(rawTags), pendingUpdates: parsePendingUpdates(rawUpdates), hostHealth: parseHostHealth(rawHealth), thresholds: parseThresholds(rawThresholds) }, thresholdDefaults: (await getSettings()).healthChecks, ipAddresses, hardware, dnsMatches, links, relations, agentLatest: latestAgentVersions() }); })); const tagsSchema = z.object({ tags: z.array(z.string().max(100)).max(50) }); diff --git a/server/src/routes/settings.ts b/server/src/routes/settings.ts index f315698..a9b9e72 100644 --- a/server/src/routes/settings.ts +++ b/server/src/routes/settings.ts @@ -76,6 +76,7 @@ const updateSchema = z.object({ pbsVerificationCheck: z.boolean(), healthAlerts: z.boolean(), automationAlerts: z.boolean(), + updateRunResults: z.boolean(), domainExpiryCheck: z.boolean(), secretCheckTime: z.string().regex(/^\d{2}:\d{2}$/), timezone: z.string(), @@ -95,7 +96,7 @@ const updateSchema = z.object({ .partial() .optional(), healthChecks: z - .object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90) }) + .object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160) }) .partial() .optional(), quietHours: z diff --git a/server/src/routes/updateRuns.ts b/server/src/routes/updateRuns.ts new file mode 100644 index 0000000..e32ee48 --- /dev/null +++ b/server/src/routes/updateRuns.ts @@ -0,0 +1,78 @@ +import { Router } from "express"; +import { z } from "zod"; +import { requireAuth, requireRole } from "../auth/middleware.js"; +import { asyncHandler } from "../utils/asyncHandler.js"; +import { ACTIVE_STATUSES, cancelUpdateRun, listUpdateRuns, startUpdateRun, UpdateRunError } from "../services/updateRuns.js"; + +export const updateRunsRouter = Router(); + +// Anyone signed in can see how updates are going, like the Updates page itself. Starting or stopping one is for operators. +updateRunsRouter.use(requireAuth); + +// Which hosts the playbook may touch: names, addresses and ansible patterns — and nothing that could be taken for an option. The same rule the +// Semaphore page uses, except that it may not be left out: an update run is for one server. +const startSchema = z.object({ + serverId: z.number().int().positive(), + integrationId: z.number().int().positive(), + projectId: z.number().int().positive(), + templateId: z.number().int().positive(), + limit: z.string().trim().regex(/^[A-Za-z0-9][A-Za-z0-9._,:*\-]{0,199}$/, "Use the host name as it is in the Semaphore inventory, like web01"), + windowMinutes: z.number().int().min(15).max(480).default(90), +}); + +const shape = (r: Awaited>[number]) => ({ + id: r.id, + serverId: r.serverId, + serverName: r.serverName, + integrationId: r.integrationId, + template: r.templateName, + limit: r.hostLimit, + taskId: r.taskId, + taskStatus: r.taskStatus, + status: r.status, + active: ACTIVE_STATUSES.includes(r.status as (typeof ACTIVE_STATUSES)[number]), + message: r.message, + startedBy: r.startedBy, + startedAt: r.startedAt, + taskEndedAt: r.taskEndedAt, + finishedAt: r.finishedAt, + updatesBefore: r.updatesBefore, + updatesAfter: r.updatesAfter, + rebootRequired: r.rebootRequired, +}); + +updateRunsRouter.get("/", asyncHandler(async (req, res) => { + const serverId = Number(req.query.serverId); + const runs = await listUpdateRuns({ serverId: Number.isInteger(serverId) && serverId > 0 ? serverId : undefined, limit: 30 }); + res.json({ runs: runs.map(shape) }); +})); + +updateRunsRouter.post("/", requireRole("operator"), asyncHandler(async (req, res) => { + const parsed = startSchema.safeParse(req.body); + if (!parsed.success) { + return res.status(400).json({ error: "invalid_body", message: parsed.error.issues[0]?.message ?? "Choose a server, a template and the host name it runs on.", details: parsed.error.flatten() }); + } + const actor = req.currentUser!; + try { + const run = await startUpdateRun({ ...parsed.data, actor: actor.name ?? actor.email ?? actor.oidcSub }); + const [shaped] = (await listUpdateRuns({ serverId: run.serverId, limit: 1 })).map(shape); + res.status(201).json({ run: shaped }); + } catch (err) { + if (err instanceof UpdateRunError) return res.status(err.status).json({ error: err.code, message: err.message }); + throw err; + } +})); + +updateRunsRouter.post("/:id/cancel", requireRole("operator"), asyncHandler(async (req, res) => { + const id = Number(req.params.id); + if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" }); + const actor = req.currentUser!; + try { + const run = await cancelUpdateRun(id, actor.name ?? actor.email ?? actor.oidcSub); + if (!run) return res.status(404).json({ error: "not_found" }); + res.json({ status: run.status }); + } catch (err) { + if (err instanceof UpdateRunError) return res.status(err.status).json({ error: err.code, message: err.message }); + throw err; + } +})); diff --git a/server/src/services/alerts.ts b/server/src/services/alerts.ts index 8505f68..6b9d3d5 100644 --- a/server/src/services/alerts.ts +++ b/server/src/services/alerts.ts @@ -13,6 +13,8 @@ import { eq } from "drizzle-orm"; import { db } from "../db/client.js"; import { integrations, secrets, servers } from "../db/schema.js"; import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js"; +import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js"; +import { getBackupCoverage, staleBackups } from "./backupCoverage.js"; import { loadIntegrationConfig } from "../integrations/loadIntegration.js"; import { createUptimeKumaAdapter } from "../integrations/uptimekuma/adapter.js"; import { createOsTicketAdapter } from "../integrations/osticket/adapter.js"; @@ -241,6 +243,25 @@ export async function collectAlerts(): Promise { sourceFailures.forEach((f) => cannotCheckIntegration(f, "Proxmox backups")); }), + // A guest whose newest backup is older than its limit — which Proxmox's own "latest run failed" alert can't see, because that only + // looks at the most recent run per node. + check("Backup coverage", async () => { + const now = new Date(); + const report = await getBackupCoverage({ now, rules: true }); + for (const s of staleBackups(report, now, subjects)) { + push({ + category: "backup", + severity: s.severity, + type: "proxmox", + message: s.message, + key: s.key, + link: "/backups", + silenced: s.silenced, + target: s.guest.server ? targetOf("server", s.guest.server.id) : integrationTarget(s.guest.integrationId), + }); + } + }), + check("Backup verification", async () => { const { problems, sourceFailures } = await collectPbsProblems({ skipSilenced: false }); for (const p of problems) { @@ -376,10 +397,13 @@ export async function collectAlerts(): Promise { // Not a notification: a disk that is on course to fill is something to see coming, so it is shown here, from the server's own history. check("Disk forecasts", async () => { const { healthChecks } = await getSettings(); - if (!(healthChecks.diskForecastDays > 0)) return; - for (const s of await db.select({ id: servers.id, name: servers.name }).from(servers)) { + for (const s of await db.select({ id: servers.id, name: servers.name, thresholds: servers.thresholds }).from(servers)) { + // A server may have its own forecast window (0 turns it off for that server), its own limit, and disks it is told to leave alone. + const own = parseThresholds(s.thresholds); + const forecastDays = effectiveThresholds(healthChecks, own).diskForecastDays; + if (!(forecastDays > 0)) continue; const series = await getServerMetrics(s.id, 14); - for (const f of disksFillingSoon(series, healthChecks.diskForecastDays, healthChecks.diskUsagePercent)) { + for (const f of disksFillingSoon(series, forecastDays, (mount) => diskLimit(own, mount, healthChecks.diskUsagePercent))) { push({ category: "disk", severity: f.daysToFull <= 3 ? "critical" : "warning", diff --git a/server/src/services/backupCoverage.ts b/server/src/services/backupCoverage.ts index f4515c8..52790dc 100644 --- a/server/src/services/backupCoverage.ts +++ b/server/src/services/backupCoverage.ts @@ -14,6 +14,9 @@ import { db } from "../db/client.js"; import { integrations, servers } from "../db/schema.js"; import { loadIntegrationConfig } from "../integrations/loadIntegration.js"; import { createPbsAdapter, type PbsBackupGroup } from "../integrations/pbs/adapter.js"; +import { parseThresholds } from "./serverThresholds.js"; +import { activeSubjects } from "./maintenance.js"; +import { getSettings } from "./settingsStore.js"; import { createProxmoxAdapter, guestsWithoutBackupCoverage, type ProxmoxBackupJob, type ProxmoxBackupTask, type ProxmoxGuest } from "../integrations/proxmox/adapter.js"; export type CoverageStatus = "ok" | "stale" | "failed" | "none" | "unknown"; @@ -36,6 +39,8 @@ export interface CoverageGuest { status: CoverageStatus; /** Why it is "failed", when it is. */ failure: string | null; + /** How old its newest backup may be before it counts as stale, in hours — the limit that applied to this guest (0 = never stale). */ + staleAfterHours: number; /** The server in this app that is this guest, if one is linked to it. */ server: { id: number; name: string } | null; } @@ -65,6 +70,8 @@ export interface LinkedServer { name: string; proxmoxIntegrationId: number | null; proxmoxVmid: number | null; + /** This server's own "backup older than" limit, when it has one and the caller wants it used. */ + staleHours?: number | null; } export interface CoverageReport { @@ -81,7 +88,7 @@ export interface CoverageReport { * with nothing to say when it last ran is "unknown". */ export function buildCoverage(input: { proxmox: ProxmoxSource[]; pbs: PbsSource[]; servers: LinkedServer[]; now: Date; staleHours?: number }): CoverageGuest[] { - const staleMs = (input.staleHours ?? DEFAULT_STALE_HOURS) * 3600_000; + const defaultHours = input.staleHours ?? DEFAULT_STALE_HOURS; const nowMs = input.now.getTime(); // The newest snapshot of each guest, across every Backup Server (guests are told apart there by "vm/100" or "ct/101"). @@ -113,12 +120,14 @@ export function buildCoverage(input: { proxmox: ProxmoxSource[]; pbs: PbsSource[ else if (latestTask && !latestTask.ok && latestTask.status !== "running" && (!newest || Date.parse(latestTask.startTime) > newest.at)) failure = `the latest backup didn't succeed (${latestTask.status})`; const inJob = !uncovered.has(g.vmid); + const linked = input.servers.find((s) => s.proxmoxIntegrationId === src.integrationId && s.proxmoxVmid === g.vmid); + // The server this guest is may have its own limit (a weekly backup is fine at 8 days); 0 means this one is never called stale. + const staleAfterHours = linked?.staleHours ?? defaultHours; let status: CoverageStatus; if (failure) status = "failed"; - else if (newest) status = nowMs - newest.at > staleMs ? "stale" : "ok"; + else if (newest) status = staleAfterHours > 0 && nowMs - newest.at > staleAfterHours * 3600_000 ? "stale" : "ok"; else status = inJob ? "unknown" : "none"; - const linked = input.servers.find((s) => s.proxmoxIntegrationId === src.integrationId && s.proxmoxVmid === g.vmid); out.push({ integrationId: src.integrationId, integrationName: src.integrationName, @@ -132,6 +141,7 @@ export function buildCoverage(input: { proxmox: ProxmoxSource[]; pbs: PbsSource[ lastSource: newest ? newest.source : null, status, failure, + staleAfterHours, server: linked ? { id: linked.id, name: linked.name } : null, }); } @@ -205,15 +215,76 @@ async function loadSources(): Promise { return loaded; } -export async function getBackupCoverage(options: { staleHours?: number; refresh?: boolean; now?: Date } = {}): Promise { +/** + * The coverage report. `staleHours` is how old a backup may be before it counts as stale, for every guest — what the Backups page's own + * selector asks for. With `rules: true` the limit comes from Settings instead (`healthChecks.backupStaleHours`), and a server that has its + * own limit uses that for the guest it is — which is what the alerts, the daily message, the digest and /metrics use. + * `cachedOnly` never goes to Proxmox or Backup Server: it answers from what was last read, or null if nothing has been read yet. + */ +export async function getBackupCoverage(options: { staleHours?: number; refresh?: boolean; now?: Date; rules?: boolean; cachedOnly?: boolean } = {}): Promise { + const report = await readCoverage(options); + if (!report) throw new Error("no backup coverage has been read yet"); + return report; +} + +export async function getCachedBackupCoverage(now: Date = new Date()): Promise { + return readCoverage({ now, rules: true, cachedOnly: true }); +} + +async function readCoverage(options: { staleHours?: number; refresh?: boolean; now?: Date; rules?: boolean; cachedOnly?: boolean }): Promise { const now = options.now ?? new Date(); - const staleHours = options.staleHours ?? DEFAULT_STALE_HOURS; - if (!cache || options.refresh || now.getTime() - cache.at > CACHE_MS || loaderOverride) { + if (options.cachedOnly) { + if (!cache) return null; + } else if (!cache || options.refresh || now.getTime() - cache.at > CACHE_MS || loaderOverride) { cache = { at: now.getTime(), loaded: await loadSources() }; } - const serverRows = await db.select({ id: servers.id, name: servers.name, proxmoxIntegrationId: servers.proxmoxIntegrationId, proxmoxVmid: servers.proxmoxVmid }).from(servers); - const guests = buildCoverage({ proxmox: cache.loaded.proxmox, pbs: cache.loaded.pbs, servers: serverRows, now, staleHours }).sort( - (a, b) => a.name.localeCompare(b.name) || a.vmid - b.vmid, - ); - return { guests, counts: summarize(guests), sources: cache.loaded.sources, staleHours, generatedAt: now.toISOString() }; + const loaded = cache!.loaded; + const staleHours = options.rules ? (await getSettings()).healthChecks.backupStaleHours : (options.staleHours ?? DEFAULT_STALE_HOURS); + const rows = await db.select({ id: servers.id, name: servers.name, proxmoxIntegrationId: servers.proxmoxIntegrationId, proxmoxVmid: servers.proxmoxVmid, thresholds: servers.thresholds }).from(servers); + const serverRows: LinkedServer[] = rows.map((r) => ({ id: r.id, name: r.name, proxmoxIntegrationId: r.proxmoxIntegrationId, proxmoxVmid: r.proxmoxVmid, staleHours: options.rules ? (parseThresholds(r.thresholds)?.backupStaleHours ?? null) : null })); + const guests = buildCoverage({ proxmox: loaded.proxmox, pbs: loaded.pbs, servers: serverRows, now, staleHours }).sort((a, b) => a.name.localeCompare(b.name) || a.vmid - b.vmid); + return { guests, counts: summarize(guests), sources: loaded.sources, staleHours, generatedAt: now.toISOString() }; +} + +export interface StaleBackup { + guest: CoverageGuest; + /** How long ago its newest backup was, in hours. */ + ageHours: number; + severity: "critical" | "warning"; + /** Stable for the same guest, so it alerts once. */ + key: string; + /** In maintenance: on the server it is, or on the Proxmox integration it lives in. */ + silenced: boolean; + message: string; +} + +// Hours up to three days, days after that — so a 48-hour limit and a 60-hour-old backup read as "48 h" and "60 h", not both as "2 days". +const hoursWords = (h: number) => (h < 72 ? `${Math.round(h)} h` : `${Math.floor(h / 24)} days`); + +/** + * Guests whose newest backup is older than their limit. A failed backup is not here: the latest failing run on a node, and snapshots that + * failed verification, already raise their own alerts, and a guest that goes on failing becomes stale once its limit passes. Critical at + * three times the limit. + */ +export function staleBackups(report: CoverageReport, now: Date, silencedSubjects: Set = new Set()): StaleBackup[] { + const out: StaleBackup[] = []; + for (const g of report.guests) { + if (g.status !== "stale" || !g.lastBackupAt) continue; + const ageHours = (now.getTime() - Date.parse(g.lastBackupAt)) / 3600_000; + out.push({ + guest: g, + ageHours, + severity: ageHours >= g.staleAfterHours * 3 ? "critical" : "warning", + key: `backup-stale:${g.integrationId}:${g.vmid}`, + silenced: silencedSubjects.has(`integration:${g.integrationId}`) || (g.server !== null && silencedSubjects.has(`server:${g.server.id}`)), + message: `${g.name} (#${g.vmid}) [${g.integrationName}]: its newest backup is ${hoursWords(ageHours)} old (the limit is ${hoursWords(g.staleAfterHours)})`, + }); + } + return out.sort((a, b) => b.ageHours - a.ageHours); +} + +/** What the daily check looks at: the stale guests that aren't under a maintenance window, read fresh from Proxmox and Backup Server. */ +export async function collectStaleBackups(now: Date = new Date()): Promise { + const report = await getBackupCoverage({ now, rules: true, refresh: true }); + return staleBackups(report, now, await activeSubjects(now)).filter((s) => !s.silenced); } diff --git a/server/src/services/healthMonitor.ts b/server/src/services/healthMonitor.ts index 65b0469..287ca89 100644 --- a/server/src/services/healthMonitor.ts +++ b/server/src/services/healthMonitor.ts @@ -10,6 +10,7 @@ import { getInternalFlag, getSettings, setInternalFlag } from "./settingsStore.j import { activeSubjects, subjectOfConditionKey } from "./maintenance.js"; import { coveredByParent, loadParentRows, resolveParents } from "./serverParents.js"; import { hostHealthConditions, parseHostHealth, type HostHealth } from "./hostHealth.js"; +import { diskLimit, parseThresholds, type ServerThresholds } from "./serverThresholds.js"; const STATE_FLAG = "healthActiveConditions"; /** After a restart the agents haven't had a chance to report yet (they run every 15 min), so server-derived conditions are held rather than judged. */ @@ -35,6 +36,8 @@ export interface ServerSnapshot { disks: { mount: string; sizeBytes: number; usedBytes: number }[]; /** What the agent says about failed services, disk health and ZFS pools, if it says anything. */ health?: HostHealth | null; + /** This server's own thresholds, where it has any: they replace the global ones for it. */ + thresholds?: ServerThresholds | null; } export interface ProxmoxSnapshot { integrationId: number; @@ -106,7 +109,7 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds, let offline = false; if (s.lastSeenAt) { const age = now - parseTimestamp(s.lastSeenAt); - if (Number.isFinite(age) && age > thresholds.serverOfflineMinutes * 60_000) { + if (Number.isFinite(age) && age > (s.thresholds?.serverOfflineMinutes ?? thresholds.serverOfflineMinutes) * 60_000) { offline = true; out.push({ key: `offline:server:${s.id}`, @@ -121,7 +124,9 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds, for (const c of hostHealthConditions(s.id, s.name, s.health ?? null)) out.push({ key: c.key, source: "server", message: c.message, severity: c.severity }); for (const d of s.disks) { const p = usage(d.usedBytes, d.sizeBytes); - if (p !== null && p >= limit) { + // A disk can have its own limit, or be left alone altogether; otherwise the server's, then the global one. + const diskAlertAt = diskLimit(s.thresholds, d.mount, limit); + if (diskAlertAt !== null && p !== null && p >= diskAlertAt) { out.push({ key: `disk:server:${s.id}:${d.mount}`, source: "server", @@ -300,7 +305,7 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel } catch { // a malformed report shouldn't stop every other server being checked } - snapshot.servers.push({ id: s.id, name: s.name, lastSeenAt: s.lastSeenAt, disks, health: parseHostHealth(s.hostHealth) }); + snapshot.servers.push({ id: s.id, name: s.name, lastSeenAt: s.lastSeenAt, disks, health: parseHostHealth(s.hostHealth), thresholds: parseThresholds(s.thresholds) }); } const rows = await db diff --git a/server/src/services/metrics.ts b/server/src/services/metrics.ts index 6e5ac5c..531f6e0 100644 --- a/server/src/services/metrics.ts +++ b/server/src/services/metrics.ts @@ -11,6 +11,8 @@ import { isAgentOutdated, latestAgentVersions } from "./agentVersions.js"; import { getSourceStatus } from "./integrationHealthMonitor.js"; import { parsePendingUpdates } from "./pendingUpdates.js"; import { parseHostHealth, poolIsFine } from "./hostHealth.js"; +import { parseThresholds } from "./serverThresholds.js"; +import { getCachedBackupCoverage } from "./backupCoverage.js"; import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js"; import { computeSecretStatus } from "./secretStatus.js"; import { getSettings } from "./settingsStore.js"; @@ -58,8 +60,9 @@ export async function buildMetrics(now: Date = new Date()): Promise { // ── Servers ── const latest = latestAgentVersions(); const serverRows = await db.select().from(servers); - const offlineAfterMs = settings.healthChecks.serverOfflineMinutes * 60_000; for (const s of serverRows) { + // A server may have its own "offline after"; otherwise the global one. + const offlineAfterMs = (parseThresholds(s.thresholds)?.serverOfflineMinutes ?? settings.healthChecks.serverOfflineMinutes) * 60_000; if (!s.lastSeenAt) continue; // never reported: nothing to say about it const server = s.name; const seen = Date.parse(s.lastSeenAt); @@ -115,6 +118,14 @@ export async function buildMetrics(now: Date = new Date()): Promise { // ── Alerts ── try { const report = await getAlerts(false); + // What the backup check last found (the alert pass above refreshes it); never reads Proxmox or Backup Server itself. + const coverage = await getCachedBackupCoverage(now); + if (coverage) { + for (const status of ["ok", "stale", "failed", "none", "unknown"] as const) out.add("homelab_backup_guests", "gauge", "Proxmox guests by how well they are backed up (ok, stale, failed, none, unknown).", coverage.counts[status], { status }); + for (const g of coverage.guests) { + if (g.lastBackupAt) out.add("homelab_backup_last_age_seconds", "gauge", "How long ago a guest's newest backup was.", Math.max(0, Math.floor((now.getTime() - Date.parse(g.lastBackupAt)) / 1000)), { guest: g.name, vmid: String(g.vmid), integration: g.integrationName }); + } + } for (const severity of ["critical", "warning", "info"] as const) out.add("homelab_alerts", "gauge", "Active alerts by severity (not counting silenced ones).", report.counts[severity], { severity }); out.add("homelab_alerts_silenced", "gauge", "Alerts held back by a maintenance window.", report.counts.silenced); out.add("homelab_alerts_grouped", "gauge", "Alerts folded under a server that is down.", report.counts.grouped); diff --git a/server/src/services/notify.ts b/server/src/services/notify.ts index 9487803..6e67675 100644 --- a/server/src/services/notify.ts +++ b/server/src/services/notify.ts @@ -294,6 +294,23 @@ export async function notifyProxmoxUncoveredGuests( ); } +export async function notifyStaleBackups(stale: { message: string }[]): Promise { + if (stale.length === 0) return; + if (!(await eventEnabled("proxmoxBackupCheck"))) return; + await notify( + "Homelab Manager — Backups Gone Stale", + `${stale.length} guest${stale.length !== 1 ? "s have" : " has"} no recent backup:\n\n${stale.map((s) => s.message).join("\n")}`, + ); +} + +export async function notifyUpdateRunFinished(run: { serverName: string; templateName: string; ok: boolean; message: string }): Promise { + if (!(await eventEnabled("updateRunResults"))) return; + await notify( + run.ok ? `Homelab Manager — ${run.serverName} updated` : `Homelab Manager — Update of ${run.serverName} didn't finish`, + `"${run.templateName}" on ${run.serverName}: ${run.message}`, + ); +} + export async function notifyPbsVerificationFailed( failures: { integrationName: string; datastore: string; failedCount: number; error: string | null }[], ): Promise { diff --git a/server/src/services/proxmoxBackupScheduler.ts b/server/src/services/proxmoxBackupScheduler.ts index 2ccf657..4069842 100644 --- a/server/src/services/proxmoxBackupScheduler.ts +++ b/server/src/services/proxmoxBackupScheduler.ts @@ -4,7 +4,8 @@ import { db } from "../db/client.js"; import { integrations } from "../db/schema.js"; import { loadIntegrationConfig } from "../integrations/loadIntegration.js"; import { createProxmoxAdapter, guestsWithoutBackupCoverage, type ProxmoxBackupTask } from "../integrations/proxmox/adapter.js"; -import { notifyProxmoxBackupFailure, notifyProxmoxUncoveredGuests } from "./notify.js"; +import { notifyProxmoxBackupFailure, notifyProxmoxUncoveredGuests, notifyStaleBackups } from "./notify.js"; +import { collectStaleBackups } from "./backupCoverage.js"; import { getSettings, getInternalFlag, setInternalFlag } from "./settingsStore.js"; import { activeSubjects } from "./maintenance.js"; import type { SourceFailure } from "./alertTypes.js"; @@ -92,6 +93,13 @@ async function checkProxmoxBackups(): Promise { const { failures, uncovered } = await collectProxmoxBackupProblems({ skipSilenced: true }); await notifyProxmoxBackupFailure(failures); await notifyProxmoxUncoveredGuests(uncovered); + // Guests whose newest backup has gone too old (the limit in Settings, or a server's own). A problem reading Proxmox or Backup Server + // here must not stop the other two messages, which have already gone. + try { + await notifyStaleBackups(await collectStaleBackups()); + } catch (err) { + console.error("[proxmoxBackup] couldn't check for stale backups:", err); + } } async function checkProxmoxBackupsOnce(): Promise { diff --git a/server/src/services/publicWall.ts b/server/src/services/publicWall.ts index b7d10ce..c2719c7 100644 --- a/server/src/services/publicWall.ts +++ b/server/src/services/publicWall.ts @@ -40,7 +40,7 @@ export async function buildPublicWall(detail: ShareLinkDetail, now: Date = new D getAlerts(false), getSettings(), listActiveWindows(now), - db.select({ id: servers.id, name: servers.name, osType: servers.osType, lastSeenAt: servers.lastSeenAt, disks: servers.disks }).from(servers), + db.select({ id: servers.id, name: servers.name, osType: servers.osType, lastSeenAt: servers.lastSeenAt, disks: servers.disks, thresholds: servers.thresholds }).from(servers), db.select({ id: integrations.id, name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations), activeSubjects(now), ]); diff --git a/server/src/services/serverActivity.ts b/server/src/services/serverActivity.ts index 787391f..42305f3 100644 --- a/server/src/services/serverActivity.ts +++ b/server/src/services/serverActivity.ts @@ -72,6 +72,19 @@ export function describeAudit(category: string, action: string, detail: Record number | null)): DiskForecast[] { const out: DiskForecast[] = []; for (const s of series) { if (s.kind !== "disk" || !s.trend || s.latest === null) continue; const { perDay, fit, spanDays, daysToFull } = s.trend; if (daysToFull === null || daysToFull > withinDays) continue; if (spanDays < FORECAST_MIN_SPAN_DAYS || fit < FORECAST_MIN_FIT) continue; - if (s.latest >= alreadyAlertedAt) continue; - out.push({ mount: s.metric.slice("disk:".length), latest: s.latest, perDay, daysToFull }); + const mount = s.metric.slice("disk:".length); + const limit = typeof alreadyAlertedAt === "function" ? alreadyAlertedAt(mount) : alreadyAlertedAt; + if (limit === null || s.latest >= limit) continue; + out.push({ mount, latest: s.latest, perDay, daysToFull }); } return out.sort((a, b) => a.daysToFull - b.daysToFull); } diff --git a/server/src/services/serverSummary.ts b/server/src/services/serverSummary.ts index 3537d26..1405f3c 100644 --- a/server/src/services/serverSummary.ts +++ b/server/src/services/serverSummary.ts @@ -1,4 +1,5 @@ import { evaluateHealth, type Thresholds } from "./healthMonitor.js"; +import { parseThresholds } from "./serverThresholds.js"; /** The columns of a server row the summary reads. */ export interface SummaryRow { @@ -7,6 +8,8 @@ export interface SummaryRow { osType: string; lastSeenAt: string | null; disks: string | null; // JSON, as the agent's report is stored + /** This server's own thresholds, as stored. */ + thresholds?: string | null; } export interface ServerSummary { @@ -40,7 +43,7 @@ function parseDisks(text: string | null): { mount: string; sizeBytes: number; us */ export function buildServerSummary(rows: SummaryRow[], thresholds: Thresholds, now: number, silencedSubjects: Set = new Set()): ServerSummary { const snapshot = { - servers: rows.map((r) => ({ id: r.id, name: r.name, lastSeenAt: r.lastSeenAt, disks: parseDisks(r.disks) })), + servers: rows.map((r) => ({ id: r.id, name: r.name, lastSeenAt: r.lastSeenAt, disks: parseDisks(r.disks), thresholds: parseThresholds(r.thresholds) })), proxmox: [], synology: [], }; diff --git a/server/src/services/serverThresholds.ts b/server/src/services/serverThresholds.ts new file mode 100644 index 0000000..b0c3c68 --- /dev/null +++ b/server/src/services/serverThresholds.ts @@ -0,0 +1,124 @@ +/** + * Per-server alert thresholds. The ones in Settings → Notifications apply to every server; some machines are different — a NAS that sits at + * 92% by design, a database host that should speak up at 80%, a laptop that is meant to be off at night. Each server may override the + * server-offline time, the disk-usage percentage, the disk forecast window and (for the Proxmox guest it is linked to) how old a backup may + * be, and may set a limit for one disk or say not to alert about that disk at all. Anything not overridden follows the global setting, so + * changing the global one still reaches every server that hasn't said otherwise. + */ + +export interface MountThreshold { + /** Alert when this disk is this full (instead of the server's or the global percentage). */ + percent?: number; + /** Never alert about this disk — neither when it is full nor when it is filling. */ + ignore?: boolean; +} + +export interface ServerThresholds { + serverOfflineMinutes?: number; + diskUsagePercent?: number; + /** 0 turns the "filling up" forecast off for this server. */ + diskForecastDays?: number; + /** 0 turns the stale-backup alert off for this server's guest. */ + backupStaleHours?: number; + mounts?: Record; +} + +/** The global values these override. */ +export interface GlobalThresholds { + serverOfflineMinutes: number; + diskUsagePercent: number; + diskForecastDays: number; + backupStaleHours: number; +} + +export const LIMITS = { + serverOfflineMinutes: { min: 15, max: 10080 }, + diskUsagePercent: { min: 50, max: 99 }, + diskForecastDays: { min: 0, max: 90 }, + backupStaleHours: { min: 0, max: 2160 }, +} as const; +export const MAX_MOUNT_RULES = 50; + +const inRange = (v: unknown, key: keyof typeof LIMITS): v is number => typeof v === "number" && Number.isInteger(v) && v >= LIMITS[key].min && v <= LIMITS[key].max; + +/** What is stored, read back leniently: anything that doesn't make sense is dropped, never an error. Null for no overrides at all. */ +export function parseThresholds(stored: string | null | undefined): ServerThresholds | null { + if (!stored) return null; + try { + const v = JSON.parse(stored); + if (!v || typeof v !== "object") return null; + const out: ServerThresholds = {}; + for (const key of Object.keys(LIMITS) as (keyof typeof LIMITS)[]) if (inRange(v[key], key)) out[key] = v[key]; + if (v.mounts && typeof v.mounts === "object" && !Array.isArray(v.mounts)) { + const mounts: Record = {}; + for (const [mount, rule] of Object.entries(v.mounts as Record).slice(0, MAX_MOUNT_RULES)) { + if (!rule || typeof rule !== "object") continue; + const r = rule as { percent?: unknown; ignore?: unknown }; + const clean: MountThreshold = {}; + if (inRange(r.percent, "diskUsagePercent")) clean.percent = r.percent; + if (r.ignore === true) clean.ignore = true; + if (clean.percent !== undefined || clean.ignore) mounts[mount] = clean; + } + if (Object.keys(mounts).length > 0) out.mounts = mounts; + } + return Object.keys(out).length > 0 ? out : null; + } catch { + return null; + } +} + +/** What a request may say. Strict where parseThresholds is lenient: a wrong number is refused, with a message that says which. */ +export function checkThresholds(input: unknown): { ok: true; value: ServerThresholds | null } | { ok: false; message: string } { + if (input === null) return { ok: true, value: null }; + if (typeof input !== "object" || Array.isArray(input)) return { ok: false, message: "Thresholds should be an object." }; + const v = input as Record; + const out: ServerThresholds = {}; + const names: Record = { + serverOfflineMinutes: "Offline after (minutes)", + diskUsagePercent: "Disk usage alert at (%)", + diskForecastDays: "Disk forecast (days)", + backupStaleHours: "Backup older than (hours)", + }; + for (const key of Object.keys(LIMITS) as (keyof typeof LIMITS)[]) { + if (v[key] === undefined || v[key] === null) continue; + if (!inRange(v[key], key)) return { ok: false, message: `${names[key]} must be a whole number from ${LIMITS[key].min} to ${LIMITS[key].max}.` }; + out[key] = v[key] as number; + } + if (v.mounts !== undefined && v.mounts !== null) { + if (typeof v.mounts !== "object" || Array.isArray(v.mounts)) return { ok: false, message: "Disk rules should be an object." }; + const entries = Object.entries(v.mounts as Record); + if (entries.length > MAX_MOUNT_RULES) return { ok: false, message: `At most ${MAX_MOUNT_RULES} disks can have their own rule.` }; + const mounts: Record = {}; + for (const [mount, rule] of entries) { + if (mount.length === 0 || mount.length > 500) return { ok: false, message: "A disk's name is missing or too long." }; + if (!rule || typeof rule !== "object" || Array.isArray(rule)) return { ok: false, message: `The rule for ${mount} should be an object.` }; + const r = rule as { percent?: unknown; ignore?: unknown }; + const clean: MountThreshold = {}; + if (r.percent !== undefined && r.percent !== null) { + if (!inRange(r.percent, "diskUsagePercent")) return { ok: false, message: `The limit for ${mount} must be a whole number from ${LIMITS.diskUsagePercent.min} to ${LIMITS.diskUsagePercent.max}.` }; + clean.percent = r.percent; + } + if (r.ignore === true) clean.ignore = true; + if (clean.percent !== undefined || clean.ignore) mounts[mount] = clean; + } + if (Object.keys(mounts).length > 0) out.mounts = mounts; + } + return { ok: true, value: Object.keys(out).length > 0 ? out : null }; +} + +/** What applies to one server: its own value where it gave one, the global one otherwise. */ +export function effectiveThresholds(global: GlobalThresholds, own: ServerThresholds | null | undefined): GlobalThresholds { + return { + serverOfflineMinutes: own?.serverOfflineMinutes ?? global.serverOfflineMinutes, + diskUsagePercent: own?.diskUsagePercent ?? global.diskUsagePercent, + diskForecastDays: own?.diskForecastDays ?? global.diskForecastDays, + backupStaleHours: own?.backupStaleHours ?? global.backupStaleHours, + }; +} + +/** The usage percentage at which a disk alerts — its own rule first, then the server's, then the global one — or null if it is to be left alone. */ +export function diskLimit(own: ServerThresholds | null | undefined, mount: string, globalPercent: number): number | null { + const rule = own?.mounts?.[mount]; + if (rule?.ignore) return null; + return rule?.percent ?? own?.diskUsagePercent ?? globalPercent; +} diff --git a/server/src/services/settingsStore.ts b/server/src/services/settingsStore.ts index 09f1a07..821389b 100644 --- a/server/src/services/settingsStore.ts +++ b/server/src/services/settingsStore.ts @@ -50,6 +50,8 @@ export interface NotificationEvents { healthAlerts: boolean; /** A Semaphore template or Gitea repo whose latest run failed, and when it succeeds again. */ automationAlerts: boolean; + /** How an update run started from the Updates page ended: what is still waiting, whether a restart is due, or why it failed. */ + updateRunResults: boolean; /** Daily reminder while a domain registration is close to expiring or has expired. */ domainExpiryCheck: boolean; secretCheckTime: string; // "HH:MM" — shared by the secret-expiry, Tailscale key-expiry, Docker update, and Proxmox backup checks @@ -97,6 +99,8 @@ export interface HealthCheckSettings { domainWarnDays: number; /** An agent-reported disk that is on course to be full within this many days is flagged on the Alerts page. 0 turns it off. */ diskForecastDays: number; + /** A Proxmox guest whose newest backup is older than this many hours is raised as stale. 0 turns it off. */ + backupStaleHours: number; } /** The weekly digest: one message a week through the notification channels. Off until an admin switches it on under Settings → Notifications. */ @@ -208,6 +212,7 @@ const DEFAULTS: AppSettings = { pbsVerificationCheck: true, healthAlerts: true, automationAlerts: true, + updateRunResults: true, domainExpiryCheck: true, secretCheckTime: "08:00", timezone: "UTC", @@ -223,7 +228,7 @@ const DEFAULTS: AppSettings = { inventory: { currency: "", pricePerKwh: null }, agents: { autoUpdate: false }, signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" }, - healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14 }, + healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48 }, // Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every // Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN. consistency: { excludedRanges: ["172.16.0.0/12"] }, diff --git a/server/src/services/topology.ts b/server/src/services/topology.ts index 0a4b2af..a0c9746 100644 --- a/server/src/services/topology.ts +++ b/server/src/services/topology.ts @@ -10,6 +10,7 @@ import { dnsRecordsCache, dnsZonesCache, integrations, servers } from "../db/sch import { loadIntegrationConfig } from "../integrations/loadIntegration.js"; import { createUptimeKumaAdapter } from "../integrations/uptimekuma/adapter.js"; import { getSettings } from "./settingsStore.js"; +import { parseThresholds } from "./serverThresholds.js"; import { attachMatchedServers, toMatchableServers } from "./uptimeKumaMatch.js"; export type MapStatus = "ok" | "bad" | "warn" | "unknown"; @@ -64,7 +65,6 @@ export async function buildTopology(now: Date = new Date()): Promise { const notes: string[] = []; const { healthChecks } = await getSettings(); - const offlineAfterMs = healthChecks.serverOfflineMinutes * 60_000; const serverRows = await db.select().from(servers); const integrationRows = await db.select({ id: integrations.id, name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations); const integrationName = new Map(integrationRows.map((i) => [i.id, i.name])); @@ -73,6 +73,7 @@ export async function buildTopology(now: Date = new Date()): Promise { const hostIds = new Set(); for (const s of serverRows) { const lastSeen = s.lastSeenAt ? Date.parse(s.lastSeenAt) : null; + const offlineAfterMs = (parseThresholds(s.thresholds)?.serverOfflineMinutes ?? healthChecks.serverOfflineMinutes) * 60_000; const status: MapStatus = lastSeen === null ? "unknown" : now.getTime() - lastSeen > offlineAfterMs ? "bad" : "ok"; nodes.push({ id: `server:${s.id}`, kind: "server", label: s.name, sub: s.hostname, status, href: `/servers/${s.id}` }); if (s.proxmoxIntegrationId && s.proxmoxNode) { diff --git a/server/src/services/updateRuns.ts b/server/src/services/updateRuns.ts new file mode 100644 index 0000000..848bbc7 --- /dev/null +++ b/server/src/services/updateRuns.ts @@ -0,0 +1,271 @@ +/** + * An update run: apply a server's waiting updates in one go, and see it through. It puts the server under maintenance (so nothing alerts + * while it is being worked on and restarted), starts a Semaphore playbook limited to that server, follows the run until it is over, waits + * for the server's agent to report again, then says how many updates are still waiting and whether a restart is due, and ends the + * maintenance window it made. Every step is written down on the run, which survives a restart of this app — a poller picks up where it was. + * + * What it does not do: it does not restart the server (the playbook does, if it is written to), and it cannot stop a run in Semaphore — + * cancelling here only stops watching and ends the window. + */ +import { and, desc, eq, inArray } from "drizzle-orm"; +import { db } from "../db/client.js"; +import { maintenanceWindows, servers, updateRuns } from "../db/schema.js"; +import { loadIntegrationConfig } from "../integrations/loadIntegration.js"; +import { createSemaphoreAdapter, type SemaphoreAdapter, type SemaphoreTask } from "../integrations/semaphore/adapter.js"; +import { invalidateAlertsCache } from "./alerts.js"; +import { recordAudit } from "./audit.js"; +import { startOrExtendWindow } from "./maintenance.js"; +import { notifyUpdateRunFinished } from "./notify.js"; +import { parsePendingUpdates } from "./pendingUpdates.js"; + +export type UpdateRunRow = typeof updateRuns.$inferSelect; +export type UpdateRunStatus = "running" | "waiting_for_host" | "done" | "failed" | "cancelled"; +export const ACTIVE_STATUSES: UpdateRunStatus[] = ["running", "waiting_for_host"]; + +/** How long a run may take from start to finish before it is given up on. */ +export const RUN_LIMIT_MS = 3 * 3600_000; +/** After the playbook is over, how long the server has to report again (it may be restarting). */ +export const HOST_WAIT_MS = 30 * 60_000; +/** Consecutive polls Semaphore may fail to answer before the run is given up on (at the poller's pace, about five minutes). */ +export const MAX_POLL_ERRORS = 10; +export const POLL_INTERVAL_MS = 20_000; + +const FINISHED_TASK = new Set(["success", "error", "stopped", "rejected"]); + +// ─── Semaphore, replaceable for the tests ─────────────────────────────────── + +let adapterOverride: ((integrationId: number) => Promise) | null = null; +export function setSemaphoreForTests(factory: ((integrationId: number) => Promise) | null): void { + adapterOverride = factory; +} + +async function semaphoreFor(integrationId: number): Promise { + if (adapterOverride) return adapterOverride(integrationId); + const loaded = await loadIntegrationConfig(integrationId); + if (!loaded || loaded.integration.type !== "semaphore" || !loaded.integration.enabled) return null; + return createSemaphoreAdapter(loaded.config as any); +} + +// ─── Starting ─────────────────────────────────────────────────────────────── + +export class UpdateRunError extends Error { + constructor( + readonly status: number, + readonly code: string, + message: string, + ) { + super(message); + } +} + +export interface StartInput { + serverId: number; + integrationId: number; + projectId: number; + templateId: number; + /** Which hosts of the inventory the playbook may touch (ansible's --limit). Never empty: an update run is always for this one server. */ + limit: string; + /** How long the maintenance window lasts at most; it ends sooner when the run does. */ + windowMinutes: number; + actor: string; +} + +export async function startUpdateRun(input: StartInput, now: Date = new Date()): Promise { + const [server] = await db.select().from(servers).where(eq(servers.id, input.serverId)).limit(1); + if (!server) throw new UpdateRunError(404, "server_not_found", "That server doesn't exist."); + if (!server.lastSeenAt) throw new UpdateRunError(400, "never_reported", "This server has never reported, so there is nothing to wait for after the run."); + + const [busy] = await db + .select({ id: updateRuns.id }) + .from(updateRuns) + .where(and(eq(updateRuns.serverId, server.id), inArray(updateRuns.status, ACTIVE_STATUSES))) + .limit(1); + if (busy) throw new UpdateRunError(409, "already_running", `An update run for ${server.name} is already under way.`); + + const adapter = await semaphoreFor(input.integrationId); + if (!adapter) throw new UpdateRunError(400, "no_semaphore", "That Semaphore integration doesn't exist or isn't switched on."); + + let templateName: string; + try { + const { templates } = await adapter.checkTemplates(); + const template = templates.find((t) => t.projectId === input.projectId && t.id === input.templateId); + if (!template) throw new UpdateRunError(404, "template_not_found", "That template isn't in Semaphore (any more)."); + templateName = template.name; + } catch (err) { + if (err instanceof UpdateRunError) throw err; + throw new UpdateRunError(502, "semaphore_unreachable", `Couldn't reach Semaphore: ${err instanceof Error ? err.message : String(err)}`); + } + + // Quiet first, so nothing alerts about the server being worked on — and if the playbook can't be started the quiet is taken back. + const started = await startOrExtendWindow("server", server.id, input.windowMinutes, `Update run: ${templateName}`, input.actor, now); + let task: SemaphoreTask; + try { + task = await adapter.runTemplate(input.projectId, input.templateId, { limit: input.limit, message: `Update run for ${server.name}, started in Homelab Manager by ${input.actor} (limited to ${input.limit})` }); + } catch (err) { + if (started && !started.extended) await db.update(maintenanceWindows).set({ endsAt: now.toISOString() }).where(eq(maintenanceWindows.id, started.window.id)); + invalidateAlertsCache(); + throw new UpdateRunError(502, "semaphore_failed", `Semaphore wouldn't start it: ${err instanceof Error ? err.message : String(err)}`); + } + invalidateAlertsCache(); + + const updates = parsePendingUpdates(server.pendingUpdates); + const [run] = await db + .insert(updateRuns) + .values({ + serverId: server.id, + integrationId: input.integrationId, + projectId: input.projectId, + templateId: input.templateId, + templateName, + hostLimit: input.limit, + taskId: task.id, + taskStatus: task.status, + windowId: started?.window.id ?? null, + windowCreated: started ? !started.extended : false, + status: "running", + message: "The playbook has been started.", + startedBy: input.actor, + startedAt: now.toISOString(), + updatesBefore: updates?.count ?? null, + }) + .returning(); + await recordAudit({ category: "update_run", action: "start", targetType: "server", targetId: server.id, detail: { run: run.id, template: templateName, limit: input.limit, task: task.id, by: input.actor } }); + return run; +} + +// ─── Following ────────────────────────────────────────────────────────────── + +/** Ends the maintenance window an update run made (one it only extended belongs to whoever made it, and is left to run out). */ +async function endWindow(run: UpdateRunRow, now: Date): Promise { + if (!run.windowCreated || run.windowId === null) return; + await db + .update(maintenanceWindows) + .set({ endsAt: now.toISOString() }) + .where(and(eq(maintenanceWindows.id, run.windowId))); + invalidateAlertsCache(); +} + +async function finish(run: UpdateRunRow, status: "done" | "failed" | "cancelled", message: string, now: Date, extra: Partial = {}): Promise { + await endWindow(run, now); + await db.update(updateRuns).set({ status, message, finishedAt: now.toISOString(), ...extra }).where(eq(updateRuns.id, run.id)); + const [server] = await db.select({ name: servers.name }).from(servers).where(eq(servers.id, run.serverId)).limit(1); + await recordAudit({ category: "update_run", action: status === "done" ? "finished" : status, targetType: "server", targetId: run.serverId, detail: { run: run.id, template: run.templateName, message } }); + if (status !== "cancelled") { + await notifyUpdateRunFinished({ serverName: server?.name ?? `#${run.serverId}`, templateName: run.templateName, ok: status === "done", message }).catch((err) => console.error("[updateRuns] couldn't send the result:", err)); + } +} + +const updatesWords = (n: number) => (n === 1 ? "1 update" : `${n} updates`); + +/** What a finished run says about the server, from the report that came after it. */ +export function summarizeOutcome(before: number | null, after: number | null, reboot: boolean): string { + const parts: string[] = []; + if (after !== null) { + parts.push(after === 0 ? "No updates are waiting now" : `${updatesWords(after)} still waiting`); + if (before !== null && before > after) parts[0] += ` (${updatesWords(before - after)} installed)`; + } else { + parts.push("The server reported again, but doesn't say what is waiting"); + } + parts.push(reboot ? "a restart is still needed" : "no restart is needed"); + return `${parts.join("; ")}.`; +} + +async function advance(run: UpdateRunRow, now: Date): Promise { + if (now.getTime() - Date.parse(run.startedAt) > RUN_LIMIT_MS) { + await finish(run, "failed", "It took longer than three hours, so it is no longer being followed. Check Semaphore.", now); + return; + } + + if (run.status === "running") { + const adapter = await semaphoreFor(run.integrationId); + let task: SemaphoreTask | null = null; + try { + task = adapter && run.taskId !== null ? await adapter.getTask(run.projectId, run.taskId) : null; + } catch { + task = null; + } + if (!task) { + const errors = run.pollErrors + 1; + if (errors >= MAX_POLL_ERRORS) await finish(run, "failed", "Lost contact with Semaphore, so it is no longer being followed. Check Semaphore.", now, { pollErrors: errors }); + else await db.update(updateRuns).set({ pollErrors: errors }).where(eq(updateRuns.id, run.id)); + return; + } + if (!FINISHED_TASK.has(task.status)) { + if (task.status !== run.taskStatus || run.pollErrors > 0) await db.update(updateRuns).set({ taskStatus: task.status, pollErrors: 0 }).where(eq(updateRuns.id, run.id)); + return; + } + if (task.status === "success") { + const ended = task.end ? new Date(task.end).toISOString() : now.toISOString(); + await db.update(updateRuns).set({ status: "waiting_for_host", taskStatus: task.status, taskEndedAt: ended, pollErrors: 0, message: "The playbook finished. Waiting for the server to report again." }).where(eq(updateRuns.id, run.id)); + return; + } + const why = task.status === "error" ? "The playbook failed in Semaphore" : task.status === "rejected" ? "The run was rejected in Semaphore" : "The run was stopped in Semaphore"; + await finish(run, "failed", `${why} (run #${run.taskId}). The server is no longer under maintenance.`, now, { taskStatus: task.status }); + return; + } + + // waiting_for_host: the agent has reported since the playbook ended (and, where it reports updates, those are newer than the run too). + const ended = Date.parse(run.taskEndedAt ?? run.startedAt); + const [server] = await db.select().from(servers).where(eq(servers.id, run.serverId)).limit(1); + if (!server) { + await finish(run, "failed", "The server was removed.", now); + return; + } + const reported = server.lastSeenAt ? Date.parse(server.lastSeenAt) > ended : false; + const updates = parsePendingUpdates(server.pendingUpdates); + const updatesFresh = updates ? Date.parse(updates.reportedAt) > ended : true; + if (reported && updatesFresh) { + await finish(run, "done", summarizeOutcome(run.updatesBefore, updates?.count ?? null, updates?.rebootRequired ?? false), now, { + hostReportedAt: server.lastSeenAt, + updatesAfter: updates?.count ?? null, + rebootRequired: updates?.rebootRequired ?? false, + }); + return; + } + if (now.getTime() - ended > HOST_WAIT_MS) { + await finish(run, "failed", `The playbook finished, but the server hasn't reported again in ${Math.round(HOST_WAIT_MS / 60_000)} minutes. Check that it came back.`, now); + } +} + +/** One pass over every run that isn't over. Returns how many it looked at (for the tests). */ +export async function advanceUpdateRuns(now: Date = new Date()): Promise { + const active = await db.select().from(updateRuns).where(inArray(updateRuns.status, ACTIVE_STATUSES)); + for (const run of active) { + try { + await advance(run, now); + } catch (err) { + console.error(`[updateRuns] run ${run.id}:`, err); + } + } + return active.length; +} + +export async function cancelUpdateRun(id: number, actor: string, now: Date = new Date()): Promise { + const [run] = await db.select().from(updateRuns).where(eq(updateRuns.id, id)).limit(1); + if (!run) return null; + if (!ACTIVE_STATUSES.includes(run.status as UpdateRunStatus)) throw new UpdateRunError(409, "not_active", "That run is already over."); + await finish(run, "cancelled", `Stopped being followed by ${actor}. A playbook already started keeps running in Semaphore.`, now); + const [after] = await db.select().from(updateRuns).where(eq(updateRuns.id, id)).limit(1); + return after; +} + +/** Recent runs, those still going first; for one server or for all. */ +export async function listUpdateRuns(options: { serverId?: number; limit?: number } = {}): Promise<(UpdateRunRow & { serverName: string | null })[]> { + const rows = await db + .select({ run: updateRuns, serverName: servers.name }) + .from(updateRuns) + .leftJoin(servers, eq(servers.id, updateRuns.serverId)) + .where(options.serverId !== undefined ? eq(updateRuns.serverId, options.serverId) : undefined) + .orderBy(desc(updateRuns.startedAt)) + .limit(options.limit ?? 30); + const out = rows.map((r) => ({ ...r.run, serverName: r.serverName })); + return [...out.filter((r) => ACTIVE_STATUSES.includes(r.status as UpdateRunStatus)), ...out.filter((r) => !ACTIVE_STATUSES.includes(r.status as UpdateRunStatus))]; +} + +let timer: ReturnType | null = null; + +/** Follows runs that are under way, every twenty seconds — including ones that were under way when this app last stopped. */ +export function initUpdateRuns(): void { + if (timer) return; + timer = setInterval(() => void advanceUpdateRuns().catch((err) => console.error("[updateRuns]", err)), POLL_INTERVAL_MS); +} diff --git a/server/src/services/weeklyDigest.ts b/server/src/services/weeklyDigest.ts index 1501fb2..243db5f 100644 --- a/server/src/services/weeklyDigest.ts +++ b/server/src/services/weeklyDigest.ts @@ -17,6 +17,7 @@ import { notify, notifyNow } from "./notify.js"; import { parsePendingUpdates } from "./pendingUpdates.js"; import { computeSecretStatus } from "./secretStatus.js"; import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js"; +import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js"; import { getSettings } from "./settingsStore.js"; import { getUptime } from "./uptimeHistory.js"; @@ -68,7 +69,7 @@ export async function buildDigest(now: Date = new Date()): Promise { const sections: Section[] = []; const serverRows = await db - .select({ id: servers.id, name: servers.name, lastSeenAt: servers.lastSeenAt, pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince }) + .select({ id: servers.id, name: servers.name, lastSeenAt: servers.lastSeenAt, pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince, thresholds: servers.thresholds }) .from(servers); const serverName = new Map(serverRows.map((s) => [s.id, s.name])); @@ -134,11 +135,14 @@ export async function buildDigest(now: Date = new Date()): Promise { // ── Disks heading for full ── try { - const { diskForecastDays, diskUsagePercent } = settings.healthChecks; - if (diskForecastDays > 0) { + const { diskUsagePercent } = settings.healthChecks; + { const lines: string[] = []; for (const s of serverRows) { - for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), diskForecastDays, diskUsagePercent)) { + const own = parseThresholds(s.thresholds); + const forecastDays = effectiveThresholds(settings.healthChecks, own).diskForecastDays; + if (!(forecastDays > 0)) continue; + for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), forecastDays, (mount) => diskLimit(own, mount, diskUsagePercent))) { lines.push(`- ${s.name}: ${f.mount} is ${Math.round(f.latest)}% full and filling by about ${f.perDay.toFixed(1)} points a day — full in about ${plural(f.daysToFull, "day")}`); } } @@ -196,7 +200,7 @@ export async function buildDigest(now: Date = new Date()): Promise { // ── Backups ── try { - const coverage = await getBackupCoverage({ now }); + const coverage = await getBackupCoverage({ now, rules: true }); const order = { failed: 0, none: 1, stale: 2 } as const; const bad = coverage.guests.filter((g): g is typeof g & { status: keyof typeof order } => g.status === "failed" || g.status === "none" || g.status === "stale").sort((a, b) => order[a.status] - order[b.status] || a.name.localeCompare(b.name)); if (bad.length > 0) { diff --git a/server/test/serverThresholds.test.ts b/server/test/serverThresholds.test.ts new file mode 100644 index 0000000..73174bc --- /dev/null +++ b/server/test/serverThresholds.test.ts @@ -0,0 +1,236 @@ +/** + * Alert thresholds that belong to one server: how they are read and checked, what they change in the alerts, the forecast and /metrics, + * and who may set them. + * + * Run with: npm test --workspace server + */ +import { test, describe, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { AddressInfo } from "node:net"; +import type { Server } from "node:http"; + +const dir = mkdtempSync(join(tmpdir(), "hm-thresholds-")); +process.env.DATABASE_PATH = join(dir, "test.sqlite"); +process.env.SESSION_DIR = join(dir, "sessions"); +process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); +process.env.HEALTH_STARTUP_GRACE_MS = "0"; + +const express = (await import("express")).default; +const { db } = await import("../src/db/client.js"); +const { users, servers, serverMetrics, auditLog } = await import("../src/db/schema.js"); +const { runMigrations } = await import("../src/db/migrate.js"); +const { parseThresholds, checkThresholds, effectiveThresholds, diskLimit } = await import("../src/services/serverThresholds.js"); +const { evaluateHealth } = await import("../src/services/healthMonitor.js"); +const { disksFillingSoon } = await import("../src/services/serverMetrics.js"); +const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js"); +const { buildMetrics } = await import("../src/services/metrics.js"); +const { serversRouter } = await import("../src/routes/servers.js"); +const { hashToken } = await import("../src/services/tokens.js"); +const { eq } = await import("drizzle-orm"); + +await runMigrations(); + +const GLOBAL = { serverOfflineMinutes: 60, diskUsagePercent: 90, diskForecastDays: 14, backupStaleHours: 48 }; + +describe("reading and checking thresholds", () => { + test("what is stored is read back leniently: nonsense is dropped, never an error", () => { + assert.equal(parseThresholds(null), null); + assert.equal(parseThresholds("not json"), null); + assert.equal(parseThresholds("{}"), null); + assert.deepEqual(parseThresholds(JSON.stringify({ diskUsagePercent: 95, serverOfflineMinutes: 5, diskForecastDays: "soon", mounts: { "/data": { percent: 97 }, "/x": { percent: 3 }, "/y": "no", "/z": { ignore: true } } })), { + diskUsagePercent: 95, + mounts: { "/data": { percent: 97 }, "/z": { ignore: true } }, + }); + }); + + test("a request is refused with a message that says which number is wrong", () => { + assert.deepEqual(checkThresholds(null), { ok: true, value: null }); + assert.deepEqual(checkThresholds({}), { ok: true, value: null }); + assert.deepEqual(checkThresholds({ diskUsagePercent: 95, serverOfflineMinutes: null }), { ok: true, value: { diskUsagePercent: 95 } }); + const bad = (input: unknown) => { + const r = checkThresholds(input); + assert.equal(r.ok, false, JSON.stringify(input)); + return (r as { message: string }).message; + }; + assert.match(bad({ diskUsagePercent: 20 }), /Disk usage alert at \(%\) must be a whole number from 50 to 99/); + assert.match(bad({ serverOfflineMinutes: 5 }), /Offline after \(minutes\)/); + assert.match(bad({ diskForecastDays: 1.5 }), /Disk forecast/); + assert.match(bad({ backupStaleHours: 99999 }), /Backup older than/); + assert.match(bad({ mounts: { "/data": { percent: 100 } } }), /limit for \/data/); + assert.match(bad({ mounts: { "": { ignore: true } } }), /missing or too long/); + assert.match(bad("high"), /should be an object/); + assert.match(bad({ mounts: Object.fromEntries(Array.from({ length: 51 }, (_, i) => [`/m${i}`, { ignore: true }])) }), /At most 50/); + }); + + test("a value of its own wins, and the rest follows Settings — including 0, which means off", () => { + assert.deepEqual(effectiveThresholds(GLOBAL, null), GLOBAL); + assert.deepEqual(effectiveThresholds(GLOBAL, { diskUsagePercent: 95, diskForecastDays: 0 }), { ...GLOBAL, diskUsagePercent: 95, diskForecastDays: 0 }); + }); + + test("a disk's limit: its own rule, then the server's, then Settings; or none at all", () => { + const own = { diskUsagePercent: 95, mounts: { "/data": { percent: 80 }, "/boot": { ignore: true } } }; + assert.equal(diskLimit(own, "/data", 90), 80); + assert.equal(diskLimit(own, "/", 90), 95); + assert.equal(diskLimit(own, "/boot", 90), null); + assert.equal(diskLimit(null, "/", 90), 90); + assert.equal(diskLimit({}, "/", 90), 90); + }); +}); + +describe("what it changes in the health rules", () => { + const NOW = Date.parse("2026-10-17T12:00:00Z"); + const disk = (mount: string, used: number) => ({ mount, sizeBytes: 100, usedBytes: used }); + const snap = (extra: object) => ({ servers: [{ id: 1, name: "nas", lastSeenAt: new Date(NOW - 5 * 60_000).toISOString(), disks: [disk("/", 92), disk("/data", 85), disk("/boot", 99)], ...extra }], proxmox: [], synology: [] }); + const keys = (extra: object, now = NOW) => evaluateHealth(snap(extra), { serverOfflineMinutes: 60, diskUsagePercent: 90 }, now).map((c) => c.key).sort(); + + test("with nothing of its own, the global limit applies to every disk", () => { + assert.deepEqual(keys({}), ["disk:server:1:/", "disk:server:1:/boot"]); + }); + + test("a server's own percentage replaces the global one; a disk's own rule replaces both; an ignored disk says nothing", () => { + assert.deepEqual(keys({ thresholds: { diskUsagePercent: 95 } }), ["disk:server:1:/boot"], "92% is fine for a server that alerts at 95"); + assert.deepEqual(keys({ thresholds: { diskUsagePercent: 80 } }), ["disk:server:1:/", "disk:server:1:/boot", "disk:server:1:/data"]); + assert.deepEqual(keys({ thresholds: { mounts: { "/boot": { ignore: true }, "/data": { percent: 80 } } } }), ["disk:server:1:/", "disk:server:1:/data"]); + assert.deepEqual(keys({ thresholds: { diskUsagePercent: 99, mounts: { "/": { percent: 91 }, "/boot": { ignore: true } } } }), ["disk:server:1:/"]); + assert.deepEqual(keys({ thresholds: { diskUsagePercent: 99, mounts: { "/": { percent: 91 } } } }), ["disk:server:1:/", "disk:server:1:/boot"], "99% is at the limit"); + }); + + test("a server's own 'offline after' replaces the global one in either direction", () => { + const quietFor = (minutes: number) => new Date(NOW - minutes * 60_000).toISOString(); + assert.deepEqual(keys({ lastSeenAt: quietFor(120), disks: [] }), ["offline:server:1"]); + assert.deepEqual(keys({ lastSeenAt: quietFor(120), disks: [], thresholds: { serverOfflineMinutes: 300 } }), [], "a laptop that is off at night"); + assert.deepEqual(keys({ lastSeenAt: quietFor(30), disks: [], thresholds: { serverOfflineMinutes: 15 } }), ["offline:server:1"], "a host that should speak up sooner"); + }); + + test("the forecast leaves out a disk it is told to ignore, and uses the disk's own limit", () => { + const series = [ + { metric: "disk:/data", kind: "disk" as const, label: "Disk /data", points: [], latest: 85, trend: { perDay: 2, fit: 0.99, spanDays: 7, daysToFull: 7 } }, + { metric: "disk:/scratch", kind: "disk" as const, label: "Disk /scratch", points: [], latest: 60, trend: { perDay: 2, fit: 0.99, spanDays: 7, daysToFull: 20 } }, + ]; + assert.deepEqual(disksFillingSoon(series, 30, 90).map((f) => f.mount), ["/data", "/scratch"]); + assert.deepEqual(disksFillingSoon(series, 30, (m) => (m === "/scratch" ? null : 90)).map((f) => f.mount), ["/data"], "told to leave /scratch alone"); + assert.deepEqual(disksFillingSoon(series, 30, (m) => (m === "/data" ? 80 : 90)).map((f) => f.mount), ["/scratch"], "/data is past its own 80%, so the full-disk alert has it"); + }); +}); + +describe("through the server routes, the alerts and /metrics", () => { + let app: Server; + let base = ""; + let who = "admin"; + let id = 0; + const call = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) }); + const patch = (body: unknown) => call("PATCH", `/servers/${id}`, body); + + before(async () => { + await db.insert(users).values([ + { oidcSub: "admin", name: "Ada", role: "admin" }, + { oidcSub: "operator", name: "Olle", role: "operator" }, + ]); + const twoHoursAgo = new Date(Date.now() - 2 * 3600_000).toISOString(); + const [s] = await db + .insert(servers) + .values({ name: "laptop", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t", lastSeenAt: twoHoursAgo, disks: JSON.stringify([{ mount: "/", sizeBytes: 100, usedBytes: 93 }, { mount: "/boot", sizeBytes: 100, usedBytes: 99 }]) }) + .returning(); + id = s.id; + const a = express(); + a.use(express.json()); + a.use((req, _res, next) => { + (req as unknown as { session: unknown }).session = { user: { sub: who } }; + next(); + }); + a.use("/servers", serversRouter); + app = await new Promise((resolve) => { + const x = a.listen(0, "127.0.0.1", () => resolve(x)); + }); + base = `http://127.0.0.1:${(app.address() as AddressInfo).port}`; + }); + after(() => { + app.close(); + try { + rmSync(dir, { recursive: true, force: true }); + } catch { + // Windows keeps the database file locked until the process exits. + } + }); + + const alertIds = async () => { + invalidateAlertsCache(); + return (await getAlerts(true)).alerts.filter((a) => a.id.includes(`:server:${id}`)).map((a) => a.id).sort(); + }; + + test("only an administrator can set them, a wrong number is refused, and nothing changes until it is right", async () => { + who = "operator"; + assert.equal((await patch({ thresholds: { diskUsagePercent: 95 } })).status, 403); + who = "admin"; + const bad = await patch({ thresholds: { diskUsagePercent: 20 } }); + assert.equal(bad.status, 400); + const body = await bad.json(); + assert.equal(body.error, "invalid_thresholds"); + assert.match(body.message, /Disk usage alert at/); + const detail = await (await call("GET", `/servers/${id}/detail`)).json(); + assert.equal(detail.server.thresholds, null); + assert.deepEqual(detail.thresholdDefaults.diskUsagePercent, 90, "the page can say what it would follow"); + assert.equal(detail.thresholdDefaults.backupStaleHours, 48); + }); + + test("with none set, the laptop has been quiet for two hours, so it is offline (and an offline server's disks aren't judged)", async () => { + assert.deepEqual(await alertIds(), [`offline:offline:server:${id}`]); + }); + + test("the Alerts page doesn't go on showing the old answer for a minute after a server's limits change", async () => { + await patch({ thresholds: null }); + invalidateAlertsCache(); + assert.ok((await getAlerts(false)).alerts.some((a) => a.id === `offline:offline:server:${id}`), "offline, as the cache now holds it"); + await patch({ thresholds: { serverOfflineMinutes: 600 } }); + assert.equal((await getAlerts(false)).alerts.some((a) => a.id === `offline:offline:server:${id}`), false, "no longer, straight away — without anyone clearing the cache"); + await patch({ thresholds: null }); + }); + + test("its own rules quiet exactly what they say, show on the detail, and are in the audit log", async () => { + const res = await patch({ thresholds: { serverOfflineMinutes: 600, diskUsagePercent: 95, mounts: { "/boot": { ignore: true } } } }); + assert.equal(res.status, 200); + assert.deepEqual((await res.json()).server.thresholds, { serverOfflineMinutes: 600, diskUsagePercent: 95, mounts: { "/boot": { ignore: true } } }); + assert.deepEqual(await alertIds(), [], "not offline for ten hours, 93% is under 95, and /boot is left alone"); + const detail = await (await call("GET", `/servers/${id}/detail`)).json(); + assert.equal(detail.server.thresholds.diskUsagePercent, 95); + const entry = (await db.select().from(auditLog)).filter((e) => e.action === "update" && e.targetId === String(id)).pop()!; + assert.deepEqual(JSON.parse(entry.detail!).thresholds, { serverOfflineMinutes: 600, diskUsagePercent: 95, mounts: { "/boot": { ignore: true } } }); + // the list shows them too, as an object + const row = (await (await call("GET", "/servers")).json()).servers.find((x: { id: number }) => x.id === id); + assert.equal(row.thresholds.serverOfflineMinutes, 600); + }); + + test("a disk's own limit brings its alert back, and the server's own 'offline after' is what /metrics uses", async () => { + await patch({ thresholds: { serverOfflineMinutes: 600, diskUsagePercent: 95, mounts: { "/": { percent: 90 }, "/boot": { ignore: true } } } }); + assert.deepEqual(await alertIds(), [`disk:disk:server:${id}:/`]); + assert.match(await buildMetrics(), /homelab_server_up\{server="laptop"\} 1/, "two hours quiet is fine when it may be quiet for ten"); + await patch({ thresholds: null }); + assert.match(await buildMetrics(), /homelab_server_up\{server="laptop"\} 0/, "and offline when it follows the global hour"); + assert.equal((await (await call("GET", `/servers/${id}/detail`)).json()).server.thresholds, null, "null goes back to following Settings"); + }); + + test("the forecast follows the server's own days (0 turns it off for it) and its ignored disks", async () => { + const day = 86_400_000; + const rows = []; + for (let i = 0; i < 10; i++) rows.push({ serverId: id, metric: "disk:/data", value: 30 + i * 4, recordedAt: new Date(Date.now() - (10 - i) * day).toISOString() }); + await db.insert(serverMetrics).values(rows); + await db.update(servers).set({ lastSeenAt: new Date().toISOString() }).where(eq(servers.id, id)); // online, so its figures count + const forecast = async () => { + invalidateAlertsCache(); + return (await getAlerts(true)).alerts.filter((a) => a.id.includes(`disk-forecast:${id}:`)).map((a) => a.id); + }; + assert.equal((await forecast()).length, 1, "66% and +4 a day: full in about nine days"); + await patch({ thresholds: { diskForecastDays: 0 } }); + assert.deepEqual(await forecast(), [], "0 turns it off for this server"); + await patch({ thresholds: { diskForecastDays: 60, mounts: { "/data": { ignore: true } } } }); + assert.deepEqual(await forecast(), [], "an ignored disk has no forecast either"); + await patch({ thresholds: { diskForecastDays: 60 } }); + assert.equal((await forecast()).length, 1, "a longer window still sees it"); + // and the stale-backup limit rides along, checked like the others + assert.equal((await patch({ thresholds: { backupStaleHours: 240 } })).status, 200); + assert.equal((await patch({ thresholds: { backupStaleHours: 5000 } })).status, 400); + }); +}); diff --git a/server/test/staleBackups.test.ts b/server/test/staleBackups.test.ts new file mode 100644 index 0000000..c00d46d --- /dev/null +++ b/server/test/staleBackups.test.ts @@ -0,0 +1,185 @@ +/** + * Guests whose newest backup has gone too old: the limit (Settings, or a server's own), the alert, what silences it, the daily list and /metrics. + * + * Run with: npm test --workspace server + */ +import { test, describe, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const dir = mkdtempSync(join(tmpdir(), "hm-stale-")); +process.env.DATABASE_PATH = join(dir, "test.sqlite"); +process.env.SESSION_DIR = join(dir, "sessions"); +process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); +process.env.HEALTH_STARTUP_GRACE_MS = "0"; + +const { db } = await import("../src/db/client.js"); +const { servers, maintenanceWindows, integrations } = await import("../src/db/schema.js"); +const { runMigrations } = await import("../src/db/migrate.js"); +const { buildCoverage, staleBackups, collectStaleBackups, getBackupCoverage, setCoverageLoaderForTests } = await import("../src/services/backupCoverage.js"); +const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js"); +const { buildMetrics } = await import("../src/services/metrics.js"); +const { updateSettings } = await import("../src/services/settingsStore.js"); +const { notifyStaleBackups } = await import("../src/services/notify.js"); +const { hashToken } = await import("../src/services/tokens.js"); + +await runMigrations(); +after(() => { + setCoverageLoaderForTests(null); + try { + rmSync(dir, { recursive: true, force: true }); + } catch { + // Windows keeps the database file locked until the process exits. + } +}); + +const HOUR = 3600_000; +const guest = (vmid: number, name: string) => ({ vmid, name, node: "pve1", type: "qemu" as const, status: "running", cpu: null, maxmem: null, mem: null, uptime: null }); +const allJob = { id: "j1", enabled: true, schedule: "02:00", storage: "pbs", node: null, allGuests: true, vmids: null, exclude: null }; +const px = (guests: ReturnType[]) => [{ integrationId: 1, integrationName: "Felicia", guests, jobs: [allJob], tasks: [] }]; +const snapAgo = (now: number, vmid: number, hours: number) => ({ backupType: "vm", backupId: String(vmid), backupTime: Math.floor((now - hours * HOUR) / 1000), verification: "ok" as const }); + +describe("when a guest is stale", () => { + const NOW = new Date("2026-10-17T12:00:00Z"); + const now = NOW.getTime(); + const input = (hoursAgo: number, extra: { servers?: Parameters[0]["servers"]; staleHours?: number } = {}) => ({ + proxmox: px([guest(100, "web")]), + pbs: [{ integrationName: "PBS", groups: [snapAgo(now, 100, hoursAgo)] }], + servers: extra.servers ?? [], + now: NOW, + staleHours: extra.staleHours, + }); + + test("a guest older than the limit is stale, and the limit it was judged by is on the row", () => { + const [g] = buildCoverage(input(60)); + assert.equal(g.status, "stale"); + assert.equal(g.staleAfterHours, 48); + assert.equal(buildCoverage(input(60, { staleHours: 72 }))[0].status, "ok"); + assert.equal(buildCoverage(input(60, { staleHours: 72 }))[0].staleAfterHours, 72); + }); + + test("a server that is this guest can have its own limit — longer for a weekly backup, or 0 for never", () => { + const linked = (staleHours: number | null) => [{ id: 7, name: "web", proxmoxIntegrationId: 1, proxmoxVmid: 100, staleHours }]; + assert.equal(buildCoverage(input(200, { servers: linked(240) }))[0].status, "ok", "eight days is fine for a limit of ten"); + assert.equal(buildCoverage(input(300, { servers: linked(240) }))[0].status, "stale"); + assert.equal(buildCoverage(input(5000, { servers: linked(0) }))[0].status, "ok", "0 means this one is never called stale"); + assert.equal(buildCoverage(input(200, { servers: linked(null) }))[0].status, "stale", "no limit of its own: the global one"); + // a failure is a failure whatever the limit says + const failed = buildCoverage({ ...input(1, { servers: linked(0) }), pbs: [{ integrationName: "PBS", groups: [{ ...snapAgo(now, 100, 1), verification: "failed" as const }] }] }); + assert.equal(failed[0].status, "failed"); + }); + + test("only stale guests are listed, oldest first, critical at three times the limit", () => { + const report = { + guests: buildCoverage({ + proxmox: px([guest(100, "web"), guest(101, "db"), guest(102, "fresh"), guest(103, "never")]), + pbs: [{ integrationName: "PBS", groups: [snapAgo(now, 100, 60), snapAgo(now, 101, 160), snapAgo(now, 102, 2)] }], + servers: [], + now: NOW, + }), + counts: { ok: 1, stale: 2, failed: 0, none: 0, unknown: 1, total: 4 }, + sources: [], + staleHours: 48, + generatedAt: NOW.toISOString(), + }; + const stale = staleBackups(report, NOW); + assert.deepEqual(stale.map((s) => [s.guest.name, s.severity]), [["db", "critical"], ["web", "warning"]]); + assert.equal(stale[0].key, "backup-stale:1:101"); + assert.match(stale[0].message, /db \(#101\) \[Felicia\]: its newest backup is 6 days old \(the limit is 48 h\)/); + assert.match(stale[1].message, /60 h old \(the limit is 48 h\)/); + }); + + test("it is silenced by maintenance on the Proxmox integration or on the server the guest is", () => { + const base = { guests: buildCoverage({ ...input(60), servers: [{ id: 7, name: "web", proxmoxIntegrationId: 1, proxmoxVmid: 100 }] }), counts: { ok: 0, stale: 1, failed: 0, none: 0, unknown: 0, total: 1 }, sources: [], staleHours: 48, generatedAt: NOW.toISOString() }; + assert.equal(staleBackups(base, NOW)[0].silenced, false); + assert.equal(staleBackups(base, NOW, new Set(["integration:1"]))[0].silenced, true); + assert.equal(staleBackups(base, NOW, new Set(["server:7"]))[0].silenced, true); + assert.equal(staleBackups(base, NOW, new Set(["server:8", "integration:2"]))[0].silenced, false); + }); +}); + +describe("through the alerts, the daily list and /metrics", () => { + before(async () => { + // The guests are Proxmox's, read from a stand-in; the integration they belong to still has to exist for a server to be linked to one. + await db.insert(integrations).values({ id: 1, type: "proxmox", name: "Felicia", baseUrl: "https://pve.example", enabled: false }); + }); + + // The alert pass judges by the real clock, so these backups are dated from it. + const fixture = () => { + const now = Date.now(); + return { + proxmox: px([guest(100, "web"), guest(101, "db"), guest(102, "weekly"), guest(103, "fine")]).map((p) => ({ ...p })), + pbs: [{ integrationName: "PBS", groups: [snapAgo(now, 100, 60), snapAgo(now, 101, 200), snapAgo(now, 102, 150), snapAgo(now, 103, 3)] }], + sources: [{ kind: "proxmox" as const, name: "Felicia", ok: true }, { kind: "pbs" as const, name: "PBS", ok: true }], + }; + }; + const staleAlerts = async () => { + invalidateAlertsCache(); + return (await getAlerts(true)).alerts.filter((a) => a.id.startsWith("backup:backup-stale:")); + }; + + test("each stale guest is an alert of the backup kind, with a link, a target and a severity", async () => { + setCoverageLoaderForTests(async () => fixture()); + const alerts = await staleAlerts(); + assert.deepEqual(alerts.map((a) => a.id).sort(), ["backup:backup-stale:1:100", "backup:backup-stale:1:101", "backup:backup-stale:1:102"]); + const db101 = alerts.find((a) => a.id.endsWith(":101"))!; + assert.equal(db101.category, "backup"); + assert.equal(db101.severity, "critical", "200 hours is more than three times 48"); + assert.equal(alerts.find((a) => a.id.endsWith(":100"))!.severity, "warning"); + assert.equal(db101.link, "/backups"); + assert.deepEqual(db101.target, { type: "integration", id: 1, name: "Felicia" }); + assert.equal(db101.silenced, false); + }); + + test("a server's own limit changes its guest's alert, and that server's maintenance silences it", async () => { + const [s] = await db.insert(servers).values({ name: "weekly", apiTokenHash: hashToken("t"), apiTokenPrefix: "t", proxmoxIntegrationId: 1, proxmoxVmid: 102, thresholds: JSON.stringify({ backupStaleHours: 240 }) }).returning(); + setCoverageLoaderForTests(async () => fixture()); + let alerts = await staleAlerts(); + assert.equal(alerts.some((a) => a.id.endsWith(":102")), false, "150 hours is fine for a limit of 240"); + await db.update(servers).set({ thresholds: JSON.stringify({ backupStaleHours: 100 }) }); + alerts = await staleAlerts(); + const weekly = alerts.find((a) => a.id.endsWith(":102"))!; + assert.ok(weekly, "150 hours is over a limit of 100"); + assert.deepEqual(weekly.target && { type: weekly.target.type, id: weekly.target.id }, { type: "server", id: s.id }, "it is the server that can be silenced"); + + await db.insert(maintenanceWindows).values({ targetType: "server", targetId: s.id, reason: "x", startedAt: new Date(Date.now() - 60_000).toISOString(), endsAt: new Date(Date.now() + 3600_000).toISOString() }); + assert.equal((await staleAlerts()).find((a) => a.id.endsWith(":102"))!.silenced, true); + await db.delete(maintenanceWindows); + await db.update(servers).set({ thresholds: null }); + }); + + test("the limit in Settings is what the others follow, and 0 turns the alerts off", async () => { + setCoverageLoaderForTests(async () => fixture()); + await updateSettings({ healthChecks: { backupStaleHours: 100 } }); + assert.deepEqual((await staleAlerts()).map((a) => a.id.split(":").pop()).sort(), ["101", "102"], "60 hours is fine for a limit of 100"); + await updateSettings({ healthChecks: { backupStaleHours: 0 } }); + assert.deepEqual(await staleAlerts(), []); + await updateSettings({ healthChecks: { backupStaleHours: 48 } }); + }); + + test("the daily list is the stale guests that aren't under maintenance", async () => { + setCoverageLoaderForTests(async () => fixture()); + const list = await collectStaleBackups(); + assert.equal(list.length, 3); + await db.insert(maintenanceWindows).values({ targetType: "integration", targetId: 1, reason: "x", startedAt: new Date(Date.now() - 60_000).toISOString(), endsAt: new Date(Date.now() + 3600_000).toISOString() }); + assert.deepEqual(await collectStaleBackups(), [], "the whole Proxmox integration is being worked on"); + await db.delete(maintenanceWindows); + await notifyStaleBackups([]); // nothing to say, nothing sent, and no error with no channel set up + await notifyStaleBackups(list); + }); + + test("/metrics says how many guests are in each state and how old each newest backup is, from what was last read", async () => { + setCoverageLoaderForTests(async () => fixture()); + invalidateAlertsCache(); + const text = await buildMetrics(); + assert.match(text, /homelab_backup_guests\{status="ok"\} 1/); + assert.match(text, /homelab_backup_guests\{status="stale"\} 3/); + const age = /homelab_backup_last_age_seconds\{guest="web",vmid="100",integration="Felicia"\} (\d+)/.exec(text); + assert.ok(age, "an age for each guest with a backup"); + assert.ok(Math.abs(Number(age[1]) - 60 * 3600) < 120, `about 60 hours, got ${age[1]}`); + // the page's own selector still asks for its own limit for every guest + assert.equal((await getBackupCoverage({ staleHours: 24 * 14 })).counts.stale, 0); + }); +}); diff --git a/server/test/updateRuns.test.ts b/server/test/updateRuns.test.ts new file mode 100644 index 0000000..3da812b --- /dev/null +++ b/server/test/updateRuns.test.ts @@ -0,0 +1,390 @@ +/** + * An update run: the maintenance window, the playbook, following it, waiting for the server to report again, and every way it can end. + * Semaphore is a stand-in that answers as told. + * + * Run with: npm test --workspace server + */ +import { test, describe, before, after, beforeEach } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { AddressInfo } from "node:net"; +import type { Server } from "node:http"; + +const dir = mkdtempSync(join(tmpdir(), "hm-updaterun-")); +process.env.DATABASE_PATH = join(dir, "test.sqlite"); +process.env.SESSION_DIR = join(dir, "sessions"); +process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); +process.env.HEALTH_STARTUP_GRACE_MS = "0"; + +const express = (await import("express")).default; +const { db } = await import("../src/db/client.js"); +const { users, servers, updateRuns, auditLog, maintenanceWindows } = await import("../src/db/schema.js"); +const { runMigrations } = await import("../src/db/migrate.js"); +const { startUpdateRun, advanceUpdateRuns, cancelUpdateRun, listUpdateRuns, setSemaphoreForTests, summarizeOutcome, UpdateRunError, HOST_WAIT_MS, MAX_POLL_ERRORS, RUN_LIMIT_MS } = await import("../src/services/updateRuns.js"); +const { listActiveWindows } = await import("../src/services/maintenance.js"); +const { getActivity } = await import("../src/services/serverActivity.js").then((m) => ({ getActivity: m.getServerActivity })); +const { updateRunsRouter } = await import("../src/routes/updateRuns.js"); +const { hashToken } = await import("../src/services/tokens.js"); +const { eq } = await import("drizzle-orm"); + +await runMigrations(); + +// ── a Semaphore that answers as told ── +interface Fake { + templates: { id: number; projectId: number; projectName: string; name: string; playbook: string; lastTask: null }[]; + runs: { projectId: number; templateId: number; options?: { limit?: string; message?: string } }[]; + task: { id: number; status: string; created: string | null; start: string | null; end: string | null }; + refuseToStart: boolean; + getTaskFails: boolean; +} +const fake: Fake = { templates: [], runs: [], task: { id: 41, status: "waiting", created: null, start: null, end: null }, refuseToStart: false, getTaskFails: false }; +const resetFake = () => { + fake.templates = [{ id: 7, projectId: 2, projectName: "Homelab", name: "Apply updates", playbook: "updates.yml", lastTask: null }]; + fake.runs = []; + fake.task = { id: 41, status: "waiting", created: null, start: null, end: null }; + fake.refuseToStart = false; + fake.getTaskFails = false; +}; +setSemaphoreForTests(async (integrationId) => + integrationId === 9 + ? { + ping: async () => ({ ok: true }), + listTemplatesWithStatus: async () => fake.templates, + checkTemplates: async () => ({ templates: fake.templates, failedProjectIds: [] }), + runTemplate: async (projectId, templateId, options) => { + if (fake.refuseToStart) throw new Error("Semaphore is having a moment"); + fake.runs.push({ projectId, templateId, options }); + return { ...fake.task } as never; + }, + getTask: async () => { + if (fake.getTaskFails) throw new Error("connection refused"); + return { ...fake.task } as never; + }, + } + : null, +); + +const T0 = new Date("2026-10-17T12:00:00.000Z"); +const at = (minutes: number) => new Date(T0.getTime() + minutes * 60_000); +const input = (serverId: number, extra: object = {}) => ({ serverId, integrationId: 9, projectId: 2, templateId: 7, limit: "elsa", windowMinutes: 90, actor: "Olle", ...extra }); +const runRow = async (id: number) => (await db.select().from(updateRuns).where(eq(updateRuns.id, id)))[0]; + +describe("starting a run", () => { + let serverId = 0; + before(async () => { + await db.insert(users).values([{ oidcSub: "admin", name: "Ada", role: "admin" }]); + const [s] = await db + .insert(servers) + .values({ name: "elsa", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t", lastSeenAt: T0.toISOString(), pendingUpdates: JSON.stringify({ manager: "apt", count: 12, security: 3, rebootRequired: false, reportedAt: T0.toISOString() }) }) + .returning(); + serverId = s.id; + }); + beforeEach(async () => { + resetFake(); + await db.delete(updateRuns); + await db.delete(maintenanceWindows); + }); + after(() => { + try { + rmSync(dir, { recursive: true, force: true }); + } catch { + // Windows keeps the database file locked until the process exits. + } + }); + + test("it puts the server under maintenance, starts the playbook limited to that server, and writes the run down", async () => { + const run = await startUpdateRun(input(serverId), T0); + assert.equal(run.status, "running"); + assert.equal(run.templateName, "Apply updates"); + assert.equal(run.updatesBefore, 12); + assert.equal(run.taskId, 41); + assert.equal(run.windowCreated, true); + assert.deepEqual(fake.runs.map((r) => [r.projectId, r.templateId, r.options?.limit]), [[2, 7, "elsa"]]); + assert.match(fake.runs[0].options!.message!, /Update run for elsa, started in Homelab Manager by Olle \(limited to elsa\)/); + + const windows = await listActiveWindows(T0); + assert.equal(windows.length, 1); + assert.equal(windows[0].targetId, serverId); + assert.equal(windows[0].reason, "Update run: Apply updates"); + assert.equal(Date.parse(windows[0].endsAt) - T0.getTime(), 90 * 60_000); + assert.equal(run.windowId, windows[0].id); + }); + + test("it won't start without something to wait for, with another run going, or on a Semaphore or template that isn't there", async () => { + const refused = async (promise: Promise, status: number, code: string) => { + await assert.rejects(promise, (e: unknown) => e instanceof UpdateRunError && e.status === status && e.code === code, code); + }; + await refused(startUpdateRun(input(99999), T0), 404, "server_not_found"); + const [never] = await db.insert(servers).values({ name: "ghost", apiTokenHash: hashToken("g"), apiTokenPrefix: "g" }).returning(); + await refused(startUpdateRun(input(never.id), T0), 400, "never_reported"); + await refused(startUpdateRun(input(serverId, { integrationId: 5 }), T0), 400, "no_semaphore"); + await refused(startUpdateRun(input(serverId, { templateId: 999 }), T0), 404, "template_not_found"); + assert.equal(fake.runs.length, 0, "nothing was started by any of those"); + assert.equal((await listActiveWindows(T0)).length, 0, "and no quiet was left behind"); + + await startUpdateRun(input(serverId), T0); + await refused(startUpdateRun(input(serverId), at(1)), 409, "already_running"); + assert.equal(fake.runs.length, 1); + }); + + test("if Semaphore won't start it the quiet is taken back, and no run is recorded", async () => { + fake.refuseToStart = true; + await assert.rejects(startUpdateRun(input(serverId), T0), (e: unknown) => e instanceof UpdateRunError && e.status === 502 && /Semaphore wouldn't start it/.test(e.message)); + assert.equal((await db.select().from(updateRuns)).length, 0); + assert.equal((await listActiveWindows(at(1))).length, 0, "the window it made is over"); + }); + + test("a window that was already there is extended and left to its owner; it is not ended when the run is", async () => { + await db.insert(maintenanceWindows).values({ targetType: "server", targetId: serverId, reason: "patch night", startedAt: T0.toISOString(), endsAt: at(30).toISOString(), createdBy: "Ada" }); + const run = await startUpdateRun(input(serverId), at(1)); + assert.equal(run.windowCreated, false); + fake.task = { ...fake.task, status: "error" }; + await advanceUpdateRuns(at(5)); + assert.equal((await runRow(run.id)).status, "failed"); + assert.equal((await listActiveWindows(at(6))).length, 1, "someone else's window still stands"); + }); +}); + +describe("following a run to the end", () => { + let serverId = 0; + before(async () => { + const [s] = await db.insert(servers).values({ name: "anna", apiTokenHash: hashToken("tok2"), apiTokenPrefix: "t2", lastSeenAt: T0.toISOString(), pendingUpdates: JSON.stringify({ manager: "apt", count: 12, security: 3, rebootRequired: false, reportedAt: T0.toISOString() }) }).returning(); + serverId = s.id; + }); + beforeEach(async () => { + resetFake(); + await db.delete(updateRuns); + await db.delete(maintenanceWindows); + await db.update(servers).set({ lastSeenAt: T0.toISOString(), pendingUpdates: JSON.stringify({ manager: "apt", count: 12, security: 3, rebootRequired: false, reportedAt: T0.toISOString() }) }).where(eq(servers.id, serverId)); + }); + + const reportBack = async (minutes: number, count: number, reboot: boolean) => + db.update(servers).set({ lastSeenAt: at(minutes).toISOString(), pendingUpdates: JSON.stringify({ manager: "apt", count, security: 0, rebootRequired: reboot, reportedAt: at(minutes).toISOString() }) }).where(eq(servers.id, serverId)); + + test("a good run: running, then waiting for the host, then done with what is left — and the window ends", async () => { + const run = await startUpdateRun(input(serverId), T0); + fake.task = { ...fake.task, status: "running" }; + await advanceUpdateRuns(at(1)); + let r = await runRow(run.id); + assert.equal(r.status, "running"); + assert.equal(r.taskStatus, "running"); + + fake.task = { ...fake.task, status: "success", end: at(8).toISOString() }; + await advanceUpdateRuns(at(9)); + r = await runRow(run.id); + assert.equal(r.status, "waiting_for_host"); + assert.equal(r.taskEndedAt, at(8).toISOString()); + assert.equal((await listActiveWindows(at(9))).length, 1, "still quiet while it comes back"); + + await advanceUpdateRuns(at(12)); + assert.equal((await runRow(run.id)).status, "waiting_for_host", "no report since the playbook ended"); + await reportBack(7, 12, false); // a report from before the playbook finished doesn't count + await advanceUpdateRuns(at(13)); + assert.equal((await runRow(run.id)).status, "waiting_for_host"); + + await reportBack(14, 2, true); + await advanceUpdateRuns(at(15)); + r = await runRow(run.id); + assert.equal(r.status, "done"); + assert.equal(r.updatesBefore, 12); + assert.equal(r.updatesAfter, 2); + assert.equal(r.rebootRequired, true); + assert.equal(r.message, "2 updates still waiting (10 updates installed); a restart is still needed."); + assert.equal(r.finishedAt, at(15).toISOString()); + assert.equal((await listActiveWindows(at(16))).length, 0, "the window it made is over"); + assert.equal((await advanceUpdateRuns(at(17))), 0, "a finished run is left alone"); + }); + + test("a playbook that fails ends the run and the window", async () => { + const run = await startUpdateRun(input(serverId), T0); + fake.task = { ...fake.task, status: "error" }; + await advanceUpdateRuns(at(3)); + const r = await runRow(run.id); + assert.equal(r.status, "failed"); + assert.match(r.message!, /The playbook failed in Semaphore \(run #41\)/); + assert.equal((await listActiveWindows(at(4))).length, 0); + for (const [status, words] of [["rejected", /rejected/], ["stopped", /stopped/]] as const) { + await db.delete(updateRuns); + const again = await startUpdateRun(input(serverId), at(10)); + fake.task = { ...fake.task, status }; + await advanceUpdateRuns(at(11)); + assert.match((await runRow(again.id)).message!, words); + } + }); + + test("a Semaphore that stops answering ends it after ten tries; one answer in between starts the count again", async () => { + const run = await startUpdateRun(input(serverId), T0); + fake.getTaskFails = true; + for (let i = 0; i < MAX_POLL_ERRORS - 1; i++) await advanceUpdateRuns(at(1 + i)); + assert.equal((await runRow(run.id)).status, "running"); + assert.equal((await runRow(run.id)).pollErrors, MAX_POLL_ERRORS - 1); + fake.getTaskFails = false; + fake.task = { ...fake.task, status: "running" }; + await advanceUpdateRuns(at(20)); + assert.equal((await runRow(run.id)).pollErrors, 0); + fake.getTaskFails = true; + for (let i = 0; i < MAX_POLL_ERRORS; i++) await advanceUpdateRuns(at(21 + i)); + const r = await runRow(run.id); + assert.equal(r.status, "failed"); + assert.match(r.message!, /Lost contact with Semaphore/); + }); + + test("a host that doesn't come back, or a run that never ends, is given up on, and the quiet ends so the alerts can speak", async () => { + const run = await startUpdateRun(input(serverId), T0); + fake.task = { ...fake.task, status: "success", end: at(5).toISOString() }; + await advanceUpdateRuns(at(6)); + await advanceUpdateRuns(at(5 + HOST_WAIT_MS / 60_000 - 1)); + assert.equal((await runRow(run.id)).status, "waiting_for_host"); + await advanceUpdateRuns(at(5 + HOST_WAIT_MS / 60_000 + 1)); + const r = await runRow(run.id); + assert.equal(r.status, "failed"); + assert.match(r.message!, /hasn't reported again in 30 minutes/); + assert.equal((await listActiveWindows(at(40))).length, 0); + + await db.delete(updateRuns); + const slow = await startUpdateRun(input(serverId), at(100)); + fake.task = { ...fake.task, status: "running" }; + await advanceUpdateRuns(new Date(at(100).getTime() + RUN_LIMIT_MS + 60_000)); + assert.match((await runRow(slow.id)).message!, /longer than three hours/); + }); + + test("a server whose agent doesn't report updates is done as soon as it reports at all", async () => { + await db.update(servers).set({ pendingUpdates: null }).where(eq(servers.id, serverId)); + const run = await startUpdateRun(input(serverId), T0); + fake.task = { ...fake.task, status: "success", end: at(5).toISOString() }; + await advanceUpdateRuns(at(6)); + await db.update(servers).set({ lastSeenAt: at(10).toISOString() }).where(eq(servers.id, serverId)); + await advanceUpdateRuns(at(11)); + const r = await runRow(run.id); + assert.equal(r.status, "done"); + assert.equal(r.updatesAfter, null); + assert.match(r.message!, /doesn't say what is waiting; no restart is needed/); + }); + + test("it can be stopped: the window ends, the Semaphore run is left alone, and a finished run can't be stopped again", async () => { + const run = await startUpdateRun(input(serverId), T0); + const stopped = await cancelUpdateRun(run.id, "Ada", at(2)); + assert.equal(stopped!.status, "cancelled"); + assert.match(stopped!.message!, /Stopped being followed by Ada/); + assert.equal((await listActiveWindows(at(3))).length, 0); + assert.equal(fake.runs.length, 1, "nothing was sent to Semaphore to stop it"); + await assert.rejects(cancelUpdateRun(run.id, "Ada", at(3)), (e: unknown) => e instanceof UpdateRunError && e.status === 409); + assert.equal(await cancelUpdateRun(99999, "Ada", at(3)), null); + }); + + test("the story is on the server's history, and a run that was under way when the app stopped is picked up again", async () => { + const run = await startUpdateRun(input(serverId), T0); + fake.task = { ...fake.task, status: "error" }; + await advanceUpdateRuns(at(3)); + const events = await getActivity(serverId, at(5)); + const titles = events.filter((e) => /Update run/.test(e.title)).map((e) => e.title); + // (the tests before this one wrote their own lines too, so this looks for the kinds rather than the number) + for (const wanted of ["Update run started", "Update run didn't finish", "Update run finished", "Update run stopped being followed"]) assert.ok(titles.includes(wanted), `${wanted} in ${JSON.stringify([...new Set(titles)])}`); + const failedLine = events.find((e) => e.title === "Update run didn't finish")!; + assert.equal(failedLine.tone, "bad"); + assert.match(failedLine.detail!, /The playbook failed in Semaphore/); + assert.ok((await db.select().from(auditLog)).some((a) => a.category === "update_run" && a.action === "failed")); + + // nothing is held in memory: a fresh pass over the table is all it takes + await db.delete(updateRuns); + const survivor = await startUpdateRun(input(serverId), at(20)); + fake.task = { ...fake.task, status: "success", end: at(22).toISOString() }; + assert.equal(await advanceUpdateRuns(at(23)), 1); + assert.equal((await runRow(survivor.id)).status, "waiting_for_host"); + void run; + }); + + test("the outcome is worded plainly for each case", () => { + assert.equal(summarizeOutcome(5, 0, false), "No updates are waiting now (5 updates installed); no restart is needed."); + assert.equal(summarizeOutcome(1, 0, true), "No updates are waiting now (1 update installed); a restart is still needed."); + assert.equal(summarizeOutcome(null, 3, false), "3 updates still waiting; no restart is needed."); + assert.equal(summarizeOutcome(3, 3, false), "3 updates still waiting; no restart is needed."); + assert.equal(summarizeOutcome(3, null, true), "The server reported again, but doesn't say what is waiting; a restart is still needed."); + }); +}); + +describe("the routes", () => { + let app: Server; + let base = ""; + let who = "operator"; + let serverId = 0; + const call = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) }); + + before(async () => { + await db.insert(users).values([ + { oidcSub: "operator", name: "Olle", role: "operator" }, + { oidcSub: "viewer", name: "Vera", role: "viewer" }, + ]); + const [s] = await db.insert(servers).values({ name: "zoe", apiTokenHash: hashToken("tok3"), apiTokenPrefix: "t3", lastSeenAt: T0.toISOString() }).returning(); + serverId = s.id; + const a = express(); + a.use(express.json()); + a.use((req, _res, next) => { + (req as unknown as { session: unknown }).session = { user: { sub: who } }; + next(); + }); + a.use("/api/update-runs", updateRunsRouter); + app = await new Promise((resolve) => { + const x = a.listen(0, "127.0.0.1", () => resolve(x)); + }); + base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/update-runs`; + }); + beforeEach(async () => { + resetFake(); + await db.delete(updateRuns); + await db.delete(maintenanceWindows); + }); + after(() => app.close()); + + const body = (extra: object = {}) => ({ serverId, integrationId: 9, projectId: 2, templateId: 7, limit: "zoe", windowMinutes: 60, ...extra }); + + test("an operator starts one and everyone signed in can see it; a viewer can't start or stop one", async () => { + who = "viewer"; + assert.equal((await call("POST", "/", body())).status, 403); + who = "operator"; + const res = await call("POST", "/", body()); + assert.equal(res.status, 201); + const { run } = await res.json(); + assert.equal(run.status, "running"); + assert.equal(run.active, true); + assert.equal(run.serverName, "zoe"); + assert.equal(run.template, "Apply updates"); + assert.equal(run.startedBy, "Olle"); + + who = "viewer"; + const list = await (await call("GET", "/")).json(); + assert.equal(list.runs.length, 1); + assert.equal(list.runs[0].id, run.id); + assert.equal((await (await call("GET", `/?serverId=${serverId}`)).json()).runs.length, 1); + assert.equal((await (await call("GET", "/?serverId=999")).json()).runs.length, 0); + assert.equal((await call("POST", `/${run.id}/cancel`)).status, 403); + + who = "operator"; + assert.equal((await call("POST", `/${run.id}/cancel`)).status, 200); + assert.equal((await call("POST", `/${run.id}/cancel`)).status, 409, "already over"); + assert.equal((await call("POST", "/99999/cancel")).status, 404); + }); + + test("what can't be done says why: a second run, a bad limit, a missing field, Semaphore refusing", async () => { + assert.equal((await call("POST", "/", body())).status, 201); + const again = await call("POST", "/", body()); + assert.equal(again.status, 409); + assert.equal((await again.json()).error, "already_running"); + await db.delete(updateRuns); + + for (const bad of [body({ limit: "" }), body({ limit: "--extra-vars x=1" }), body({ limit: "a b" }), body({ windowMinutes: 5 }), body({ windowMinutes: 9999 }), { serverId }]) { + assert.equal((await call("POST", "/", bad)).status, 400, JSON.stringify(bad)); + } + const limitReply = await (await call("POST", "/", body({ limit: "--extra-vars x=1" }))).json(); + assert.match(limitReply.message, /host name as it is in the Semaphore inventory/); + assert.equal(fake.runs.length, 1, "only the one that was fine reached Semaphore"); + + fake.refuseToStart = true; + const refused = await call("POST", "/", body()); + assert.equal(refused.status, 502); + assert.match((await refused.json()).message, /Semaphore wouldn't start it/); + assert.equal((await listUpdateRuns()).length, 0); + }); +}); diff --git a/web/src/App.tsx b/web/src/App.tsx index c18213f..d4fe582 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -141,7 +141,7 @@ function Shelled({ user }: { user: CurrentUser }) { } /> } /> } /> - } /> + } /> } /> } /> } /> diff --git a/web/src/api/client.ts b/web/src/api/client.ts index 9c49e24..3fdda8d 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -383,6 +383,7 @@ export interface NotificationEvents { pbsVerificationCheck: boolean; healthAlerts: boolean; automationAlerts: boolean; + updateRunResults: boolean; domainExpiryCheck: boolean; secretCheckTime: string; timezone: string; @@ -411,6 +412,8 @@ export interface HealthCheckSettings { domainWarnDays: number; /** A disk on course to be full within this many days is flagged on the Alerts page; 0 is off. */ diskForecastDays: number; + /** A Proxmox guest whose newest backup is older than this many hours is raised as stale; 0 is off. */ + backupStaleHours: number; } export interface QuietHoursSettings { @@ -531,6 +534,8 @@ export interface ServerRecord { agentVersion: number | null; /** Failed services, disk health and ZFS pools, as the agent says; null until it says anything. */ hostHealth: HostHealth | null; + /** This server's own alert limits; null when it follows Settings entirely. */ + thresholds: ServerThresholds | null; } /** The agent version a new install would be, per system; null when this app can't tell. */ @@ -557,6 +562,46 @@ export interface PendingUpdates { } /** One server on the Updates page. */ +/** A server's own alert limits; whatever is left out follows Settings. */ +export interface ServerThresholds { + serverOfflineMinutes?: number; + diskUsagePercent?: number; + diskForecastDays?: number; + backupStaleHours?: number; + mounts?: Record; +} + +export interface UpdateRun { + id: number; + serverId: number; + serverName: string | null; + integrationId: number; + template: string; + limit: string; + taskId: number | null; + /** Semaphore's own word for the run. */ + taskStatus: string | null; + status: "running" | "waiting_for_host" | "done" | "failed" | "cancelled"; + active: boolean; + message: string | null; + startedBy: string | null; + startedAt: string; + taskEndedAt: string | null; + finishedAt: string | null; + updatesBefore: number | null; + updatesAfter: number | null; + rebootRequired: boolean; +} + +export interface UpdateRunInput { + serverId: number; + integrationId: number; + projectId: number; + templateId: number; + limit: string; + windowMinutes: number; +} + export interface UpdatesServer { id: number; name: string; @@ -615,6 +660,8 @@ export interface ServerDetail { links: ServerLink[]; relations: { parent: ServerRelation | null; children: ServerRelation[] }; agentLatest: AgentLatest; + /** The limits in Settings, which this server follows wherever it has none of its own. */ + thresholdDefaults: HealthCheckSettings; } /** A token that opens /metrics for Prometheus, as the list shows it (never the secret). */ @@ -648,6 +695,8 @@ export type CoverageStatus = "ok" | "stale" | "failed" | "none" | "unknown"; /** One Proxmox guest and how well it is backed up. */ export interface CoverageGuest { + /** How old its newest backup may be before it counts as stale, in hours — the limit that applied to it (0 = never). */ + staleAfterHours: number; integrationId: number; integrationName: string; node: string; @@ -865,6 +914,8 @@ export interface ServerUpdateInput { parentServerId?: number | null; /** True says it runs on nothing, which stops the guess from its Proxmox node. */ noParent?: boolean; + /** Its own alert limits; null goes back to following Settings. */ + thresholds?: ServerThresholds | null; } /** A maintenance that repeats: "every Sunday 02:00–04:00 on these servers". */ @@ -1443,6 +1494,11 @@ export const api = { remove: (id: number) => request(`/api/event-webhooks/${id}`, { method: "DELETE" }), test: (id: number) => request<{ ok: boolean; result: string }>(`/api/event-webhooks/${id}/test`, { method: "POST" }), }, + updateRuns: { + list: (serverId?: number) => request<{ runs: UpdateRun[] }>(`/api/update-runs${serverId ? `?serverId=${serverId}` : ""}`), + start: (data: UpdateRunInput) => request<{ run: UpdateRun }>("/api/update-runs", { method: "POST", body: JSON.stringify(data) }), + cancel: (id: number) => request<{ status: string }>(`/api/update-runs/${id}/cancel`, { method: "POST" }), + }, backups: { coverage: (staleDays: number, refresh = false) => request(`/api/backups/coverage?staleDays=${staleDays}${refresh ? "&refresh=1" : ""}`), }, diff --git a/web/src/components/ServerThresholds.tsx b/web/src/components/ServerThresholds.tsx new file mode 100644 index 0000000..a786138 --- /dev/null +++ b/web/src/components/ServerThresholds.tsx @@ -0,0 +1,206 @@ +import { useState } from "react"; +import { api, type HealthCheckSettings, type ServerRecord, type ServerThresholds as Thresholds } from "../api/client"; +import { readableError } from "../utils/errors"; +import { toast } from "../utils/toast"; +import { ErrorNotice } from "./Notice"; + +type Field = "serverOfflineMinutes" | "diskUsagePercent" | "diskForecastDays" | "backupStaleHours"; + +const FIELDS: { key: Field; label: string; unit: string; min: number; max: number; hint: string; onlyForGuests?: boolean }[] = [ + { key: "serverOfflineMinutes", label: "Offline after", unit: "min", min: 15, max: 10080, hint: "How long without a report before this server counts as offline — longer for a laptop that is off at night." }, + { key: "diskUsagePercent", label: "Disk usage alert at", unit: "%", min: 50, max: 99, hint: "For every disk on this server that has no limit of its own below." }, + { key: "diskForecastDays", label: "Warn when a disk will fill within", unit: "days", min: 0, max: 90, hint: "0 turns the forecast off for this server." }, + { key: "backupStaleHours", label: "Its backup is too old after", unit: "hours", min: 0, max: 2160, hint: "For the Proxmox guest this server is. 0 means it is never called stale.", onlyForGuests: true }, +]; + +const has = (t: Thresholds | null) => !!t && (Object.keys(t).some((k) => k !== "mounts") || Object.keys(t.mounts ?? {}).length > 0); + +/** + * This server's own alert limits. Anything left empty follows Settings → Notifications, so changing the global limit still reaches every + * server that hasn't said otherwise. A disk can have a limit of its own, or be left alone altogether. Everyone can see what is set; only + * an administrator changes it. + */ +export default function ServerThresholds({ server, defaults, disks, canEdit, onChanged }: { server: ServerRecord; defaults: HealthCheckSettings; disks: string[]; canEdit: boolean; onChanged: () => void }) { + const own = server.thresholds; + const isGuest = server.proxmoxVmid !== null; + + const [draft, setDraft] = useState>(() => ({ + serverOfflineMinutes: String(own?.serverOfflineMinutes ?? ""), + diskUsagePercent: String(own?.diskUsagePercent ?? ""), + diskForecastDays: String(own?.diskForecastDays ?? ""), + backupStaleHours: String(own?.backupStaleHours ?? ""), + })); + const [mounts, setMounts] = useState>(() => + Object.fromEntries(Object.entries(own?.mounts ?? {}).map(([m, r]) => [m, { percent: String(r.percent ?? ""), ignore: !!r.ignore }])), + ); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + + // The disks it reports now, plus any that have a rule but aren't reported any more (so the rule can still be taken away). + const mountNames = [...new Set([...disks, ...Object.keys(mounts)])]; + + function build(): Thresholds | null { + const out: Thresholds = {}; + for (const f of FIELDS) { + const v = draft[f.key].trim(); + if (v !== "") out[f.key] = Number(v); + } + const rules: NonNullable = {}; + for (const [m, r] of Object.entries(mounts)) { + const rule: { percent?: number; ignore?: boolean } = {}; + if (r.percent.trim() !== "") rule.percent = Number(r.percent); + if (r.ignore) rule.ignore = true; + if (rule.percent !== undefined || rule.ignore) rules[m] = rule; + } + if (Object.keys(rules).length > 0) out.mounts = rules; + return Object.keys(out).length > 0 ? out : null; + } + + async function save(value: Thresholds | null) { + setBusy(true); + setError(null); + try { + await api.servers.update(server.id, { thresholds: value }); + toast.success(value ? "Saved. This server now follows its own limits where it has them." : "This server follows Settings again."); + onChanged(); + } catch (err) { + setError(readableError(err)); + } finally { + setBusy(false); + } + } + + if (!canEdit) { + if (!has(own)) return null; + return ( +
+
+

Alert limits

+
+
+
This server has limits of its own; everything else follows Settings.
+
    + {FIELDS.filter((f) => own?.[f.key] !== undefined).map((f) => ( +
  • + {f.label}: {own![f.key]} {f.unit} +
  • + ))} + {Object.entries(own?.mounts ?? {}).map(([m, r]) => ( +
  • + {m}: {r.ignore ? "left alone" : `alerts at ${r.percent}%`} +
  • + ))} +
+
+
+ ); + } + + return ( +
+
+

Alert limits

+ {has(own) && ( +
+ its own limits +
+ )} +
+
{ + e.preventDefault(); + void save(build()); + }} + > +
+

Leave a box empty to follow Settings → Notifications (shown inside the box). Only what you fill in is different for this server.

+
+ {FIELDS.filter((f) => !f.onlyForGuests || isGuest || own?.[f.key] !== undefined).map((f) => ( +
+ + setDraft((d) => ({ ...d, [f.key]: e.target.value }))} + /> +
{f.hint}
+
+ ))} +
+ + {mountNames.length > 0 && ( +
+
Disks
+
+ + + + + + + + + + {mountNames.map((m) => { + const r = mounts[m] ?? { percent: "", ignore: false }; + const set = (patch: Partial) => setMounts((all) => ({ ...all, [m]: { ...r, ...patch } })); + return ( + + + + + + ); + })} + +
DiskAlert at (%)Leave it alone
+ {m} + {!disks.includes(m) && (not reported now)} + + set({ percent: e.target.value })} + /> + + +
+
+
+ )} +
+ +
+
+
+ + {has(own) && ( + + )} +
+
+
+ ); +} diff --git a/web/src/components/UpdateRunDrawer.tsx b/web/src/components/UpdateRunDrawer.tsx new file mode 100644 index 0000000..3a1308b --- /dev/null +++ b/web/src/components/UpdateRunDrawer.tsx @@ -0,0 +1,157 @@ +import { useEffect, useState } from "react"; +import { Link } from "react-router-dom"; +import { api, type IntegrationSummary, type SemaphoreTemplate, type UpdatesServer } from "../api/client"; +import { confirmDialog } from "../utils/dialogs"; +import { readableError } from "../utils/errors"; +import { toast } from "../utils/toast"; +import DetailDrawer from "./DetailDrawer"; +import { BlockSkeleton } from "./Skeleton"; +import UpdatesBadge from "./UpdatesBadge"; + +interface Loaded { + integration: IntegrationSummary; + templates: SemaphoreTemplate[]; +} + +const WINDOWS = [ + { minutes: 45, label: "45 minutes" }, + { minutes: 90, label: "1½ hours" }, + { minutes: 180, label: "3 hours" }, + { minutes: 360, label: "6 hours" }, +]; + +const LAST_KEY = "hlm.updateRun.template"; +const remembered = () => { + try { + return localStorage.getItem(LAST_KEY) ?? ""; + } catch { + return ""; + } +}; + +/** + * "Update this server now": pick the playbook that applies updates, and Homelab Manager puts the server under maintenance, runs it for this + * server only, follows it, waits for the server to report again and says what is left and whether a restart is due. It asks before it + * starts, and says what it will do. + */ +export default function UpdateRunDrawer({ server, onClose, onStarted }: { server: UpdatesServer; onClose: () => void; onStarted: () => void }) { + const [loaded, setLoaded] = useState(null); + const [failed, setFailed] = useState(false); + const [choice, setChoice] = useState(remembered); + const [limit, setLimit] = useState(server.name); + const [minutes, setMinutes] = useState(90); + const [busy, setBusy] = useState(false); + + useEffect(() => { + let alive = true; + api.integrations + .list() + .then(async (res) => { + const sems = res.integrations.filter((i) => i.type === "semaphore" && i.enabled); + const all = await Promise.all(sems.map((integration) => api.integrations.semaphore.templates(integration.id).then((r): Loaded => ({ integration, templates: r.templates })).catch((): Loaded => ({ integration, templates: [] })))); + if (alive) setLoaded(all); + }) + .catch(() => alive && setFailed(true)); + return () => { + alive = false; + }; + }, []); + + const options = (loaded ?? []).flatMap((l) => + l.templates.map((t) => ({ key: `${l.integration.id}:${t.projectId}:${t.id}`, integration: l.integration, template: t, label: `${t.name} — ${t.projectName}${(loaded ?? []).length > 1 ? ` (${l.integration.name})` : ""}` })), + ); + const picked = options.find((o) => o.key === choice); + const count = server.updates?.count ?? null; + + async function start() { + if (!picked) return; + const ok = await confirmDialog({ + title: `Update ${server.name} now?`, + message: `This silences alerts for ${server.name} for up to ${WINDOWS.find((w) => w.minutes === minutes)?.label ?? `${minutes} minutes`}, runs "${picked.template.name}" in ${picked.integration.name} limited to ${limit.trim()}, and waits for the server to report again. The playbook may restart it.`, + confirmLabel: "Start the update", + }); + if (!ok) return; + setBusy(true); + try { + await api.updateRuns.start({ serverId: server.id, integrationId: picked.integration.id, projectId: picked.template.projectId, templateId: picked.template.id, limit: limit.trim(), windowMinutes: minutes }); + try { + localStorage.setItem(LAST_KEY, picked.key); + } catch { + // a remembered choice is only a convenience + } + toast.success(`Started updating ${server.name}.`); + onStarted(); + onClose(); + } catch (err) { + toast.error(readableError(err)); + } finally { + setBusy(false); + } + } + + return ( + +
+ + {count !== null && count > 0 && server.updates && server.updates.security !== null && {server.updates.security} of them security} +
+ + {failed ? ( +
Couldn't load the integrations.
+ ) : loaded === null ? ( + + ) : loaded.length === 0 ? ( +
+ There is no Semaphore connected. Add one under Integrations, with a playbook that applies updates, and this server can be updated from here. +
+ ) : options.length === 0 ? ( +
No templates found in Semaphore.
+ ) : ( + <> +
    +
  1. Alerts for this server are silenced (a maintenance window).
  2. +
  3. The playbook runs in Semaphore, for this server only.
  4. +
  5. Homelab Manager waits for the server to report again — it may be restarting.
  6. +
  7. It tells you what is still waiting and whether a restart is due, and the window ends.
  8. +
+
+ + +
+
+ + setLimit(e.target.value)} placeholder="e.g. web01" /> +
An update run is always for this one server, never the whole inventory.
+
+
+ + +
The window ends sooner, as soon as the run does.
+
+ + + )} +
+ ); +} diff --git a/web/src/components/UpdateRunsCard.tsx b/web/src/components/UpdateRunsCard.tsx new file mode 100644 index 0000000..740ed72 --- /dev/null +++ b/web/src/components/UpdateRunsCard.tsx @@ -0,0 +1,127 @@ +import { useEffect, useState } from "react"; +import { Link } from "react-router-dom"; +import { api, type UpdateRun } from "../api/client"; +import { confirmDialog } from "../utils/dialogs"; +import { readableError } from "../utils/errors"; +import { toast } from "../utils/toast"; +import TimeAgo from "./TimeAgo"; + +const TASK_WORDS: Record = { + waiting: "waiting to start in Semaphore", + starting: "starting in Semaphore", + running: "playbook running", + waiting_confirmation: "waiting for approval in Semaphore", + confirmed: "approved in Semaphore", + stopping: "stopping", +}; + +/** What a run is doing now, in words: the badge and the sentence beside it. */ +export function describeRun(run: UpdateRun): { label: string; badge: string } { + switch (run.status) { + case "running": + return { label: TASK_WORDS[run.taskStatus ?? ""] ?? "playbook running", badge: "bg-blue-lt text-blue" }; + case "waiting_for_host": + return { label: "waiting for the server to report again", badge: "bg-blue-lt text-blue" }; + case "done": + return { label: "finished", badge: run.rebootRequired ? "bg-yellow-lt text-yellow" : "bg-green-lt text-green" }; + case "failed": + return { label: "didn't finish", badge: "bg-red-lt text-red" }; + default: + return { label: "stopped being followed", badge: "bg-secondary-lt text-secondary" }; + } +} + +/** + * Update runs under way and recently finished: for each, where it has got to (quiet → playbook → server back → done), what is still waiting + * afterwards and whether a restart is due. Looks again every few seconds while anything is under way. Hidden when there has been none. + * `reloadKey` makes it look straight away (after a run was started elsewhere on the page). + */ +export default function UpdateRunsCard({ canEdit, reloadKey, onFinished }: { canEdit: boolean; reloadKey: number; onFinished?: () => void }) { + const [runs, setRuns] = useState(null); + const active = (runs ?? []).some((r) => r.active); + + useEffect(() => { + let alive = true; + const load = () => + api.updateRuns + .list() + .then((res) => { + if (!alive) return; + setRuns((before) => { + // A run that was under way and now isn't: the page's figures have probably changed. + if (before?.some((b) => b.active && !res.runs.find((r) => r.id === b.id)?.active)) onFinished?.(); + return res.runs; + }); + }) + .catch(() => {}); + void load(); + if (!active) return () => void (alive = false); + const timer = setInterval(load, 5000); + return () => { + alive = false; + clearInterval(timer); + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [reloadKey, active]); + + async function stop(run: UpdateRun) { + if ( + !(await confirmDialog({ + title: "Stop following this run?", + message: `The maintenance window on ${run.serverName} ends now. A playbook already started keeps running in Semaphore — it can't be stopped from here.`, + confirmLabel: "Stop following", + danger: true, + })) + ) + return; + try { + await api.updateRuns.cancel(run.id); + toast.success("No longer following it."); + setRuns((await api.updateRuns.list()).runs); + } catch (err) { + toast.error(readableError(err)); + } + } + + if (!runs || runs.length === 0) return null; + return ( +
+
+

Update runs

+
{active ? "checking every few seconds" : "most recent first"}
+
+
+ {runs.slice(0, 8).map((r) => { + const w = describeRun(r); + return ( +
+
+ + {r.serverName ?? `server #${r.serverId}`} + + {r.template} + {w.label} + {r.status === "done" && r.rebootRequired && restart needed} + + started by {r.startedBy ?? "someone"} + + {canEdit && r.active && ( + + )} +
+ {r.message &&
{r.message}
} + {r.active && ( + + ); + })} +
+
+ ); +} diff --git a/web/src/pages/ServerDetail.tsx b/web/src/pages/ServerDetail.tsx index b5836fe..bfdb4f6 100644 --- a/web/src/pages/ServerDetail.tsx +++ b/web/src/pages/ServerDetail.tsx @@ -19,6 +19,7 @@ import TimeAgo from "../components/TimeAgo"; import UptimeRibbon from "../components/UptimeRibbon"; import RunsOn from "../components/RunsOn"; import HostHealthCard from "../components/HostHealthCard"; +import ServerThresholds from "../components/ServerThresholds"; import RunPlaybook from "../components/RunPlaybook"; import RunbookPanel from "../components/RunbookPanel"; import UpdatesBadge from "../components/UpdatesBadge"; @@ -810,6 +811,7 @@ export default function ServerDetail({ user }: { user: CurrentUser }) { {hardware.source === "agent" && } + d.mount)} canEdit={isAdmin} onChanged={() => void loadDetail()} /> {(user.role === "admin" || user.role === "operator") && }
diff --git a/web/src/pages/Updates.tsx b/web/src/pages/Updates.tsx index c497c4b..1c50038 100644 --- a/web/src/pages/Updates.tsx +++ b/web/src/pages/Updates.tsx @@ -1,6 +1,6 @@ import { useEffect, useState } from "react"; import { Link } from "react-router-dom"; -import { api, type AgentLatest, type DockhandContainer, type UpdatesServer } from "../api/client"; +import { api, type AgentLatest, type CurrentUser, type DockhandContainer, type UpdatesServer } from "../api/client"; import { agentState, agentVersionWords } from "../utils/agent"; import SortableTh from "../components/SortableTh"; import TableToolbar from "../components/TableToolbar"; @@ -13,6 +13,8 @@ import { useTableSearch } from "../hooks/useTableSearch"; import { downloadCsv } from "../utils/csv"; import { readableError } from "../utils/errors"; import ExportTableButton from "../components/ExportTableButton"; +import UpdateRunDrawer from "../components/UpdateRunDrawer"; +import UpdateRunsCard from "../components/UpdateRunsCard"; interface Row { id: number; @@ -53,13 +55,27 @@ const DAY_MS = 86_400_000; * whether a restart is due, and how long they have been waiting) and the Docker containers that have a newer image. It only looks — * nothing here installs anything. */ -export default function Updates() { +export default function Updates({ user }: { user: CurrentUser }) { + const canRun = user.role === "admin" || user.role === "operator"; + // The server being updated now (its drawer is open), and a counter that makes the runs card look again after one was started. + const [updating, setUpdating] = useState(null); + const [runsKey, setRunsKey] = useState(0); const [servers, setServers] = useState(null); const [agentLatest, setAgentLatest] = useState(null); const [containers, setContainers] = useState<(DockhandContainer & { integrationName: string })[] | null>(null); const [error, setError] = useState(null); const [onlyWaiting, setOnlyWaiting] = useState(false); + function reload() { + api.servers + .updates() + .then((res) => { + setServers(res.servers); + setAgentLatest(res.agentLatest); + }) + .catch(() => {}); + } + useEffect(() => { let alive = true; api.servers @@ -125,7 +141,7 @@ export default function Updates() {

Updates

- What the agents say is waiting to be installed on each server, and the Docker containers that have a newer image. This page only looks — it installs nothing. To apply updates, run a playbook from the server's page. + What the agents say is waiting to be installed on each server, and the Docker containers that have a newer image. {canRun ? "Update… applies a server's updates through a Semaphore playbook: it silences the server, runs the playbook, waits for the server to report again and tells you what is left." : "Operators can apply a server's updates from here."}

@@ -143,6 +159,8 @@ export default function Updates() { ))}
+ +
label="Waiting since" sortKeyName="since" activeKey={sortKey} direction={sortDir} onSort={requestSort} /> label="Reported" sortKeyName="reportedAt" activeKey={sortKey} direction={sortDir} onSort={requestSort} /> label="Agent" sortKeyName="agentVersion" activeKey={sortKey} direction={sortDir} onSort={requestSort} /> + {canRun && Actions} {servers === null && !error && ( - + @@ -198,11 +217,18 @@ export default function Updates() { )} + {canRun && ( + + + + )} ))} {servers !== null && sorted?.length === 0 && ( - + {tableSearch.active || onlyWaiting ? "Nothing matches that." : "No server has reported yet."} @@ -262,6 +288,8 @@ export default function Updates() {
)} + + {updating && setUpdating(null)} onStarted={() => setRunsKey((k) => k + 1)} />} ); } diff --git a/web/src/pages/settings/NotificationSettings.tsx b/web/src/pages/settings/NotificationSettings.tsx index ffe5a75..f4739a2 100644 --- a/web/src/pages/settings/NotificationSettings.tsx +++ b/web/src/pages/settings/NotificationSettings.tsx @@ -54,6 +54,7 @@ const DEFAULT_NOTIFICATIONS: NotificationEvents = { pbsVerificationCheck: true, healthAlerts: true, automationAlerts: true, + updateRunResults: true, domainExpiryCheck: true, secretCheckTime: "08:00", timezone: "UTC", @@ -72,7 +73,7 @@ const DIGEST_DAYS: { value: number; label: string }[] = [ { value: 6, label: "Saturday" }, { value: 0, label: "Sunday" }, ]; -const DEFAULT_HEALTH_CHECKS: HealthCheckSettings = { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14 }; +const DEFAULT_HEALTH_CHECKS: HealthCheckSettings = { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48 }; type TestResult = { ok: boolean; message: string } | null; type EventKey = { [K in keyof NotificationEvents]: NotificationEvents[K] extends boolean ? K : never }[keyof NotificationEvents]; @@ -688,7 +689,16 @@ export default function NotificationSettings() {

Backups and updates

- {toggle("proxmoxBackupCheck", "Proxmox backup failed or a guest has no coverage")} + {toggle("proxmoxBackupCheck", "Proxmox backup failed, a guest has no coverage, or its newest backup is too old")} + setHealthChecks((h) => ({ ...h, backupStaleHours: v }))} + /> {toggle("pbsVerificationCheck", "Proxmox Backup Server snapshot failed verification")} {toggle("dockerUpdateCheck", "Docker image update available")}
@@ -713,6 +723,7 @@ export default function NotificationSettings() { onChange={(v) => setNotifications((n) => ({ ...n, integrationFailureThreshold: v }))} />
{toggle("automationAlerts", "Semaphore template or Gitea workflow run failed")}
+
{toggle("updateRunResults", "An update run you started finished, or didn't")}