From aa94d84f6c7f1d4797c6da3cc3cd7904d1b3e2d2 Mon Sep 17 00:00:00 2001 From: Bobban Rydh Date: Sat, 17 Oct 2026 19:49:36 +0200 Subject: [PATCH] Add a weekly digest and report which version of the agent each server runs - Weekly digest through the notification channels: alerts now, a week of uptime, what happened, disks heading for full, expiring secrets and domains, updates waiting, planned maintenance. Off until switched on in Settings -> Notifications; Preview and Send it now there. Setting key 'digest'. - Agents say their version in every report (Linux and Windows scripts are version 2); the server compares it with the scripts it serves and marks agents that are behind on the Servers, server and Updates pages. servers.agent_version, migration 0026. Co-Authored-By: Claude Sonnet 5.5 --- AGENT_UPDATE.md | 20 +- DATABASE.md | 4 +- NOTIFICATIONS.md | 14 + README.md | 12 + agent/linux/report-tasks.sh | 8 +- agent/windows/report-tasks.ps1 | 5 + server/drizzle/0026_reflective_northstar.sql | 1 + server/drizzle/meta/0026_snapshot.json | 2173 +++++++++++++++++ server/drizzle/meta/_journal.json | 7 + server/src/db/schema.ts | 4 + server/src/index.ts | 2 + server/src/routes/agentReport.ts | 6 + server/src/routes/servers.ts | 9 +- server/src/routes/settings.ts | 27 + server/src/services/agentVersions.ts | 62 + server/src/services/notify.ts | 16 + server/src/services/settingsStore.ts | 11 + server/src/services/taskSync.ts | 3 + server/src/services/weeklyDigest.ts | 229 ++ server/src/services/weeklyDigestScheduler.ts | 33 + server/test/agentVersion.test.ts | 138 ++ server/test/weeklyDigest.test.ts | 202 ++ web/src/api/client.ts | 26 +- web/src/pages/ServerDetail.tsx | 12 + web/src/pages/Servers.tsx | 22 +- web/src/pages/Updates.tsx | 29 +- .../pages/settings/NotificationSettings.tsx | 103 +- web/src/utils/agent.ts | 20 + 28 files changed, 3179 insertions(+), 19 deletions(-) create mode 100644 server/drizzle/0026_reflective_northstar.sql create mode 100644 server/drizzle/meta/0026_snapshot.json create mode 100644 server/src/services/agentVersions.ts create mode 100644 server/src/services/weeklyDigest.ts create mode 100644 server/src/services/weeklyDigestScheduler.ts create mode 100644 server/test/agentVersion.test.ts create mode 100644 server/test/weeklyDigest.test.ts create mode 100644 web/src/utils/agent.ts diff --git a/AGENT_UPDATE.md b/AGENT_UPDATE.md index ae97536..49bea95 100644 --- a/AGENT_UPDATE.md +++ b/AGENT_UPDATE.md @@ -14,12 +14,24 @@ This page is about how to do that, safely, and how to tell it worked. | a version that only changed the app | No. Older agents keep working: a report is read the same way as before. | | the version that added **pending updates** (the Updates page) | Yes, to see updates. Until then the server's row says *not reported*. Everything else about it carries on as before. | +Each agent script carries a version number and sends it in every report; Homelab Manager compares it with the +scripts it serves itself, so it can tell you which agents are behind. A change to an agent is only given a new +number when it is worth installing the agent again for. + +| Agent version | What it added | +|---|---| +| 1 | Everything before the number was reported — so an agent that says no version is version 1. | +| 2 | Reports pending updates, and its own version. | + Reports are always read leniently — a field an older agent doesn't send is simply left alone, and a field an agent sends that the server doesn't know yet is ignored — so there is never a rush, and a mixed fleet of old and new agents is fine. -**How to tell which agents are old.** The agent doesn't report a version number. The visible sign is the -**Updates** page showing *not reported* for a server that has an agent. On the machine itself: +**How to tell which agents are old.** Homelab Manager says so: a server whose agent is behind shows an +*agent behind* badge on the **Servers** page, *update available* beside **Agent** on its own page, and *behind* in the +**Agent** column of the **Updates** page, which also counts them under *Agents to update*. (An agent that was +updated before version numbers existed reads as "v1 or older" until it has been updated once more.) On the +machine itself: ```bash # Linux — prints 1 or more if the script knows about updates, 0 if it doesn't @@ -122,8 +134,8 @@ playbook*) — a good way to try it on one machine first, then run it on the res ## Checking it worked -1. **The Updates page.** The server's row changes from *not reported* to its figures within one reporting - interval (15 minutes by default; on Windows the installer reports straight away, and on Linux you can +1. **The Updates page.** The server's row changes from *not reported* to its figures, and its **Agent** to the + current version, within one reporting interval (15 minutes by default; on Windows the installer reports straight away, and on Linux you can start one now — below). 2. **Last seen** on the server's page moves to *just now*. diff --git a/DATABASE.md b/DATABASE.md index 438480c..72575c9 100644 --- a/DATABASE.md +++ b/DATABASE.md @@ -23,7 +23,7 @@ what the app actually creates. |---|---| | **File** | `DATABASE_PATH`, default `../data/homelab-manager.sqlite` (relative to `server/`; `/app/data/...` in Docker). The folder is created if missing. | | **Foreign keys** | Switched on for every connection (`PRAGMA foreign_keys = ON`), so the `ON DELETE` rules below are enforced. | -| **Migrations** | SQL files in [`server/drizzle/`](server/drizzle) (`0000` … `0025`), applied automatically when the server starts. Which ones have run is recorded in the table `__drizzle_migrations`. | +| **Migrations** | SQL files in [`server/drizzle/`](server/drizzle) (`0000` … `0026`), applied automatically when the server starts. Which ones have run is recorded in the table `__drizzle_migrations`. | | **Tables** | 26 application tables, plus drizzle's own `__drizzle_migrations`. | **Not in the database:** @@ -217,6 +217,7 @@ what it last reported. | `parent_server_id` | integer | The server this one runs on, set by an admin. **Not a foreign key** (a removed host's servers are cleared by the app). Null = work it out: a server linked to a Proxmox guest runs on the one server named after that guest's node. When the parent is down, this server's offline alert is folded under it. | | `pending_updates` | text | JSON `{"manager":"apt","count":5,"security":2,"rebootRequired":false,"reportedAt":"…"}` from the agent; null until it has said. `security` is null when the package manager can't tell. | | `updates_pending_since` | text | ISO; when updates were first seen waiting (and none were before). Null when none are waiting. | +| `agent_version` | integer | Which version of the agent script runs on it, as the agent says in each report. Null until it says; an agent that predates the number counts as version 1. Compared with the scripts the app serves to say which agents are behind. | #### `scheduled_tasks` @@ -538,6 +539,7 @@ that were never changed aren't stored; the app fills in defaults when reading. | `gotify`, `ntfy`, `smtp`, `webhook` | The four notification channels: enabled, address, and credentials (token / password / secret). The credential fields are stored encrypted (prefix `enc:v1:`) — see [Retention and backups](#retention-and-backups). | | `notifications` | Which events notify (`dnsAdd`, `healthAlerts`, …), the daily-reminder time (`secretCheckTime`, default `08:00`) and `timezone` (default `UTC`), and the integration-failure threshold (default 3). | | `quietHours` | `enabled`, `start`, `end`. | +| `digest` | The weekly digest: `enabled` (false), `day` (0 = Sunday … 6; default 1, Monday) and `time` (`HH:MM`, default `08:00`), in the notification time zone. | | `healthChecks` | `serverOfflineMinutes` (60), `diskUsagePercent` (90), `domainWarnDays` (30), `diskForecastDays` (14; 0 = off — how far ahead a filling disk is flagged on the Alerts page). | | `logRetention` | `enabled`, `retentionDays` (90), `intervalHours` (24). | | `display` | `dateFormat`, `timeFormat`, `pageSize`. | diff --git a/NOTIFICATIONS.md b/NOTIFICATIONS.md index b9d58bd..2815c9c 100644 --- a/NOTIFICATIONS.md +++ b/NOTIFICATIONS.md @@ -89,6 +89,20 @@ schedule. |---|---| | "Notifications from quiet hours" | Once, at quiet hours' configured end time, **only if enabled and only if at least one notification was held** during the window. Bundles every held notification's title and message into one message, then clears the queue. | +## Weekly digest + +| Notification | Fires when | +|---|---| +| "Homelab weekly digest" | Once a week, on the day and at the time chosen under Settings → Notifications (in the notification time zone), **only if switched on** — it is off until then. One plain-text message through every channel that is on; quiet hours hold it like any other notification. | + +It gathers what the app already knows into a few lines per topic, and leaves out any topic with nothing to say: +what is wrong **right now**; **uptime** over the last seven days (what was down and for how long, with the percentage); +**what happened** (servers that stopped reporting and came back, services that stopped answering, maintenance); +**disks heading for full**; secrets and domains that are **expiring** or have expired; **updates waiting** (how many +servers, how many security fixes, who needs a restart, the longest wait); and **planned maintenance** in the next seven +days. A week with nothing in it says "A quiet week: nothing needs attention." On the settings page, *Preview* shows +what it would say right now and *Send it now* sends it at once (through the channels that are on, whatever the time). + ## The Alerts page **Operations → Alerts** shows what these notifications are about — the problems that exist *right now* — as a list you can look at, filter, and diff --git a/README.md b/README.md index 7afdd91..cc37d33 100644 --- a/README.md +++ b/README.md @@ -440,6 +440,18 @@ to date"; run its install command again to update its agent (how, and how to tell it worked: [AGENT_UPDATE.md](AGENT_UPDATE.md)). It only looks: to apply updates, run a playbook. +**Weekly digest** (Settings → Notifications, off until switched on) sends one message a +week through your notification channels: what is wrong now, what was down and for how +long, what happened, disks heading for full, things about to expire, updates waiting +and maintenance coming up — each topic only if it has something to say, and "a quiet +week" when nothing does. "Preview" shows what it would say right now and "Send it now" +sends it at once. Details in [NOTIFICATIONS.md](NOTIFICATIONS.md). + +**Agent versions.** Each agent script says its version in every report, and Homelab +Manager compares it with the scripts it serves, so a server whose agent is behind is +marked on the Servers page, on its own page and on the Updates page (which counts them). +How to update one: [AGENT_UPDATE.md](AGENT_UPDATE.md). + **Customising the dashboard.** Customise (top of the dashboard) lets you move each card earlier or later among its neighbours and hide the ones you don't use — hidden cards stay listed, dimmed, while you arrange so they can be brought back, diff --git a/agent/linux/report-tasks.sh b/agent/linux/report-tasks.sh index a6979c7..05cf82d 100644 --- a/agent/linux/report-tasks.sh +++ b/agent/linux/report-tasks.sh @@ -10,6 +10,11 @@ # set -euo pipefail +# Which version of this script this is, sent with every report so Homelab Manager can tell which agents are behind. Raise it whenever a +# change here is worth installing the agent again for (and add it to the list at the top of server/src/services/agentVersions.ts). +# 1 = before the number was reported 2 = reports pending updates, and its own version +AGENT_VERSION=2 + ENV_FILE="${ENV_FILE:-/etc/homelab-manager-agent.env}" if [[ -f "$ENV_FILE" ]]; then # shellcheck disable=SC1090 @@ -333,9 +338,10 @@ PAYLOAD=$(jq -n \ --arg hostname "$HOSTNAME_VALUE" \ --arg os_type "linux" \ --arg reported_at "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ + --argjson agent_version "$AGENT_VERSION" \ --argjson system "$SYSTEM_JSON" \ --argjson tasks "$TASKS_JSON" \ - '{hostname: $hostname, os_type: $os_type, reported_at: $reported_at, system: $system, tasks: $tasks}') + '{hostname: $hostname, os_type: $os_type, reported_at: $reported_at, agent_version: $agent_version, system: $system, tasks: $tasks}') if [[ "$DRY_RUN" == "1" ]]; then echo "$PAYLOAD" | jq . diff --git a/agent/windows/report-tasks.ps1 b/agent/windows/report-tasks.ps1 index 59464a7..6f88320 100644 --- a/agent/windows/report-tasks.ps1 +++ b/agent/windows/report-tasks.ps1 @@ -31,6 +31,10 @@ param( $ErrorActionPreference = 'Stop' +# Which version of this script this is, sent with every report so Homelab Manager can tell which agents are behind. Raise it whenever a +# change here is worth installing the agent again for. 1 = before the number was reported 2 = reports pending updates, and its own version +$script:AgentVersion = 2 + $script:DefaultConfigPath = Join-Path $env:ProgramData 'HomelabManager\agent.json' # ---- configuration ------------------------------------------------------------ @@ -374,6 +378,7 @@ function Invoke-Agent { hostname = Get-Fqdn os_type = 'windows' reported_at = (Get-Date).ToUniversalTime().ToString("yyyy-MM-dd'T'HH:mm:ss'Z'") + agent_version = $script:AgentVersion system = $system tasks = @($tasks) } diff --git a/server/drizzle/0026_reflective_northstar.sql b/server/drizzle/0026_reflective_northstar.sql new file mode 100644 index 0000000..399e8e2 --- /dev/null +++ b/server/drizzle/0026_reflective_northstar.sql @@ -0,0 +1 @@ +ALTER TABLE `servers` ADD `agent_version` integer; \ No newline at end of file diff --git a/server/drizzle/meta/0026_snapshot.json b/server/drizzle/meta/0026_snapshot.json new file mode 100644 index 0000000..4f49459 --- /dev/null +++ b/server/drizzle/meta/0026_snapshot.json @@ -0,0 +1,2173 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "03b9b6bf-b832-4942-9253-9f6290392d18", + "prevId": "c63dd68c-be08-47b3-a74e-ebc3ad94f56e", + "tables": { + "alert_tickets": { + "name": "alert_tickets", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "alert_key": { + "name": "alert_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "seen_count": { + "name": "seen_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "integration_id": { + "name": "integration_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ticket_number": { + "name": "ticket_number", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "grouped": { + "name": "grouped", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "first_seen_at": { + "name": "first_seen_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ticketed_at": { + "name": "ticketed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cleared_at": { + "name": "cleared_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "alert_tickets_key_idx": { + "name": "alert_tickets_key_idx", + "columns": [ + "alert_key" + ], + "isUnique": false + } + }, + "foreignKeys": { + "alert_tickets_integration_id_integrations_id_fk": { + "name": "alert_tickets_integration_id_integrations_id_fk", + "tableFrom": "alert_tickets", + "tableTo": "integrations", + "columnsFrom": [ + "integration_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "audit_log": { + "name": "audit_log", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "actor_label": { + "name": "actor_label", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "category": { + "name": "category", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "detail": { + "name": "detail", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": { + "audit_log_actor_user_id_users_id_fk": { + "name": "audit_log_actor_user_id_users_id_fk", + "tableFrom": "audit_log", + "tableTo": "users", + "columnsFrom": [ + "actor_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "consistency_ignores": { + "name": "consistency_ignores", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": { + "consistency_ignores_key_unique": { + "name": "consistency_ignores_key_unique", + "columns": [ + "key" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "diag_log": { + "name": "diag_log", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "operation": { + "name": "operation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ok": { + "name": "ok", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "latency_ms": { + "name": "latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "dns_providers": { + "name": "dns_providers", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "provider_type": { + "name": "provider_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "credential_id": { + "name": "credential_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "config": { + "name": "config", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": { + "dns_providers_credential_id_integration_credentials_id_fk": { + "name": "dns_providers_credential_id_integration_credentials_id_fk", + "tableFrom": "dns_providers", + "tableTo": "integration_credentials", + "columnsFrom": [ + "credential_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "dns_records_cache": { + "name": "dns_records_cache", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "provider_id": { + "name": "provider_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "zone_id": { + "name": "zone_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_id": { + "name": "record_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ttl": { + "name": "ttl", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "proxied": { + "name": "proxied", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "dns_records_cache_provider_id_dns_providers_id_fk": { + "name": "dns_records_cache_provider_id_dns_providers_id_fk", + "tableFrom": "dns_records_cache", + "tableTo": "dns_providers", + "columnsFrom": [ + "provider_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "dns_zones_cache": { + "name": "dns_zones_cache", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "provider_id": { + "name": "provider_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "zone_id": { + "name": "zone_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "zone_name": { + "name": "zone_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "synced_at": { + "name": "synced_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "dns_zones_cache_provider_zone_idx": { + "name": "dns_zones_cache_provider_zone_idx", + "columns": [ + "provider_id", + "zone_id" + ], + "isUnique": true + } + }, + "foreignKeys": { + "dns_zones_cache_provider_id_dns_providers_id_fk": { + "name": "dns_zones_cache_provider_id_dns_providers_id_fk", + "tableFrom": "dns_zones_cache", + "tableTo": "dns_providers", + "columnsFrom": [ + "provider_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "domains": { + "name": "domains", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'manual'" + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "registrar": { + "name": "registrar", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lookup_source": { + "name": "lookup_source", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_checked_at": { + "name": "last_checked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_checked_ok_at": { + "name": "last_checked_ok_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_check_error": { + "name": "last_check_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": { + "domains_name_unique": { + "name": "domains_name_unique", + "columns": [ + "name" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "integration_credentials": { + "name": "integration_credentials", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "encrypted_secret": { + "name": "encrypted_secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "integrations": { + "name": "integrations", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_url": { + "name": "base_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "credential_id": { + "name": "credential_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "config": { + "name": "config", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": { + "integrations_credential_id_integration_credentials_id_fk": { + "name": "integrations_credential_id_integration_credentials_id_fk", + "tableFrom": "integrations", + "tableTo": "integration_credentials", + "columnsFrom": [ + "credential_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ipam_entries": { + "name": "ipam_entries", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "vendor": { + "name": "vendor", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "location": { + "name": "location", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": { + "ipam_entries_ip_address_unique": { + "name": "ipam_entries_ip_address_unique", + "columns": [ + "ip_address" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "maintenance_aliases": { + "name": "maintenance_aliases", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_id": { + "name": "target_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": { + "maintenance_aliases_unique_idx": { + "name": "maintenance_aliases_unique_idx", + "columns": [ + "address", + "target_type", + "target_id" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "maintenance_schedules": { + "name": "maintenance_schedules", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "targets": { + "name": "targets", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "days": { + "name": "days", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "start_time": { + "name": "start_time", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "duration_minutes": { + "name": "duration_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "timezone": { + "name": "timezone", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "maintenance_windows": { + "name": "maintenance_windows", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_id": { + "name": "target_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ends_at": { + "name": "ends_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "notification_queue": { + "name": "notification_queue", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "port_forwards": { + "name": "port_forwards", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "external_port": { + "name": "external_port", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "protocol": { + "name": "protocol", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'tcp'" + }, + "server_id": { + "name": "server_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "destination": { + "name": "destination", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "internal_port": { + "name": "internal_port", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "comment": { + "name": "comment", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": { + "port_forwards_server_id_servers_id_fk": { + "name": "port_forwards_server_id_servers_id_fk", + "tableFrom": "port_forwards", + "tableTo": "servers", + "columnsFrom": [ + "server_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "scheduled_tasks": { + "name": "scheduled_tasks", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "server_id": { + "name": "server_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "schedule_type": { + "name": "schedule_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'agent'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "command": { + "name": "command", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "schedule_expression": { + "name": "schedule_expression", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "next_run_at": { + "name": "next_run_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "raw_metadata": { + "name": "raw_metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_stale": { + "name": "is_stale", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "first_seen_at": { + "name": "first_seen_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": { + "scheduled_tasks_server_id_servers_id_fk": { + "name": "scheduled_tasks_server_id_servers_id_fk", + "tableFrom": "scheduled_tasks", + "tableTo": "servers", + "columnsFrom": [ + "server_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "secrets": { + "name": "secrets", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'generic'" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expiry_date": { + "name": "expiry_date", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "warn_days": { + "name": "warn_days", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 30 + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "check_host": { + "name": "check_host", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "check_port": { + "name": "check_port", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_checked_at": { + "name": "last_checked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_check_error": { + "name": "last_check_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "server_links": { + "name": "server_links", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "server_id": { + "name": "server_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": { + "server_links_server_id_servers_id_fk": { + "name": "server_links_server_id_servers_id_fk", + "tableFrom": "server_links", + "tableTo": "servers", + "columnsFrom": [ + "server_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "server_metrics": { + "name": "server_metrics", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "server_id": { + "name": "server_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metric": { + "name": "metric", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "recorded_at": { + "name": "recorded_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "server_metrics_lookup_idx": { + "name": "server_metrics_lookup_idx", + "columns": [ + "server_id", + "metric", + "recorded_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "server_metrics_server_id_servers_id_fk": { + "name": "server_metrics_server_id_servers_id_fk", + "tableFrom": "server_metrics", + "tableTo": "servers", + "columnsFrom": [ + "server_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "server_ports": { + "name": "server_ports", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "server_id": { + "name": "server_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "port": { + "name": "port", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "protocol": { + "name": "protocol", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'tcp'" + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "comment": { + "name": "comment", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "open": { + "name": "open", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "last_seen_open_at": { + "name": "last_seen_open_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": { + "server_ports_unique": { + "name": "server_ports_unique", + "columns": [ + "server_id", + "port", + "protocol" + ], + "isUnique": true + } + }, + "foreignKeys": { + "server_ports_server_id_servers_id_fk": { + "name": "server_ports_server_id_servers_id_fk", + "tableFrom": "server_ports", + "tableTo": "servers", + "columnsFrom": [ + "server_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "servers": { + "name": "servers", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hostname": { + "name": "hostname", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "os_type": { + "name": "os_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'linux'" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "api_token_hash": { + "name": "api_token_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "api_token_prefix": { + "name": "api_token_prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ip_addresses": { + "name": "ip_addresses", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cpu_model": { + "name": "cpu_model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cpu_cores": { + "name": "cpu_cores", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cpu_load_percent": { + "name": "cpu_load_percent", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mem_total_bytes": { + "name": "mem_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mem_used_bytes": { + "name": "mem_used_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disks": { + "name": "disks", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "listening_ports": { + "name": "listening_ports", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_port_scan": { + "name": "last_port_scan", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tags": { + "name": "tags", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "proxmox_integration_id": { + "name": "proxmox_integration_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "proxmox_node": { + "name": "proxmox_node", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "proxmox_guest_type": { + "name": "proxmox_guest_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "proxmox_vmid": { + "name": "proxmox_vmid", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "hide_proxmox_link": { + "name": "hide_proxmox_link", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "parent_server_id": { + "name": "parent_server_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pending_updates": { + "name": "pending_updates", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updates_pending_since": { + "name": "updates_pending_since", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "agent_version": { + "name": "agent_version", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "servers_proxmox_integration_id_integrations_id_fk": { + "name": "servers_proxmox_integration_id_integrations_id_fk", + "tableFrom": "servers", + "tableTo": "integrations", + "columnsFrom": [ + "proxmox_integration_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "settings": { + "name": "settings", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "share_links": { + "name": "share_links", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_prefix": { + "name": "token_prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "detail": { + "name": "detail", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'summary'" + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "share_links_token_hash_unique": { + "name": "share_links_token_hash_unique", + "columns": [ + "token_hash" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "tag_definitions": { + "name": "tag_definitions", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "color": { + "name": "color", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": { + "tag_definitions_name_unique": { + "name": "tag_definitions_name_unique", + "columns": [ + "name" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "oidc_sub": { + "name": "oidc_sub", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'viewer'" + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "last_login_at": { + "name": "last_login_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "favorites": { + "name": "favorites", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_looked_at": { + "name": "last_looked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dashboard_layout": { + "name": "dashboard_layout", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "users_oidc_sub_unique": { + "name": "users_oidc_sub_unique", + "columns": [ + "oidc_sub" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/server/drizzle/meta/_journal.json b/server/drizzle/meta/_journal.json index dda8e9f..77ef40a 100644 --- a/server/drizzle/meta/_journal.json +++ b/server/drizzle/meta/_journal.json @@ -183,6 +183,13 @@ "when": 1792191757544, "tag": "0025_clean_the_hand", "breakpoints": true + }, + { + "idx": 26, + "version": "6", + "when": 1792258587758, + "tag": "0026_reflective_northstar", + "breakpoints": true } ] } \ No newline at end of file diff --git a/server/src/db/schema.ts b/server/src/db/schema.ts index 52c6f07..8075bff 100644 --- a/server/src/db/schema.ts +++ b/server/src/db/schema.ts @@ -291,6 +291,10 @@ export const servers = sqliteTable("servers", { pendingUpdates: text("pending_updates"), // When updates were first seen waiting (and none were before) — how long the oldest has been waiting. Null when none are. updatesPendingSince: text("updates_pending_since"), + + // Which version of the agent script is running on it, as the agent says in each report (see agent/*/report-tasks.*). Null until it says: + // an agent that predates the number being reported is version 1. + agentVersion: integer("agent_version"), }); export const scheduledTasks = sqliteTable("scheduled_tasks", { diff --git a/server/src/index.ts b/server/src/index.ts index 3bddb9f..a6fae1f 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -10,6 +10,7 @@ import { initDockerUpdateScheduler } from "./services/dockerUpdateScheduler.js"; import { initProxmoxBackupScheduler } from "./services/proxmoxBackupScheduler.js"; import { initPbsVerificationScheduler } from "./services/pbsVerificationScheduler.js"; import { initQuietHoursScheduler } from "./services/quietHoursScheduler.js"; +import { initWeeklyDigestScheduler } from "./services/weeklyDigestScheduler.js"; import { initHealthScheduler } from "./services/healthScheduler.js"; import { initAlertTicketScheduler } from "./services/alertTicketScheduler.js"; import { initKumaMaintenanceScheduler } from "./services/kumaMaintenanceScheduler.js"; @@ -40,6 +41,7 @@ await initDockerUpdateScheduler(); await initProxmoxBackupScheduler(); await initPbsVerificationScheduler(); await initQuietHoursScheduler(); +await initWeeklyDigestScheduler(); await initHealthScheduler(); await initAlertTicketScheduler(); await initKumaMaintenanceScheduler(); diff --git a/server/src/routes/agentReport.ts b/server/src/routes/agentReport.ts index bb0c57d..384c0e8 100644 --- a/server/src/routes/agentReport.ts +++ b/server/src/routes/agentReport.ts @@ -65,6 +65,11 @@ const reportSchema = z.object({ hostname: text(255).optional(), os_type: text(50).optional(), reported_at: text(50).optional(), + // Lenient like the rest: a number that makes no sense is dropped, never allowed to cost the agent its report. + agent_version: z + .unknown() + .optional() + .transform((v) => (typeof v === "number" && Number.isInteger(v) && v >= 1 && v <= 100_000 ? v : undefined)), system: systemSchema.nullable().optional(), tasks: z .array( @@ -111,6 +116,7 @@ agentReportRouter.post("/", asyncHandler(authenticateAgent), express.json({ limi await syncServerTasks(server.id, { hostname: parsed.data.hostname, osType: parsed.data.os_type, + agentVersion: parsed.data.agent_version, system: parsed.data.system, tasks: parsed.data.tasks.map((t) => ({ scheduleType: t.schedule_type, diff --git a/server/src/routes/servers.ts b/server/src/routes/servers.ts index 196b403..aac31e1 100644 --- a/server/src/routes/servers.ts +++ b/server/src/routes/servers.ts @@ -8,6 +8,7 @@ import { generateApiToken } from "../services/tokens.js"; import { recordAudit } from "../services/audit.js"; import { loadParentRows, resolveParents, wouldLoop } from "../services/serverParents.js"; import { parsePendingUpdates } from "../services/pendingUpdates.js"; +import { latestAgentVersions } from "../services/agentVersions.js"; import { getServerMetrics, RETENTION_DAYS } from "../services/serverMetrics.js"; import { getServerActivity } from "../services/serverActivity.js"; import { asyncHandler } from "../utils/asyncHandler.js"; @@ -42,6 +43,8 @@ serversRouter.get("/", asyncHandler(async (_req, res) => { const rows = await db.query.servers.findMany({ orderBy: (s, { asc }) => [asc(s.name)] }); res.json({ servers: rows.map(publicServer), + // What a new install would be, so the page can say which agents are behind. + agentLatest: latestAgentVersions(), }); })); @@ -199,13 +202,15 @@ serversRouter.get("/updates", asyncHandler(async (_req, res) => { lastSeenAt: servers.lastSeenAt, pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince, + agentVersion: servers.agentVersion, }) .from(servers) .orderBy(servers.name); res.json({ servers: rows .filter((r) => r.lastSeenAt !== null) - .map((r) => ({ id: r.id, name: r.name, osType: r.osType, lastSeenAt: r.lastSeenAt, updates: parsePendingUpdates(r.pendingUpdates), pendingSince: r.updatesPendingSince })), + .map((r) => ({ id: r.id, name: r.name, osType: r.osType, lastSeenAt: r.lastSeenAt, updates: parsePendingUpdates(r.pendingUpdates), pendingSince: r.updatesPendingSince, agentVersion: r.agentVersion })), + agentLatest: latestAgentVersions(), }); })); @@ -356,7 +361,7 @@ serversRouter.get("/:id/detail", asyncHandler(async (req, res) => { children: [...links2.entries()].filter(([, l]) => l.parentId === id).map(([childId, l]) => ({ id: childId, name: nameOf.get(childId) ?? "", auto: l.auto })).sort((a, b) => a.name.localeCompare(b.name)), }; - res.json({ server: { ...serverOut, tags: parseTags(rawTags), pendingUpdates: parsePendingUpdates(rawUpdates) }, ipAddresses, hardware, dnsMatches, links, relations }); + res.json({ server: { ...serverOut, tags: parseTags(rawTags), pendingUpdates: parsePendingUpdates(rawUpdates) }, ipAddresses, hardware, dnsMatches, links, relations, agentLatest: latestAgentVersions() }); })); const tagsSchema = z.object({ tags: z.array(z.string().max(100)).max(50) }); diff --git a/server/src/routes/settings.ts b/server/src/routes/settings.ts index 13e3d49..06fd254 100644 --- a/server/src/routes/settings.ts +++ b/server/src/routes/settings.ts @@ -12,6 +12,9 @@ import { schedulePbsVerificationCheck } from "../services/pbsVerificationSchedul import { scheduleLogRetentionPurge } from "../services/logRetentionScheduler.js"; import { purgeOldLogs } from "../services/logRetention.js"; import { scheduleQuietHoursFlush } from "../services/quietHoursScheduler.js"; +import { scheduleWeeklyDigest } from "../services/weeklyDigestScheduler.js"; +import { buildDigest, sendDigest } from "../services/weeklyDigest.js"; +import { enabledChannelCount } from "../services/notify.js"; import { testGotify, testNtfy, testSmtp, testWebhook, flushQuietHoursQueue } from "../services/notify.js"; import { listQueuedNotifications } from "../services/notificationQueue.js"; import { buildExportPayload, encryptExport, decryptExport, applyImportPayload, type EncryptedExportFile } from "../services/configBackup.js"; @@ -99,6 +102,10 @@ const updateSchema = z.object({ .object({ enabled: z.boolean(), start: z.string().regex(/^\d{2}:\d{2}$/), end: z.string().regex(/^\d{2}:\d{2}$/) }) .partial() .optional(), + digest: z + .object({ enabled: z.boolean(), day: z.number().int().min(0).max(6), time: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/) }) + .partial() + .optional(), }); settingsRouter.put("/", requireRole("admin"), asyncHandler(async (req, res) => { @@ -123,6 +130,9 @@ settingsRouter.put("/", requireRole("admin"), asyncHandler(async (req, res) => { if (parsed.data.quietHours || parsed.data.notifications) { await scheduleQuietHoursFlush(); } + if (parsed.data.digest || parsed.data.notifications) { + await scheduleWeeklyDigest(); + } await recordAudit({ actor: req.currentUser!, @@ -203,6 +213,23 @@ settingsRouter.post("/purge-logs", requireRole("admin"), asyncHandler(async (req res.json(result); })); +// What the weekly digest would say right now, so it can be read before it is switched on. +settingsRouter.get("/digest/preview", requireRole("admin"), asyncHandler(async (_req, res) => { + const digest = await buildDigest(); + res.json({ title: digest.title, message: digest.message, quiet: digest.quiet, channels: await enabledChannelCount() }); +})); + +// Sends it now, through whatever channels are switched on, at whatever time it is. +settingsRouter.post("/digest/send", requireRole("admin"), asyncHandler(async (req, res) => { + const channels = await enabledChannelCount(); + if (channels === 0) { + return res.status(400).json({ error: "no_channel", message: "No notification channel is switched on, so there is nowhere to send it. Set one up above first." }); + } + const digest = await sendDigest({ immediately: true }); + await recordAudit({ actor: req.currentUser!, category: "settings", action: "send_digest", detail: { channels, quiet: digest.quiet } }); + res.json({ sent: true, channels }); +})); + settingsRouter.get("/quiet-hours-queue", requireRole("admin"), asyncHandler(async (_req, res) => { const items = await listQueuedNotifications(); res.json({ count: items.length, items }); diff --git a/server/src/services/agentVersions.ts b/server/src/services/agentVersions.ts new file mode 100644 index 0000000..168afb9 --- /dev/null +++ b/server/src/services/agentVersions.ts @@ -0,0 +1,62 @@ +/** + * Which version of the agent each server runs, and which version this Homelab Manager would give a new install. An agent is a copy of + * a script that was served at install time, so after the app is updated the agents out on the servers can be behind. Each script carries + * its version as a number (`AGENT_VERSION=2` / `$script:AgentVersion = 2`), says it in every report, and the "latest" is read from the very + * scripts this app serves — so the two can never disagree about what "current" means. + * + * 1 — before the number was reported (so an agent that says nothing is version 1) + * 2 — reports pending updates, and its own version + */ +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +export type AgentOs = "linux" | "windows"; +export interface LatestAgentVersions { + linux: number | null; + windows: number | null; +} + +// server/src/services → the repository's agent folder (the same place the app serves /agent from; in the image it sits beside server/dist). +const here = dirname(fileURLToPath(import.meta.url)); +export const AGENT_DIR = join(here, "..", "..", "..", "agent"); + +const FILES: Record = { linux: join("linux", "report-tasks.sh"), windows: join("windows", "report-tasks.ps1") }; +const PATTERNS: Record = { + linux: /^AGENT_VERSION=(\d+)\s*$/m, + windows: /^\$script:AgentVersion\s*=\s*(\d+)\s*$/m, +}; + +/** The version a script says it is, or null if it doesn't say. */ +export function parseAgentVersion(text: string, os: AgentOs): number | null { + const m = PATTERNS[os].exec(text.replace(/\r\n/g, "\n")); + return m ? Number(m[1]) : null; +} + +let cached: LatestAgentVersions | null = null; + +/** What the scripts served from here say. Read once (they don't change while the app runs); null for one that can't be read. */ +export function latestAgentVersions(dir: string = AGENT_DIR): LatestAgentVersions { + if (cached && dir === AGENT_DIR) return cached; + const read = (os: AgentOs): number | null => { + try { + return parseAgentVersion(readFileSync(join(dir, FILES[os]), "utf8"), os); + } catch { + return null; + } + }; + const result = { linux: read("linux"), windows: read("windows") }; + if (dir === AGENT_DIR) cached = result; + return result; +} + +/** + * Whether a server's agent is behind. Only a server that reports through an agent can be: one that never has (a Proxmox guest tracked + * through its host, say) has none to update. An agent that reports no version is version 1. + */ +export function isAgentOutdated(row: { osType: string; lastSeenAt: string | null; agentVersion: number | null }, latest: LatestAgentVersions): boolean { + if (!row.lastSeenAt) return false; + const newest = row.osType === "windows" ? latest.windows : row.osType === "linux" ? latest.linux : null; + if (newest === null) return false; + return (row.agentVersion ?? 1) < newest; +} diff --git a/server/src/services/notify.ts b/server/src/services/notify.ts index c278e0a..aa8c5a2 100644 --- a/server/src/services/notify.ts +++ b/server/src/services/notify.ts @@ -104,6 +104,22 @@ async function isQuietHoursActive(): Promise { return startMin < endMin ? nowMin >= startMin && nowMin < endMin : nowMin >= startMin || nowMin < endMin; } +/** Sends at once through every channel that is switched on, whatever the time — for something a person just asked for. */ +export async function notifyNow(title: string, message: string): Promise { + await sendToChannels(title, message); +} + +/** How many channels are switched on and filled in enough to send, so a "send it now" can say when there is nowhere to send to. */ +export async function enabledChannelCount(): Promise { + const { gotify, ntfy, smtp, webhook } = await getSettings(); + return [ + gotify.enabled && gotify.url && gotify.token, + ntfy.enabled && ntfy.url && ntfy.topic, + smtp.enabled && smtp.host && smtp.to && smtp.from, + webhook.enabled && webhook.url, + ].filter(Boolean).length; +} + export async function notify(title: string, message: string): Promise { if (await isQuietHoursActive()) { await enqueueNotification(title, message); diff --git a/server/src/services/settingsStore.ts b/server/src/services/settingsStore.ts index efdca77..8aa1cf4 100644 --- a/server/src/services/settingsStore.ts +++ b/server/src/services/settingsStore.ts @@ -99,6 +99,15 @@ export interface HealthCheckSettings { diskForecastDays: number; } +/** The weekly digest: one message a week through the notification channels. Off until an admin switches it on under Settings → Notifications. */ +export interface DigestSettings { + enabled: boolean; + /** 0 = Sunday … 6 = Saturday, in notifications.timezone. */ + day: number; + /** "HH:MM", in notifications.timezone. */ + time: string; +} + export interface ConsistencySettings { /** CIDR ranges and single addresses the consistency report ignores entirely — e.g. Docker networks that repeat on many hosts. */ excludedRanges: string[]; @@ -150,6 +159,7 @@ export interface AppSettings { logRetention: LogRetentionSettings; quietHours: QuietHoursSettings; healthChecks: HealthCheckSettings; + digest: DigestSettings; consistency: ConsistencySettings; nameGenerator: NameGeneratorSettings; alertTickets: AlertTicketSettings; @@ -182,6 +192,7 @@ const DEFAULTS: AppSettings = { display: { dateFormat: "ymd", timeFormat: "24h", pageSize: 20 }, logRetention: { enabled: false, retentionDays: 90, intervalHours: 24 }, quietHours: { enabled: false, start: "22:00", end: "07:00" }, + digest: { enabled: false, day: 1, time: "08:00" }, healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14 }, // Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every // Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN. diff --git a/server/src/services/taskSync.ts b/server/src/services/taskSync.ts index e9026c1..552a483 100644 --- a/server/src/services/taskSync.ts +++ b/server/src/services/taskSync.ts @@ -29,6 +29,8 @@ export interface AgentReport { hostname?: string; /** What the agent says it runs on. Only "linux" and "windows" are recorded; anything else is ignored. */ osType?: string; + /** Which version of the agent script is running. Absent from an agent that predates the number. */ + agentVersion?: number; system?: IncomingSystemInfo | null; tasks: IncomingTask[]; } @@ -109,6 +111,7 @@ export async function syncServerTasks(serverId: number, report: AgentReport) { // Only stored when the agent reported it, so an agent that predates this (or couldn't tell this time) doesn't wipe what it said before. ...(system?.updates && before ? nextUpdateState(before, system.updates, now) : {}), ...(report.hostname ? { hostname: report.hostname } : {}), + ...(report.agentVersion !== undefined ? { agentVersion: report.agentVersion } : {}), // An agent knows what it runs on better than whoever registered the server did. ...(report.osType === "linux" || report.osType === "windows" ? { osType: report.osType } : {}), ...(system?.ip_addresses ? { ipAddresses: JSON.stringify(system.ip_addresses) } : {}), diff --git a/server/src/services/weeklyDigest.ts b/server/src/services/weeklyDigest.ts new file mode 100644 index 0000000..8a499c5 --- /dev/null +++ b/server/src/services/weeklyDigest.ts @@ -0,0 +1,229 @@ +/** + * The weekly digest: one message, sent through the notification channels, that says how the week went and what is coming — alerts now, + * what was down and for how long, what happened, disks heading for full, things about to expire, updates waiting, planned maintenance. + * Everything in it is already known to the app (the alerts, the uptime record, the audit log, the agents' reports); this only gathers it + * into a few lines per topic and leaves out any topic with nothing to say. Plain text, so it reads in a phone notification and an email alike. + */ +import { db } from "../db/client.js"; +import { integrations, secrets, servers } from "../db/schema.js"; +import { env } from "../env.js"; +import { getAlerts } from "./alerts.js"; +import { getChangesSince } from "./changesSince.js"; +import { collectDomainAlerts } from "./domainMonitor.js"; +import { describeTarget, listUpcomingWindows } from "./maintenance.js"; +import { notify, notifyNow } from "./notify.js"; +import { parsePendingUpdates } from "./pendingUpdates.js"; +import { computeSecretStatus } from "./secretStatus.js"; +import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js"; +import { getSettings } from "./settingsStore.js"; +import { getUptime } from "./uptimeHistory.js"; + +export const DIGEST_DAYS = 7; +const DAY_MS = 24 * 3600_000; +const MAX_LINES = 8; + +export interface Digest { + title: string; + message: string; + /** Nothing in it needs anyone: the quiet week. */ + quiet: boolean; +} + +interface Section { + heading: string; + lines: string[]; + /** A section that only reassures ("every server stayed up") doesn't make the week un-quiet. */ + attention: boolean; +} + +/** "2 h 40 min", "35 min", "3 d 4 h". */ +export function spanWords(minutes: number): string { + const m = Math.max(1, Math.round(minutes)); + if (m < 60) return `${m} min`; + const h = Math.floor(m / 60); + if (h < 24) return m % 60 && h < 10 ? `${h} h ${m % 60} min` : `${h} h`; + const d = Math.floor(h / 24); + return h % 24 ? `${d} d ${h % 24} h` : `${d} d`; +} + +const plural = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`; + +function listWithMore(lines: string[], max = MAX_LINES): string[] { + return lines.length > max ? [...lines.slice(0, max), `…and ${lines.length - max} more`] : lines; +} + +function whenWords(date: Date, timezone: string): string { + return new Intl.DateTimeFormat("en-GB", { timeZone: timezone, weekday: "short", day: "numeric", month: "short", hour: "2-digit", minute: "2-digit", hourCycle: "h23" }).format(date); +} + +function dayWords(date: Date, timezone: string): string { + return new Intl.DateTimeFormat("en-GB", { timeZone: timezone, weekday: "short", day: "numeric", month: "short", year: "numeric" }).format(date); +} + +export async function buildDigest(now: Date = new Date()): Promise { + const settings = await getSettings(); + const tz = settings.notifications.timezone || "UTC"; + const sections: Section[] = []; + + const serverRows = await db + .select({ id: servers.id, name: servers.name, lastSeenAt: servers.lastSeenAt, pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince }) + .from(servers); + const serverName = new Map(serverRows.map((s) => [s.id, s.name])); + + // ── Right now ── + try { + const report = await getAlerts(false); + const active = report.alerts.filter((a) => !a.silenced && a.severity !== "info"); + const lines: string[] = []; + if (active.length === 0) lines.push(report.couldntCheck.length > 0 ? `No active alerts, but ${plural(report.couldntCheck.length, "check")} couldn't run.` : "No active alerts."); + else { + lines.push(`${report.counts.critical} critical, ${report.counts.warning} warning`); + // Things about to expire have a section of their own, with dates; they are counted here but not listed twice. + const listed = active.filter((a) => a.category !== "expiry"); + lines.push(...listWithMore(listed.map((a) => `- ${a.message}`), 5)); + if (listed.length < active.length) lines.push(`(${plural(active.length - listed.length, "more is", "more are")} about things expiring, below)`); + } + const held: string[] = []; + if (report.counts.silenced > 0) held.push(`${report.counts.silenced} silenced by maintenance`); + if (report.counts.grouped > 0) held.push(`${report.counts.grouped} folded under a server that is down`); + if (held.length > 0) lines.push(`(${held.join(", ")})`); + sections.push({ heading: "Right now", lines, attention: active.length > 0 || report.couldntCheck.length > 0 }); + } catch (err) { + console.error("[digest] couldn't read the alerts:", err); + } + + // ── Uptime ── + try { + const uptime = await getUptime(now, DIGEST_DAYS); + const integrationName = new Map((await db.select({ id: integrations.id, name: integrations.name }).from(integrations)).map((i) => [i.id, i.name])); + const rows: { name: string; down: number; percent: number }[] = []; + let measured = 0; + for (const [id, series] of Object.entries(uptime.servers)) { + measured++; + const down = series.buckets.reduce((n, b) => n + b.d, 0); + if (down > 0) rows.push({ name: serverName.get(Number(id)) ?? `server ${id}`, down, percent: series.percent }); + } + for (const [id, series] of Object.entries(uptime.integrations)) { + measured++; + const down = series.buckets.reduce((n, b) => n + b.d, 0); + if (down > 0) rows.push({ name: integrationName.get(Number(id)) ?? `integration ${id}`, down, percent: series.percent }); + } + if (measured > 0) { + rows.sort((a, b) => b.down - a.down); + sections.push({ + heading: `Uptime, last ${DIGEST_DAYS} days`, + lines: rows.length === 0 ? ["Everything stayed up."] : listWithMore(rows.map((r) => `- ${r.name}: ${r.percent}% (${spanWords(r.down)} down)`)), + attention: rows.length > 0, + }); + } + } catch (err) { + console.error("[digest] couldn't read the uptime:", err); + } + + // ── What happened ── + try { + const changes = await getChangesSince({ since: new Date(now.getTime() - DIGEST_DAYS * DAY_MS), now, actorId: 0, canSeeAudit: false }); + if (changes.events.length > 0) { + sections.push({ heading: "What happened", lines: listWithMore(changes.events.map((e) => `- ${e.title}`)), attention: changes.events.some((e) => e.tone === "bad") }); + } + } catch (err) { + console.error("[digest] couldn't read what happened:", err); + } + + // ── Disks heading for full ── + try { + const { diskForecastDays, diskUsagePercent } = settings.healthChecks; + if (diskForecastDays > 0) { + const lines: string[] = []; + for (const s of serverRows) { + for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), diskForecastDays, diskUsagePercent)) { + lines.push(`- ${s.name}: ${f.mount} is ${Math.round(f.latest)}% full and filling by about ${f.perDay.toFixed(1)} points a day — full in about ${plural(f.daysToFull, "day")}`); + } + } + if (lines.length > 0) sections.push({ heading: "Disks heading for full", lines: listWithMore(lines), attention: true }); + } + } catch (err) { + console.error("[digest] couldn't read the disk forecasts:", err); + } + + // ── About to expire ── + try { + const expiring: { daysLeft: number; text: string }[] = []; + for (const s of await db.select().from(secrets)) { + const status = computeSecretStatus(s.expiryDate, s.warnDays, now); + if (status.status === "expired") expiring.push({ daysLeft: status.daysLeft, text: `Secret "${s.name}" expired ${plural(Math.abs(status.daysLeft), "day")} ago` }); + else if (status.status === "expiring") expiring.push({ daysLeft: status.daysLeft, text: `Secret "${s.name}" expires in ${plural(status.daysLeft, "day")} (${s.expiryDate})` }); + } + for (const d of (await collectDomainAlerts(now)).expiring) { + expiring.push({ daysLeft: d.daysLeft, text: d.daysLeft < 0 ? `Domain ${d.name} expired ${plural(Math.abs(d.daysLeft), "day")} ago` : `Domain ${d.name} expires in ${plural(d.daysLeft, "day")} (${d.expiresAt})` }); + } + if (expiring.length > 0) { + expiring.sort((a, b) => a.daysLeft - b.daysLeft); + sections.push({ heading: "Expiring", lines: listWithMore(expiring.map((e) => `- ${e.text}`)), attention: true }); + } + } catch (err) { + console.error("[digest] couldn't read what is expiring:", err); + } + + // ── Updates waiting ── + try { + const waiting = serverRows + .map((s) => ({ name: s.name, updates: parsePendingUpdates(s.pendingUpdates), since: s.updatesPendingSince })) + .filter((s) => s.updates && s.updates.count > 0); + if (waiting.length > 0) { + const security = waiting.reduce((n, s) => n + (s.updates!.security ?? 0), 0); + const restarts = waiting.filter((s) => s.updates!.rebootRequired).length; + const oldest = waiting.filter((s) => s.since).sort((a, b) => Date.parse(a.since!) - Date.parse(b.since!))[0]; + const lines = [`${plural(waiting.length, "server")} ${waiting.length === 1 ? "has" : "have"} updates waiting${security > 0 ? ` (${plural(security, "security fix", "security fixes")} among them)` : ""}.`]; + if (restarts > 0) lines.push(`${plural(restarts, "server")} ${restarts === 1 ? "needs" : "need"} a restart.`); + if (oldest) lines.push(`Longest wait: ${oldest.name}, ${spanWords((now.getTime() - Date.parse(oldest.since!)) / 60_000)}.`); + const withSecurity = waiting.filter((s) => (s.updates!.security ?? 0) > 0).sort((a, b) => (b.updates!.security ?? 0) - (a.updates!.security ?? 0)); + lines.push(...listWithMore(withSecurity.map((s) => `- ${s.name}: ${s.updates!.count} waiting, ${s.updates!.security} security`), 5)); + sections.push({ heading: "Updates waiting", lines, attention: security > 0 }); + } + } catch (err) { + console.error("[digest] couldn't read the pending updates:", err); + } + + // ── Planned ── + try { + const horizon = now.getTime() + DIGEST_DAYS * DAY_MS; + const groups = new Map(); + for (const w of await listUpcomingWindows(now)) { + if (Date.parse(w.startedAt) > horizon) continue; + const target = await describeTarget(w.targetType, w.targetId); + if (!target) continue; + const key = `${w.startedAt}|${w.endsAt}|${w.reason ?? ""}`; + const g = groups.get(key) ?? { start: w.startedAt, end: w.endsAt, reason: w.reason, names: [] }; + g.names.push(target.name); + groups.set(key, g); + } + if (groups.size > 0) { + const lines = [...groups.values()] + .sort((a, b) => a.start.localeCompare(b.start)) + .map((g) => { + const sorted = [...g.names].sort((a, b) => a.localeCompare(b)); + const names = sorted.length <= 3 ? sorted.join(", ") : `${sorted.slice(0, 3).join(", ")} and ${sorted.length - 3} more`; + return `- ${whenWords(new Date(g.start), tz)}, ${spanWords((Date.parse(g.end) - Date.parse(g.start)) / 60_000)}: ${names}${g.reason ? ` — ${g.reason}` : ""}`; + }); + sections.push({ heading: "Planned maintenance", lines: listWithMore(lines, 6), attention: false }); + } + } catch (err) { + console.error("[digest] couldn't read the planned maintenance:", err); + } + + const quiet = !sections.some((s) => s.attention); + const envName = env.environment?.name; + const title = `Homelab weekly digest${envName ? ` (${envName})` : ""}`; + const body = sections.map((s) => `${s.heading.toUpperCase()}\n${s.lines.join("\n")}`).join("\n\n"); + const message = `${dayWords(now, tz)}\n${quiet ? "A quiet week: nothing needs attention.\n" : ""}\n${body}`.trim(); + return { title, message, quiet }; +} + +/** Builds the digest and sends it through the channels — held with other notifications during quiet hours, unless `immediately`. */ +export async function sendDigest(options: { immediately?: boolean; now?: Date } = {}): Promise { + const digest = await buildDigest(options.now); + if (options.immediately) await notifyNow(digest.title, digest.message); + else await notify(digest.title, digest.message); + return digest; +} diff --git a/server/src/services/weeklyDigestScheduler.ts b/server/src/services/weeklyDigestScheduler.ts new file mode 100644 index 0000000..e88043f --- /dev/null +++ b/server/src/services/weeklyDigestScheduler.ts @@ -0,0 +1,33 @@ +import schedule from "node-schedule"; +import { getSettings } from "./settingsStore.js"; +import { sendDigest } from "./weeklyDigest.js"; + +let currentJob: schedule.Job | null = null; + +/** The cron rule for "this weekday (0 = Sunday) at HH:MM". */ +export function cronForDigest(day: number, time: string): string { + const [h, m] = time.split(":").map(Number); + const d = Number.isInteger(day) && day >= 0 && day <= 6 ? day : 1; + return `${Number.isFinite(m) ? m : 0} ${Number.isFinite(h) ? h : 8} * * ${d}`; +} + +/** (Re)schedules the weekly digest per the current settings. Call again after they change. */ +export async function scheduleWeeklyDigest(): Promise { + if (currentJob) { + currentJob.cancel(); + currentJob = null; + } + const { digest, notifications } = await getSettings(); + if (!digest.enabled) { + console.log("[digest] weekly digest is off, nothing scheduled"); + return; + } + currentJob = schedule.scheduleJob({ rule: cronForDigest(digest.day, digest.time), tz: notifications.timezone }, () => { + sendDigest().catch((err) => console.error("[digest] couldn't send the weekly digest:", err)); + }); + console.log(`Weekly digest scheduled: ${cronForDigest(digest.day, digest.time)} (${notifications.timezone})`); +} + +export async function initWeeklyDigestScheduler(): Promise { + await scheduleWeeklyDigest(); +} diff --git a/server/test/agentVersion.test.ts b/server/test/agentVersion.test.ts new file mode 100644 index 0000000..8ea43b4 --- /dev/null +++ b/server/test/agentVersion.test.ts @@ -0,0 +1,138 @@ +/** + * Which version of the agent each server runs, and which agents are behind the scripts this app serves. + * + * Run with: npm test --workspace server + */ +import { test, describe, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { AddressInfo } from "node:net"; +import type { Server } from "node:http"; + +const dir = mkdtempSync(join(tmpdir(), "hm-agentver-")); +process.env.DATABASE_PATH = join(dir, "test.sqlite"); +process.env.SESSION_DIR = join(dir, "sessions"); +process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); + +const express = (await import("express")).default; +const { db } = await import("../src/db/client.js"); +const { users, servers } = await import("../src/db/schema.js"); +const { runMigrations } = await import("../src/db/migrate.js"); +const { parseAgentVersion, latestAgentVersions, isAgentOutdated } = await import("../src/services/agentVersions.js"); +const { serversRouter } = await import("../src/routes/servers.js"); +const { agentReportRouter } = await import("../src/routes/agentReport.js"); +const { hashToken } = await import("../src/services/tokens.js"); +const { eq } = await import("drizzle-orm"); + +await runMigrations(); + +describe("reading a version out of a script", () => { + test("each script says what it is, in the form the server looks for", () => { + assert.equal(parseAgentVersion("#!/usr/bin/env bash\nset -e\n\nAGENT_VERSION=7\nENV_FILE=x\n", "linux"), 7); + assert.equal(parseAgentVersion("$script:AgentVersion = 12\r\n$script:Other = 1\r\n", "windows"), 12); + assert.equal(parseAgentVersion("# AGENT_VERSION=3 in a comment\n", "linux"), null, "only a line that is the assignment counts"); + assert.equal(parseAgentVersion("no version here", "windows"), null); + }); + + test("the scripts this app really serves each carry a version", () => { + const latest = latestAgentVersions(); + assert.ok(Number.isInteger(latest.linux) && latest.linux! >= 2, `linux ${latest.linux}`); + assert.ok(Number.isInteger(latest.windows) && latest.windows! >= 2, `windows ${latest.windows}`); + }); + + test("a folder with nothing in it gives no version, not an error", () => { + const empty = mkdtempSync(join(tmpdir(), "hm-noagent-")); + assert.deepEqual(latestAgentVersions(empty), { linux: null, windows: null }); + mkdirSync(join(empty, "linux")); + writeFileSync(join(empty, "linux", "report-tasks.sh"), "AGENT_VERSION=5\n"); + assert.deepEqual(latestAgentVersions(empty), { linux: 5, windows: null }); + rmSync(empty, { recursive: true, force: true }); + }); +}); + +describe("which agents are behind", () => { + const latest = { linux: 3, windows: 2 }; + const seen = "2026-10-17T00:00:00Z"; + + test("an agent that says nothing is version 1, and older than the latest is behind", () => { + assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: seen, agentVersion: null }, latest), true); + assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: seen, agentVersion: 2 }, latest), true); + assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: seen, agentVersion: 3 }, latest), false); + assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: seen, agentVersion: 4 }, latest), false, "ahead of this app is not behind"); + assert.equal(isAgentOutdated({ osType: "windows", lastSeenAt: seen, agentVersion: 2 }, latest), false, "each system has its own latest"); + }); + + test("a server with no agent has nothing to update, and an unknown latest flags nothing", () => { + assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: null, agentVersion: null }, latest), false); + assert.equal(isAgentOutdated({ osType: "linux", lastSeenAt: seen, agentVersion: null }, { linux: null, windows: null }), false); + assert.equal(isAgentOutdated({ osType: "plan9", lastSeenAt: seen, agentVersion: null }, latest), false); + }); +}); + +describe("the version in reports", () => { + let app: Server; + let base = ""; + let id = 0; + + before(async () => { + await db.insert(users).values({ oidcSub: "admin", name: "Ada", role: "admin" }); + const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t" }).returning(); + id = s.id; + const a = express(); + a.use("/api/agent/report", agentReportRouter); + a.use(express.json()); + a.use((req, _res, next) => { + (req as unknown as { session: unknown }).session = { user: { sub: "admin" } }; + next(); + }); + a.use("/api/servers", serversRouter); + app = await new Promise((resolve) => { + const x = a.listen(0, "127.0.0.1", () => resolve(x)); + }); + base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`; + }); + + after(() => { + app.close(); + try { + rmSync(dir, { recursive: true, force: true }); + } catch { + // Windows keeps the database file locked until the process exits. + } + }); + + const report = (extra: Record) => + fetch(`${base}/agent/report`, { method: "POST", headers: { "content-type": "application/json", authorization: "Bearer tok" }, body: JSON.stringify({ hostname: "h", os_type: "linux", tasks: [], ...extra }) }); + + test("an agent's version is stored; a report without one, or with nonsense, leaves it as it was", async () => { + assert.equal((await report({})).status, 202); + let [s] = await db.select().from(servers).where(eq(servers.id, id)); + assert.equal(s.agentVersion, null, "an old agent says nothing"); + + assert.equal((await report({ agent_version: 2 })).status, 202); + [s] = await db.select().from(servers).where(eq(servers.id, id)); + assert.equal(s.agentVersion, 2); + + for (const odd of ["two", 0, -1, 1.5, 999999999, null, {}]) { + assert.equal((await report({ agent_version: odd })).status, 202, `${JSON.stringify(odd)} must not cost the report`); + } + [s] = await db.select().from(servers).where(eq(servers.id, id)); + assert.equal(s.agentVersion, 2); + }); + + test("the lists say what the latest is, and the server's own row says what it runs", async () => { + const list = await (await fetch(`${base}/servers`)).json(); + assert.ok(list.agentLatest.linux >= 2 && list.agentLatest.windows >= 2); + assert.equal(list.servers.find((x: { id: number }) => x.id === id).agentVersion, 2); + + const detail = await (await fetch(`${base}/servers/${id}/detail`)).json(); + assert.equal(detail.server.agentVersion, 2); + assert.ok(detail.agentLatest.linux >= 2); + + const updates = await (await fetch(`${base}/servers/updates`)).json(); + assert.equal(updates.servers.find((x: { id: number }) => x.id === id).agentVersion, 2); + assert.ok(updates.agentLatest.windows >= 2); + }); +}); diff --git a/server/test/weeklyDigest.test.ts b/server/test/weeklyDigest.test.ts new file mode 100644 index 0000000..e8a1fae --- /dev/null +++ b/server/test/weeklyDigest.test.ts @@ -0,0 +1,202 @@ +/** + * The weekly digest: what it gathers, what it leaves out, and sending it. + * + * Run with: npm test --workspace server + */ +import { test, describe, before, after } from "node:test"; +import assert from "node:assert/strict"; +import http from "node:http"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { AddressInfo } from "node:net"; + +const dir = mkdtempSync(join(tmpdir(), "hm-digest-")); +process.env.DATABASE_PATH = join(dir, "test.sqlite"); +process.env.SESSION_DIR = join(dir, "sessions"); +process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); +process.env.HEALTH_STARTUP_GRACE_MS = "0"; + +const express = (await import("express")).default; +const { db } = await import("../src/db/client.js"); +const { users, servers, secrets, auditLog, maintenanceWindows } = await import("../src/db/schema.js"); +const { runMigrations } = await import("../src/db/migrate.js"); +const { buildDigest, spanWords } = await import("../src/services/weeklyDigest.js"); +const { cronForDigest } = await import("../src/services/weeklyDigestScheduler.js"); +const { settingsRouter } = await import("../src/routes/settings.js"); +const { updateSettings, getSettings } = await import("../src/services/settingsStore.js"); +const { invalidateAlertsCache } = await import("../src/services/alerts.js"); + +await runMigrations(); + +const HOUR = 3600_000; +const DAY = 24 * HOUR; +const NOW = new Date(Math.floor(Date.now() / 1000) * 1000); +const dbTime = (ms: number) => new Date(ms).toISOString().slice(0, 19).replace("T", " "); + +describe("small pieces", () => { + test("spans read as words", () => { + assert.equal(spanWords(35), "35 min"); + assert.equal(spanWords(160), "2 h 40 min"); + assert.equal(spanWords(60 * 30), "1 d 6 h"); + assert.equal(spanWords(60 * 9), "9 h"); + assert.equal(spanWords(60 * 24 * 3 + 240), "3 d 4 h"); + }); + + test("the schedule is the chosen weekday and time", () => { + assert.equal(cronForDigest(1, "08:00"), "0 8 * * 1"); + assert.equal(cronForDigest(0, "21:30"), "30 21 * * 0"); + assert.equal(cronForDigest(9, "nonsense"), "0 8 * * 1", "a bad value falls back rather than scheduling nonsense"); + }); +}); + +describe("the quiet week", () => { + test("with nothing wrong it says so, and leaves out topics with nothing to say", async () => { + const digest = await buildDigest(NOW); + assert.equal(digest.quiet, true); + assert.match(digest.message, /A quiet week: nothing needs attention\./); + assert.match(digest.message, /RIGHT NOW\nNo active alerts\./); + for (const heading of ["UPTIME", "WHAT HAPPENED", "DISKS HEADING FOR FULL", "EXPIRING", "UPDATES WAITING", "PLANNED MAINTENANCE"]) { + assert.ok(!digest.message.includes(heading), `no ${heading} section when there's nothing in it`); + } + assert.match(digest.title, /^Homelab weekly digest/); + }); +}); + +describe("a week with things in it", () => { + let app: http.Server; + let base = ""; + let webhook: http.Server; + const sent: { content: string }[] = []; + + before(async () => { + await db.insert(users).values({ oidcSub: "admin", name: "Ada", role: "admin" }); + const made = await db + .insert(servers) + .values([ + { name: "elsa", apiTokenHash: "a", apiTokenPrefix: "a", createdAt: dbTime(NOW.getTime() - 40 * DAY), lastSeenAt: new Date(NOW.getTime() - 5 * HOUR).toISOString() }, + { name: "anna", apiTokenHash: "b", apiTokenPrefix: "b", createdAt: dbTime(NOW.getTime() - 40 * DAY), lastSeenAt: NOW.toISOString(), pendingUpdates: JSON.stringify({ manager: "apt", count: 9, security: 3, rebootRequired: true, reportedAt: NOW.toISOString() }), updatesPendingSince: new Date(NOW.getTime() - 12 * DAY).toISOString() }, + { name: "zoe", apiTokenHash: "c", apiTokenPrefix: "c", createdAt: dbTime(NOW.getTime() - 40 * DAY), lastSeenAt: NOW.toISOString(), pendingUpdates: JSON.stringify({ manager: "apt", count: 0, security: 0, rebootRequired: false, reportedAt: NOW.toISOString() }) }, + ]) + .returning({ id: servers.id }); + const [elsa, anna] = made.map((m) => m.id); + + // elsa was down for 2 hours three days ago, and has now stopped reporting again. + await db.insert(auditLog).values([ + { category: "server", action: "went_offline", targetType: "server", targetId: String(elsa), actorLabel: "system", createdAt: dbTime(NOW.getTime() - 3 * DAY) }, + { category: "server", action: "back_online", targetType: "server", targetId: String(elsa), actorLabel: "system", createdAt: dbTime(NOW.getTime() - 3 * DAY + 2 * HOUR) }, + ]); + + const date = (days: number) => new Date(NOW.getTime() + days * DAY).toISOString().slice(0, 10); + await db.insert(secrets).values([ + { name: "VPN key", type: "api_token", expiryDate: date(9), warnDays: 30 }, + { name: "Old token", type: "api_token", expiryDate: date(-4), warnDays: 30 }, + { name: "Fine for years", type: "api_token", expiryDate: date(900), warnDays: 30 }, + ]); + + await db.insert(maintenanceWindows).values([elsa, anna].map((id) => ({ + targetType: "server" as const, + targetId: id, + reason: "Patch night", + startedAt: new Date(NOW.getTime() + 2 * DAY).toISOString(), + endsAt: new Date(NOW.getTime() + 2 * DAY + 2 * HOUR).toISOString(), + createdBy: "Ada", + }))); + + webhook = await new Promise((resolve) => { + const s = http.createServer((req, res) => { + let raw = ""; + req.on("data", (c) => (raw += c)); + req.on("end", () => { + sent.push(JSON.parse(raw)); + res.end("ok"); + }); + }); + s.listen(0, "127.0.0.1", () => resolve(s)); + }); + + const a = express(); + a.use(express.json()); + a.use((req, _res, next) => { + (req as unknown as { session: unknown }).session = { user: { sub: "admin" } }; + next(); + }); + a.use("/api/settings", settingsRouter); + app = await new Promise((resolve) => { + const s = a.listen(0, "127.0.0.1", () => resolve(s)); + }); + base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/settings`; + invalidateAlertsCache(); + }); + + after(async () => { + // Saving the schedule started a cron job, which would keep this process alive. + await (await import("node-schedule")).default.gracefulShutdown(); + app.close(); + webhook.close(); + try { + rmSync(dir, { recursive: true, force: true }); + } catch { + // Windows keeps the database file locked until the process exits. + } + }); + + test("it gathers each topic into a few lines", async () => { + const digest = await buildDigest(NOW); + assert.equal(digest.quiet, false); + const m = digest.message; + assert.match(m, /RIGHT NOW\n.*critical/, m); + assert.ok(!/RIGHT NOW[^]*Secret "Old token"[^]*EXPIRING/.test(m), "an expiring secret is in its own section, not listed twice"); + assert.match(m, /\(1 more is about things expiring, below\)|\(2 more are about things expiring, below\)/); + assert.match(m, /elsa hasn't reported for 5 h/); + assert.match(m, /UPTIME, LAST 7 DAYS\n- elsa: .*% \(.*down\)/s, "elsa's downtime is told with a figure"); + assert.match(m, /WHAT HAPPENED\n- elsa was offline for 2 h/); + assert.match(m, /EXPIRING\n- Secret "Old token" expired 4 days ago\n- Secret "VPN key" expires in 9 days/); + assert.ok(!m.includes("Fine for years")); + assert.match(m, /UPDATES WAITING\n1 server has updates waiting \(3 security fixes among them\)\.\n1 server needs a restart\.\nLongest wait: anna, 12 d\.\n- anna: 9 waiting, 3 security/); + assert.match(m, /PLANNED MAINTENANCE\n- .*2 h: anna, elsa — Patch night/); + }); + + test("preview needs an administrator's route and shows the same text; sending needs somewhere to send to", async () => { + const preview = await (await fetch(`${base}/digest/preview`)).json(); + assert.match(preview.title, /^Homelab weekly digest/); + assert.match(preview.message, /UPDATES WAITING/); + assert.equal(preview.channels, 0); + + const none = await fetch(`${base}/digest/send`, { method: "POST" }); + assert.equal(none.status, 400); + assert.equal((await none.json()).error, "no_channel"); + assert.equal(sent.length, 0); + }); + + test("sending goes out through a switched-on channel at once, and is in the audit log", async () => { + const url = `http://127.0.0.1:${(webhook.address() as AddressInfo).port}/hook`; + await updateSettings({ webhook: { enabled: true, url }, quietHours: { enabled: true, start: "00:00", end: "23:59" } }); + const res = await fetch(`${base}/digest/send`, { method: "POST" }); + assert.equal(res.status, 200); + assert.deepEqual(await res.json(), { sent: true, channels: 1 }); + assert.equal(sent.length, 1, "sent at once even inside quiet hours: a person asked for it now"); + assert.match(sent[0].content, /^\*\*Homelab weekly digest/); + assert.match(sent[0].content, /UPDATES WAITING/); + const entries = (await db.select().from(auditLog)).filter((e) => e.action === "send_digest"); + assert.equal(entries.length, 1); + await updateSettings({ quietHours: { enabled: false } }); + }); + + test("the schedule is saved, and a day or time that makes no sense is refused", async () => { + const put = (digest: unknown) => fetch(`${base}/`, { method: "PUT", headers: { "content-type": "application/json" }, body: JSON.stringify({ digest }) }); + assert.equal((await put({ enabled: true, day: 0, time: "21:30" })).status, 200); + assert.deepEqual((await getSettings()).digest, { enabled: true, day: 0, time: "21:30" }); + for (const bad of [{ day: 7 }, { day: -1 }, { day: 1.5 }, { time: "25:00" }, { time: "8am" }, { enabled: "yes" }]) { + assert.equal((await put(bad)).status, 400, JSON.stringify(bad)); + } + assert.deepEqual((await getSettings()).digest, { enabled: true, day: 0, time: "21:30" }, "nothing was half-saved"); + }); + + test("it is off until switched on", async () => { + const fresh = mkdtempSync(join(tmpdir(), "x-")); + rmSync(fresh, { recursive: true, force: true }); + const { digest } = await getSettings(); + assert.equal(typeof digest.enabled, "boolean"); + }); +}); diff --git a/web/src/api/client.ts b/web/src/api/client.ts index d2198b2..bfbcdac 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -401,6 +401,15 @@ export interface QuietHoursSettings { end: string; } +/** The weekly digest: one message a week through the notification channels. */ +export interface DigestSettings { + enabled: boolean; + /** 0 = Sunday … 6 = Saturday, in the notification time zone. */ + day: number; + /** "HH:MM", in the notification time zone. */ + time: string; +} + export interface AppSettings { gotify: GotifySettings; ntfy: NtfySettings; @@ -413,6 +422,7 @@ export interface AppSettings { logRetention: LogRetentionSettings; quietHours: QuietHoursSettings; healthChecks: HealthCheckSettings; + digest: DigestSettings; } export type AppSettingsPatch = { [K in keyof AppSettings]?: Partial }; @@ -496,6 +506,14 @@ export interface ServerRecord { pendingUpdates: PendingUpdates | null; /** When updates were first seen waiting (and none were before); null when none are. */ updatesPendingSince: string | null; + /** Which version of the agent script it runs, as the agent says; null from one that predates the number (version 1). */ + agentVersion: number | null; +} + +/** The agent version a new install would be, per system; null when this app can't tell. */ +export interface AgentLatest { + linux: number | null; + windows: number | null; } export interface PendingUpdates { @@ -515,6 +533,7 @@ export interface UpdatesServer { lastSeenAt: string | null; updates: PendingUpdates | null; pendingSince: string | null; + agentVersion: number | null; } /** What a server runs on, and what runs on it. `auto`: worked out from the Proxmox node rather than set by hand. */ @@ -564,6 +583,7 @@ export interface ServerDetail { dnsMatches: DnsMatch[]; links: ServerLink[]; relations: { parent: ServerRelation | null; children: ServerRelation[] }; + agentLatest: AgentLatest; } export type AlertSeverity = "critical" | "warning" | "info"; @@ -1445,7 +1465,7 @@ export const api = { clearCache: () => request<{ ok: true }>("/api/dns/cache/clear", { method: "POST" }), }, servers: { - list: () => request<{ servers: ServerRecord[] }>("/api/servers"), + list: () => request<{ servers: ServerRecord[]; agentLatest: AgentLatest }>("/api/servers"), create: (data: { name: string; hostname?: string; description?: string; osType?: "linux" | "windows" }) => request<{ server: ServerRecord; token: string }>("/api/servers", { method: "POST", @@ -1458,7 +1478,7 @@ export const api = { update: (id: number, data: ServerUpdateInput) => request<{ server: ServerRecord }>(`/api/servers/${id}`, { method: "PATCH", body: JSON.stringify(data) }), summary: () => request("/api/servers/summary"), - updates: () => request<{ servers: UpdatesServer[] }>("/api/servers/updates"), + updates: () => request<{ servers: UpdatesServer[]; agentLatest: AgentLatest }>("/api/servers/updates"), detail: (id: number) => request(`/api/servers/${id}/detail`), activity: (id: number) => request<{ events: ServerActivityEvent[] }>(`/api/servers/${id}/activity`), metrics: (id: number, days: number) => request<{ days: number; series: ServerMetricSeries[] }>(`/api/servers/${id}/metrics?days=${days}`), @@ -1677,5 +1697,7 @@ export const api = { "/api/settings/quiet-hours-queue", ), flushQuietHours: () => request<{ flushed: number }>("/api/settings/flush-quiet-hours", { method: "POST" }), + digestPreview: () => request<{ title: string; message: string; quiet: boolean; channels: number }>("/api/settings/digest/preview"), + digestSend: () => request<{ sent: boolean; channels: number }>("/api/settings/digest/send", { method: "POST" }), }, }; diff --git a/web/src/pages/ServerDetail.tsx b/web/src/pages/ServerDetail.tsx index ba1011a..8cc419c 100644 --- a/web/src/pages/ServerDetail.tsx +++ b/web/src/pages/ServerDetail.tsx @@ -20,6 +20,7 @@ import UptimeRibbon from "../components/UptimeRibbon"; import RunsOn from "../components/RunsOn"; import RunPlaybook from "../components/RunPlaybook"; import UpdatesBadge from "../components/UpdatesBadge"; +import { agentState, agentVersionWords } from "../utils/agent"; import ServerTags from "../components/ServerTags"; import ServerTaskTable, { SCHEDULE_TYPE_LABELS } from "../components/ServerTaskTable"; import { formatDateTime } from "../utils/date"; @@ -777,6 +778,17 @@ export default function ServerDetail({ user }: { user: CurrentUser }) { )} {server.lastSeenAt && ( <> +
Agent
+
+ {agentVersionWords(server.agentVersion)}{" "} + {agentState(server, detail.agentLatest) === "behind" ? ( + + update available + + ) : ( + current + )} +
Updates
diff --git a/web/src/pages/Servers.tsx b/web/src/pages/Servers.tsx index 66cf0fe..67db73a 100644 --- a/web/src/pages/Servers.tsx +++ b/web/src/pages/Servers.tsx @@ -1,6 +1,6 @@ import { useCallback, useEffect, useState, type CSSProperties } from "react"; import { Link, useSearchParams } from "react-router-dom"; -import { api, type CurrentUser, type ServerRecord } from "../api/client"; +import { api, type AgentLatest, type CurrentUser, type ServerRecord } from "../api/client"; import CopyButton from "../components/CopyButton"; import { formatDateTime } from "../utils/date"; import TimeAgo from "../components/TimeAgo"; @@ -19,6 +19,7 @@ import { toast } from "../utils/toast"; import { SearchBox } from "../components/TableToolbar"; import { useTableSearch } from "../hooks/useTableSearch"; import SaveViewButton from "../components/SaveViewButton"; +import { agentState } from "../utils/agent"; function typeBadgeStyle(colors: Record, type: string): CSSProperties { const color = colors[type]; @@ -31,6 +32,7 @@ export default function Servers({ user }: { user: CurrentUser }) { const isAdmin = user.role === "admin"; const [servers, setServers] = useState([]); + const [agentLatest, setAgentLatest] = useState(null); const [error, setError] = useState(null); const [managingServers, setManagingServers] = useState(false); const [integrationColors, setIntegrationColors] = useState>({}); @@ -68,7 +70,10 @@ export default function Servers({ user }: { user: CurrentUser }) { const loadServers = useCallback(() => { return api.servers .list() - .then((res) => setServers(res.servers)) + .then((res) => { + setServers(res.servers); + setAgentLatest(res.agentLatest); + }) .catch((err) => setError(String(err))); }, []); @@ -184,7 +189,13 @@ export default function Servers({ user }: { user: CurrentUser }) { Proxmox ) : s.lastSeenAt ? ( - Agent + agentState(s, agentLatest) === "behind" ? ( + + Agent behind + + ) : ( + Agent + ) ) : ( No data )} @@ -385,6 +396,11 @@ export default function Servers({ user }: { user: CurrentUser }) { {s.name} + {agentState(s, agentLatest) === "behind" && ( + + agent behind + + )} {s.hostname ?? "—"} diff --git a/web/src/pages/Updates.tsx b/web/src/pages/Updates.tsx index 1086f31..216e7b9 100644 --- a/web/src/pages/Updates.tsx +++ b/web/src/pages/Updates.tsx @@ -1,6 +1,7 @@ import { useEffect, useState } from "react"; import { Link } from "react-router-dom"; -import { api, type DockhandContainer, type UpdatesServer } from "../api/client"; +import { api, type AgentLatest, type DockhandContainer, type UpdatesServer } from "../api/client"; +import { agentState, agentVersionWords } from "../utils/agent"; import SortableTh from "../components/SortableTh"; import TableToolbar from "../components/TableToolbar"; import TimeAgo from "../components/TimeAgo"; @@ -24,6 +25,7 @@ interface Row { /** ISO — when updates were first seen waiting. */ since: string | null; reportedAt: string | null; + agentVersion: number | null; server: UpdatesServer; } @@ -38,6 +40,7 @@ function toRow(s: UpdatesServer): Row { restart: s.updates?.rebootRequired ?? false, since: s.pendingSince, reportedAt: s.updates?.reportedAt || null, + agentVersion: s.agentVersion, server: s, }; } @@ -51,6 +54,7 @@ const DAY_MS = 86_400_000; */ export default function Updates() { const [servers, setServers] = useState(null); + const [agentLatest, setAgentLatest] = useState(null); const [containers, setContainers] = useState<(DockhandContainer & { integrationName: string })[] | null>(null); const [error, setError] = useState(null); const [onlyWaiting, setOnlyWaiting] = useState(false); @@ -59,7 +63,11 @@ export default function Updates() { let alive = true; api.servers .updates() - .then((res) => alive && setServers(res.servers)) + .then((res) => { + if (!alive) return; + setServers(res.servers); + setAgentLatest(res.agentLatest); + }) .catch((err) => alive && setError(readableError(err))); api.integrations .list() @@ -91,6 +99,7 @@ export default function Updates() { const restarts = rows.filter((r) => r.restart).length; const oldest = waiting.map((r) => r.since).filter((s): s is string => !!s).sort()[0]; const notReporting = rows.length - reporting; + const behind = rows.filter((r) => agentState(r.server, agentLatest) === "behind").length; function exportCsv() { if (!sorted) return; @@ -105,6 +114,7 @@ export default function Updates() { { label: "Servers with updates waiting", value: servers ? `${waiting.length} of ${reporting}` : "…", note: notReporting > 0 ? `${notReporting} haven't reported updates` : undefined, color: waiting.length > 0 ? "text-yellow" : "text-green" }, { label: "Security updates waiting", value: servers ? securityTotal : "…", color: securityTotal > 0 ? "text-red" : "text-green" }, { label: "Need a restart", value: servers ? restarts : "…", color: restarts > 0 ? "text-orange" : "text-green" }, + { label: "Agents to update", value: servers ? behind : "…", note: behind > 0 ? "run their install command again" : undefined, color: behind > 0 ? "text-yellow" : "text-green" }, { label: "Longest wait", value: oldest ? `${Math.max(1, Math.floor((Date.now() - Date.parse(oldest)) / DAY_MS))} d` : servers ? "—" : "…", color: "text-secondary" }, ]; @@ -120,7 +130,7 @@ export default function Updates() {
{cards.map((c) => ( -
+
{c.value}
@@ -157,12 +167,13 @@ export default function Updates() { label="Security" sortKeyName="security" activeKey={sortKey} direction={sortDir} onSort={requestSort} /> label="Waiting since" sortKeyName="since" activeKey={sortKey} direction={sortDir} onSort={requestSort} /> label="Reported" sortKeyName="reportedAt" activeKey={sortKey} direction={sortDir} onSort={requestSort} /> + label="Agent" sortKeyName="agentVersion" activeKey={sortKey} direction={sortDir} onSort={requestSort} /> {servers === null && !error && ( - + @@ -178,11 +189,19 @@ export default function Updates() { {r.security === null ? — : r.security} {r.since ? : "—"} {r.reportedAt ? : "—"} + + {agentVersionWords(r.agentVersion)}{" "} + {agentState(r.server, agentLatest) === "behind" && ( + + behind + + )} + ))} {servers !== null && sorted?.length === 0 && ( - + {tableSearch.active || onlyWaiting ? "Nothing matches that." : "No server has reported yet."} diff --git a/web/src/pages/settings/NotificationSettings.tsx b/web/src/pages/settings/NotificationSettings.tsx index 200bc76..fc186a3 100644 --- a/web/src/pages/settings/NotificationSettings.tsx +++ b/web/src/pages/settings/NotificationSettings.tsx @@ -7,9 +7,11 @@ import { type SmtpSettings, type WebhookSettings, type NotificationEvents, + type DigestSettings, type QuietHoursSettings, type HealthCheckSettings, } from "../../api/client"; +import { readableError } from "../../utils/errors"; import { ErrorNotice } from "../../components/Notice"; import { PageSkeleton } from "../../components/Skeleton"; @@ -58,6 +60,17 @@ const DEFAULT_NOTIFICATIONS: NotificationEvents = { integrationFailureThreshold: 3, }; const DEFAULT_QUIET_HOURS: QuietHoursSettings = { enabled: false, start: "22:00", end: "07:00" }; +const DEFAULT_DIGEST: DigestSettings = { enabled: false, day: 1, time: "08:00" }; +// Monday first, as a week reads; the numbers are what the server keeps (0 = Sunday). +const DIGEST_DAYS: { value: number; label: string }[] = [ + { value: 1, label: "Monday" }, + { value: 2, label: "Tuesday" }, + { value: 3, label: "Wednesday" }, + { value: 4, label: "Thursday" }, + { value: 5, label: "Friday" }, + { value: 6, label: "Saturday" }, + { value: 0, label: "Sunday" }, +]; const DEFAULT_HEALTH_CHECKS: HealthCheckSettings = { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14 }; type TestResult = { ok: boolean; message: string } | null; @@ -85,6 +98,10 @@ export default function NotificationSettings() { const [notifications, setNotifications] = useState(DEFAULT_NOTIFICATIONS); const [quietHours, setQuietHours] = useState(DEFAULT_QUIET_HOURS); const [healthChecks, setHealthChecks] = useState(DEFAULT_HEALTH_CHECKS); + const [digest, setDigest] = useState(DEFAULT_DIGEST); + const [digestPreview, setDigestPreview] = useState<{ title: string; message: string } | null>(null); + const [digestBusy, setDigestBusy] = useState<"preview" | "send" | null>(null); + const [digestResult, setDigestResult] = useState(null); const [queuedCount, setQueuedCount] = useState(null); const [flushing, setFlushing] = useState(false); const [flushResult, setFlushResult] = useState(null); @@ -109,6 +126,7 @@ export default function NotificationSettings() { setNotifications(res.settings.notifications); setQuietHours(res.settings.quietHours); setHealthChecks(res.settings.healthChecks); + setDigest(res.settings.digest ?? DEFAULT_DIGEST); }) .catch((err) => setLoadError(err instanceof Error ? err.message : String(err))) .finally(() => setLoading(false)); @@ -123,7 +141,7 @@ export default function NotificationSettings() { setSaveError(null); setSaved(false); try { - await api.settings.update({ gotify, ntfy, smtp, webhook, notifications, quietHours, healthChecks }); + await api.settings.update({ gotify, ntfy, smtp, webhook, notifications, quietHours, healthChecks, digest }); setSaved(true); setTimeout(() => setSaved(false), 3000); } catch (err) { @@ -133,6 +151,32 @@ export default function NotificationSettings() { } } + async function handleDigestPreview() { + setDigestBusy("preview"); + setDigestResult(null); + try { + const res = await api.settings.digestPreview(); + setDigestPreview({ title: res.title, message: res.message }); + } catch (err) { + setDigestResult({ ok: false, message: err instanceof Error ? err.message : String(err) }); + } finally { + setDigestBusy(null); + } + } + + async function handleDigestSend() { + setDigestBusy("send"); + setDigestResult(null); + try { + const res = await api.settings.digestSend(); + setDigestResult({ ok: true, message: `Sent through ${res.channels} channel${res.channels === 1 ? "" : "s"}.` }); + } catch (err) { + setDigestResult({ ok: false, message: readableError(err) }); + } finally { + setDigestBusy(null); + } + } + async function handleFlushNow() { setFlushing(true); setFlushResult(null); @@ -740,6 +784,63 @@ export default function NotificationSettings() {
+ +
+
+
+

Weekly digest

+
+
+

+ One message a week through the channels above, in the time zone set above: what is wrong now, what was down and for how long, what happened, + disks heading for full, things about to expire, updates waiting and maintenance coming up. A topic with nothing to say is left out, and a + quiet week says so in one line. Quiet hours hold it like any other notification. +

+ +
+
+ + +
+
+ + setDigest((d) => ({ ...d, time: e.target.value }))} /> +
+
+
+ + +
+
Settings above are saved with the Save button; the preview and "send it now" don't depend on them.
+ + {digestPreview && ( +
+
{digestPreview.title}
+
+                    {digestPreview.message}
+                  
+
+ )} +
+
+
); diff --git a/web/src/utils/agent.ts b/web/src/utils/agent.ts new file mode 100644 index 0000000..3f8c8c1 --- /dev/null +++ b/web/src/utils/agent.ts @@ -0,0 +1,20 @@ +import type { AgentLatest } from "../api/client"; + +/** What an agent's version means for a server, for the badges: not on an agent at all, current, or behind. */ +export type AgentState = "none" | "current" | "behind"; + +/** + * Whether a server's agent is behind what this Homelab Manager would install today. The same rule the server uses: only a server that + * reports through an agent has one, and an agent that says no version is version 1. + */ +export function agentState(server: { osType: string; lastSeenAt: string | null; agentVersion: number | null }, latest: AgentLatest | null): AgentState { + if (!server.lastSeenAt) return "none"; + const newest = latest ? (server.osType === "windows" ? latest.windows : server.osType === "linux" ? latest.linux : null) : null; + if (newest === null || newest === undefined) return "current"; // nothing to compare with: don't cry wolf + return (server.agentVersion ?? 1) < newest ? "behind" : "current"; +} + +/** "v2", or "v1 or older" for one that has never said. */ +export function agentVersionWords(version: number | null): string { + return version === null ? "v1 or older" : `v${version}`; +}