From aae4f0d74f505940ce8071200f93afb39403c10e Mon Sep 17 00:00:00 2001 From: Bobban Rydh Date: Sat, 26 Sep 2026 02:23:06 +0200 Subject: [PATCH] Add maintenance mode to silence alerts while working on a server or integration Rebooting Proxmox or patching a server triggered failure/offline alerts you then had to dismiss. A maintenance window silences alerts about one server, integration, or DNS provider for a chosen time. New Maintenance page (start with a duration and optional reason, end early, see what's silenced and what isn't) and a banner in the app shell so every signed-in user can see what is currently silenced. Starting/ending is operator-only and audit-logged; starting one on a target that already has a window restarts its clock instead of stacking. Silenced for the target: server offline/disk alerts, Proxmox/Synology storage and health alerts, Proxmox backup alerts, and "integration down" alerts. Not silenced: expiry and update reminders, DNS change notices. The design goal is that this cannot hide a real outage: - Every window has a required end (5 min to 7 days); there is no open-ended option, so a forgotten window expires by itself. - A silenced problem is deliberately NOT recorded as "known". If it is still present when the window ends it alerts then, as new. A problem that was already alerted before the window stays known, so it isn't repeated, and is reported cleared only after the window ends. - Failure alerts keep counting failures during a window without marking themselves alerted, so an outage that outlasts the window alerts on the very next failed call. Known limitation, stated on the page: integration-failure alerts are tracked per service TYPE (all "proxmox"), not per configured instance, so a window on one Proxmox integration also silences a failure on a second Proxmox integration while it's open. Fixing that means threading the integration id through every adapter and the diagnostic log, which is a much larger change than this feature. Also moved the API-error-message helper out of Secrets.tsx into a shared util now that two pages use it. New table maintenance_windows (migration 0008). Verified with 44 checks: the condition-key-to-subject mapping (including server:3 vs server:33), the diff rules with silenced subjects (new problem not recorded, alerts when the window ends; already-known one carried and not repeated; clears only after the window), window expiry and integration/DNS-provider source matching, the failure tracker end to end against a webhook (silent during a window while an unrelated service still alerts; outage that outlasts the window alerts on the next failure and only once; fail-and-recover fully inside a window sends nothing), a full health pass against a real window, and the real router with a stubbed session (role rules, duration bounds including the missing-duration case, extend-not-stack, 404s, deleted targets hidden, audit entries). Real dev database mtime untouched. Not done: I haven't clicked through the new page or banner in a browser (they sit behind the Authentik login); it builds and the API behind it is tested. Co-Authored-By: Claude Sonnet 5 --- README.md | 8 + server/drizzle/0008_thin_boom_boom.sql | 9 + server/drizzle/meta/0008_snapshot.json | 1272 +++++++++++++++++ server/drizzle/meta/_journal.json | 7 + server/src/db/schema.ts | 15 + server/src/index.ts | 2 + server/src/routes/maintenance.ts | 101 ++ server/src/services/healthMonitor.ts | 22 +- .../src/services/integrationHealthMonitor.ts | 9 +- server/src/services/maintenance.ts | 71 + server/src/services/proxmoxBackupScheduler.ts | 3 + web/src/App.tsx | 2 + web/src/api/client.ts | 26 + web/src/layout/AppShell.tsx | 45 +- web/src/pages/Maintenance.tsx | 249 ++++ web/src/pages/Secrets.tsx | 16 +- web/src/utils/duration.ts | 9 + web/src/utils/errors.ts | 14 + 18 files changed, 1854 insertions(+), 26 deletions(-) create mode 100644 server/drizzle/0008_thin_boom_boom.sql create mode 100644 server/drizzle/meta/0008_snapshot.json create mode 100644 server/src/routes/maintenance.ts create mode 100644 server/src/services/maintenance.ts create mode 100644 web/src/pages/Maintenance.tsx create mode 100644 web/src/utils/duration.ts create mode 100644 web/src/utils/errors.ts diff --git a/README.md b/README.md index 4aaba9a..15a2c8c 100644 --- a/README.md +++ b/README.md @@ -142,6 +142,14 @@ raise one notification when the problem starts and one when it clears (both thresholds are set under Settings → Notifications). Active problems are remembered across restarts, so a rebuild doesn't re-alert them. +**Maintenance mode** silences alerts about one server, integration, or DNS +provider while you work on it (server offline / disk, storage and Synology +health, Proxmox backup alerts, and "integration down" for that service type). +Every window has a fixed end (5 minutes to 7 days) and expires on its own, and +a problem that began during a window and is still present when it ends alerts +then — a forgotten window can't hide an outage. A banner shows what's currently +silenced to every signed-in user. + The app is installable as a PWA — "Install app" / "Add to Home Screen" from the browser gives it its own icon and a standalone window on phone or desktop. This needs the site to be served over HTTPS (browsers only offer install on secure diff --git a/server/drizzle/0008_thin_boom_boom.sql b/server/drizzle/0008_thin_boom_boom.sql new file mode 100644 index 0000000..358963f --- /dev/null +++ b/server/drizzle/0008_thin_boom_boom.sql @@ -0,0 +1,9 @@ +CREATE TABLE `maintenance_windows` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `target_type` text NOT NULL, + `target_id` integer NOT NULL, + `reason` text, + `started_at` text NOT NULL, + `ends_at` text NOT NULL, + `created_by` text +); diff --git a/server/drizzle/meta/0008_snapshot.json b/server/drizzle/meta/0008_snapshot.json new file mode 100644 index 0000000..18887d4 --- /dev/null +++ b/server/drizzle/meta/0008_snapshot.json @@ -0,0 +1,1272 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "47ef2c5a-3706-45ff-93ec-6b8a25eed11e", + "prevId": "b8c719f5-3dfc-43a1-892e-3eed11fa47c0", + "tables": { + "audit_log": { + "name": "audit_log", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "actor_label": { + "name": "actor_label", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "category": { + "name": "category", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "detail": { + "name": "detail", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": { + "audit_log_actor_user_id_users_id_fk": { + "name": "audit_log_actor_user_id_users_id_fk", + "tableFrom": "audit_log", + "tableTo": "users", + "columnsFrom": [ + "actor_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "diag_log": { + "name": "diag_log", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "operation": { + "name": "operation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ok": { + "name": "ok", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "latency_ms": { + "name": "latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "dns_providers": { + "name": "dns_providers", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "provider_type": { + "name": "provider_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "credential_id": { + "name": "credential_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "config": { + "name": "config", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": { + "dns_providers_credential_id_integration_credentials_id_fk": { + "name": "dns_providers_credential_id_integration_credentials_id_fk", + "tableFrom": "dns_providers", + "tableTo": "integration_credentials", + "columnsFrom": [ + "credential_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "dns_records_cache": { + "name": "dns_records_cache", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "provider_id": { + "name": "provider_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "zone_id": { + "name": "zone_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_id": { + "name": "record_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ttl": { + "name": "ttl", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "proxied": { + "name": "proxied", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "dns_records_cache_provider_id_dns_providers_id_fk": { + "name": "dns_records_cache_provider_id_dns_providers_id_fk", + "tableFrom": "dns_records_cache", + "tableTo": "dns_providers", + "columnsFrom": [ + "provider_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "dns_zones_cache": { + "name": "dns_zones_cache", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "provider_id": { + "name": "provider_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "zone_id": { + "name": "zone_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "zone_name": { + "name": "zone_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "synced_at": { + "name": "synced_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "dns_zones_cache_provider_zone_idx": { + "name": "dns_zones_cache_provider_zone_idx", + "columns": [ + "provider_id", + "zone_id" + ], + "isUnique": true + } + }, + "foreignKeys": { + "dns_zones_cache_provider_id_dns_providers_id_fk": { + "name": "dns_zones_cache_provider_id_dns_providers_id_fk", + "tableFrom": "dns_zones_cache", + "tableTo": "dns_providers", + "columnsFrom": [ + "provider_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "integration_credentials": { + "name": "integration_credentials", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "encrypted_secret": { + "name": "encrypted_secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "integrations": { + "name": "integrations", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_url": { + "name": "base_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "credential_id": { + "name": "credential_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "config": { + "name": "config", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": { + "integrations_credential_id_integration_credentials_id_fk": { + "name": "integrations_credential_id_integration_credentials_id_fk", + "tableFrom": "integrations", + "tableTo": "integration_credentials", + "columnsFrom": [ + "credential_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ipam_entries": { + "name": "ipam_entries", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "vendor": { + "name": "vendor", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "location": { + "name": "location", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": { + "ipam_entries_ip_address_unique": { + "name": "ipam_entries_ip_address_unique", + "columns": [ + "ip_address" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "maintenance_windows": { + "name": "maintenance_windows", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_id": { + "name": "target_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ends_at": { + "name": "ends_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "notification_queue": { + "name": "notification_queue", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "scheduled_tasks": { + "name": "scheduled_tasks", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "server_id": { + "name": "server_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "schedule_type": { + "name": "schedule_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'agent'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "command": { + "name": "command", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "schedule_expression": { + "name": "schedule_expression", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "next_run_at": { + "name": "next_run_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "raw_metadata": { + "name": "raw_metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_stale": { + "name": "is_stale", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "first_seen_at": { + "name": "first_seen_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": { + "scheduled_tasks_server_id_servers_id_fk": { + "name": "scheduled_tasks_server_id_servers_id_fk", + "tableFrom": "scheduled_tasks", + "tableTo": "servers", + "columnsFrom": [ + "server_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "secrets": { + "name": "secrets", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'generic'" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expiry_date": { + "name": "expiry_date", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "warn_days": { + "name": "warn_days", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 30 + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "check_host": { + "name": "check_host", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "check_port": { + "name": "check_port", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_checked_at": { + "name": "last_checked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_check_error": { + "name": "last_check_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "server_links": { + "name": "server_links", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "server_id": { + "name": "server_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": { + "server_links_server_id_servers_id_fk": { + "name": "server_links_server_id_servers_id_fk", + "tableFrom": "server_links", + "tableTo": "servers", + "columnsFrom": [ + "server_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "servers": { + "name": "servers", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hostname": { + "name": "hostname", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "os_type": { + "name": "os_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'linux'" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "api_token_hash": { + "name": "api_token_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "api_token_prefix": { + "name": "api_token_prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ip_addresses": { + "name": "ip_addresses", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cpu_model": { + "name": "cpu_model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cpu_cores": { + "name": "cpu_cores", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cpu_load_percent": { + "name": "cpu_load_percent", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mem_total_bytes": { + "name": "mem_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mem_used_bytes": { + "name": "mem_used_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disks": { + "name": "disks", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "proxmox_integration_id": { + "name": "proxmox_integration_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "proxmox_node": { + "name": "proxmox_node", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "proxmox_guest_type": { + "name": "proxmox_guest_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "proxmox_vmid": { + "name": "proxmox_vmid", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "hide_proxmox_link": { + "name": "hide_proxmox_link", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": {}, + "foreignKeys": { + "servers_proxmox_integration_id_integrations_id_fk": { + "name": "servers_proxmox_integration_id_integrations_id_fk", + "tableFrom": "servers", + "tableTo": "integrations", + "columnsFrom": [ + "proxmox_integration_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "settings": { + "name": "settings", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "oidc_sub": { + "name": "oidc_sub", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'viewer'" + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "last_login_at": { + "name": "last_login_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "users_oidc_sub_unique": { + "name": "users_oidc_sub_unique", + "columns": [ + "oidc_sub" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/server/drizzle/meta/_journal.json b/server/drizzle/meta/_journal.json index fc6263f..cfd797a 100644 --- a/server/drizzle/meta/_journal.json +++ b/server/drizzle/meta/_journal.json @@ -57,6 +57,13 @@ "when": 1790372043652, "tag": "0007_secret_madripoor", "breakpoints": true + }, + { + "idx": 8, + "version": "6", + "when": 1790381917814, + "tag": "0008_thin_boom_boom", + "breakpoints": true } ] } \ No newline at end of file diff --git a/server/src/db/schema.ts b/server/src/db/schema.ts index 758fbf6..79afc99 100644 --- a/server/src/db/schema.ts +++ b/server/src/db/schema.ts @@ -59,6 +59,21 @@ export const notificationQueue = sqliteTable("notification_queue", { .default(sql`(current_timestamp)`), }); +// ─── Maintenance windows — alerts about a target are silenced until endsAt ── + +export const maintenanceTargetTypes = ["server", "integration", "dns_provider"] as const; +export type MaintenanceTargetType = (typeof maintenanceTargetTypes)[number]; + +export const maintenanceWindows = sqliteTable("maintenance_windows", { + id: integer("id").primaryKey({ autoIncrement: true }), + targetType: text("target_type").$type().notNull(), + targetId: integer("target_id").notNull(), // polymorphic — no FK; a deleted target's window is simply ignored + reason: text("reason"), + startedAt: text("started_at").notNull(), // ISO + endsAt: text("ends_at").notNull(), // ISO — required: a forgotten open-ended window would silence real problems forever + createdBy: text("created_by"), +}); + // ─── Settings (key/value) ─────────────────────────────────────────────────── export const settings = sqliteTable("settings", { diff --git a/server/src/index.ts b/server/src/index.ts index ea033a0..7e95b12 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -23,6 +23,7 @@ import { integrationsRouter } from "./routes/integrations.js"; import { settingsRouter } from "./routes/settings.js"; import { searchRouter } from "./routes/search.js"; import { sessionsRouter } from "./routes/sessions.js"; +import { maintenanceRouter } from "./routes/maintenance.js"; import { initSecretExpiryScheduler } from "./services/secretExpiryScheduler.js"; import { initTailscaleKeyExpiryScheduler } from "./services/tailscaleKeyExpiryScheduler.js"; import { initLogRetentionScheduler } from "./services/logRetentionScheduler.js"; @@ -90,6 +91,7 @@ app.use("/api/integrations", integrationsRouter); app.use("/api/settings", settingsRouter); app.use("/api/search", searchRouter); app.use("/api/sessions", sessionsRouter); +app.use("/api/maintenance", maintenanceRouter); if (existsSync(webDist)) { app.use(express.static(webDist)); diff --git a/server/src/routes/maintenance.ts b/server/src/routes/maintenance.ts new file mode 100644 index 0000000..63bfbcb --- /dev/null +++ b/server/src/routes/maintenance.ts @@ -0,0 +1,101 @@ +import { Router } from "express"; +import { eq, lt } from "drizzle-orm"; +import { z } from "zod"; +import { db } from "../db/client.js"; +import { dnsProviders, integrations, maintenanceTargetTypes, maintenanceWindows, servers } from "../db/schema.js"; +import { requireAuth, requireRole } from "../auth/middleware.js"; +import { recordAudit } from "../services/audit.js"; +import { describeTarget, listActiveWindows } from "../services/maintenance.js"; +import { asyncHandler } from "../utils/asyncHandler.js"; + +export const maintenanceRouter = Router(); + +// Anyone signed in can see what's currently silenced (so nobody is surprised by missing alerts); +// starting and ending a window is an operator action, like the other things that change how the homelab behaves. +maintenanceRouter.use(requireAuth); + +maintenanceRouter.get("/", asyncHandler(async (_req, res) => { + const windows = []; + for (const w of await listActiveWindows()) { + const target = await describeTarget(w.targetType, w.targetId); + if (!target) continue; // target was deleted — nothing left to silence + windows.push({ ...w, targetName: target.name, targetKind: target.kind }); + } + windows.sort((a, b) => a.endsAt.localeCompare(b.endsAt)); + + const [serverRows, integrationRows, providerRows] = await Promise.all([ + db.select({ id: servers.id, name: servers.name }).from(servers).orderBy(servers.name), + db.select({ id: integrations.id, name: integrations.name, type: integrations.type }).from(integrations).orderBy(integrations.name), + db.select({ id: dnsProviders.id, name: dnsProviders.name, providerType: dnsProviders.providerType }).from(dnsProviders).orderBy(dnsProviders.name), + ]); + res.json({ windows, targets: { servers: serverRows, integrations: integrationRows, dnsProviders: providerRows } }); +})); + +const startSchema = z.object({ + targetType: z.enum(maintenanceTargetTypes), + targetId: z.number().int().positive(), + // Required and capped: an open-ended window is the way this feature could quietly hide a real outage. + minutes: z.number().int().min(5).max(7 * 24 * 60), + reason: z.string().max(200).optional(), +}); + +maintenanceRouter.post("/", requireRole("operator"), asyncHandler(async (req, res) => { + const parsed = startSchema.safeParse(req.body); + if (!parsed.success) { + return res.status(400).json({ error: "invalid_body", details: parsed.error.flatten() }); + } + const { targetType, targetId, minutes, reason } = parsed.data; + + const target = await describeTarget(targetType, targetId); + if (!target) { + return res.status(404).json({ error: "not_found", message: "That target doesn't exist." }); + } + + const now = new Date(); + const endsAt = new Date(now.getTime() + minutes * 60_000).toISOString(); + const actor = req.currentUser!; + const createdBy = actor.name ?? actor.email ?? actor.oidcSub; + + // Starting maintenance on something already in maintenance restarts its clock rather than stacking windows. + const existing = (await listActiveWindows(now)).find((w) => w.targetType === targetType && w.targetId === targetId); + let window; + if (existing) { + [window] = await db.update(maintenanceWindows).set({ endsAt, reason: reason ?? null, createdBy }).where(eq(maintenanceWindows.id, existing.id)).returning(); + } else { + // Long-expired rows are just clutter; clear them out whenever a new one is added. + await db.delete(maintenanceWindows).where(lt(maintenanceWindows.endsAt, new Date(now.getTime() - 24 * 3600_000).toISOString())); + [window] = await db.insert(maintenanceWindows).values({ targetType, targetId, reason: reason ?? null, startedAt: now.toISOString(), endsAt, createdBy }).returning(); + } + + await recordAudit({ + actor, + category: "maintenance", + action: existing ? "extend" : "start", + targetType, + targetId, + detail: { name: target.name, minutes, reason: reason ?? null }, + }); + + res.status(201).json({ window: { ...window, targetName: target.name, targetKind: target.kind } }); +})); + +maintenanceRouter.delete("/:id", requireRole("operator"), asyncHandler(async (req, res) => { + const id = Number(req.params.id); + const [existing] = await db.select().from(maintenanceWindows).where(eq(maintenanceWindows.id, id)).limit(1); + if (!existing) { + return res.status(404).json({ error: "not_found" }); + } + // Ending moves the end to now, so it stops applying immediately; the row is pruned later like any expired one. + await db.update(maintenanceWindows).set({ endsAt: new Date().toISOString() }).where(eq(maintenanceWindows.id, id)); + + const target = await describeTarget(existing.targetType, existing.targetId); + await recordAudit({ + actor: req.currentUser!, + category: "maintenance", + action: "end", + targetType: existing.targetType, + targetId: existing.targetId, + detail: { name: target?.name ?? null }, + }); + res.status(204).end(); +})); diff --git a/server/src/services/healthMonitor.ts b/server/src/services/healthMonitor.ts index 9cf968a..a1294a0 100644 --- a/server/src/services/healthMonitor.ts +++ b/server/src/services/healthMonitor.ts @@ -6,6 +6,7 @@ import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/pro import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js"; import { notifyHealthIssues, notifyHealthRecovered } from "./notify.js"; import { getInternalFlag, getSettings, setInternalFlag } from "./settingsStore.js"; +import { activeSubjects, subjectOfConditionKey } from "./maintenance.js"; const STATE_FLAG = "healthActiveConditions"; /** After a restart the agents haven't had a chance to report yet (they run every 15 min), so server-derived conditions are held rather than judged. */ @@ -216,15 +217,26 @@ export function diffConditions( previous: ActiveState, current: HealthCondition[], held: Set = new Set(), + /** Subjects ("server:3", "integration:1") under an active maintenance window. */ + silenced: Set = new Set(), ): { added: HealthCondition[]; resolved: { key: string; message: string }[]; next: ActiveState } { - const next: ActiveState = {}; - for (const c of current) next[c.key] = { message: c.message, source: c.source }; + const isSilenced = (key: string) => { + const subject = subjectOfConditionKey(key); + return subject !== null && silenced.has(subject); + }; + // A silenced problem is not recorded as "known": if it's still there when the window ends it must + // alert then, as new. (One that was already alerted before the window stays known, so it isn't repeated.) + const visible = current.filter((c) => !isSilenced(c.key)); - const added = current.filter((c) => !(c.key in previous)); + const next: ActiveState = {}; + for (const c of visible) next[c.key] = { message: c.message, source: c.source }; + + const added = visible.filter((c) => !(c.key in previous)); const resolved: { key: string; message: string }[] = []; for (const [key, entry] of Object.entries(previous)) { if (key in next) continue; - if (isHeld(entry.source, held)) { + // Held (source unreadable) or silenced: leave it exactly as it was — neither cleared nor re-alerted. + if (isHeld(entry.source, held) || isSilenced(key)) { next[key] = entry; } else { resolved.push({ key, message: entry.message }); @@ -296,7 +308,7 @@ export async function runHealthCheck(now: number = Date.now()): Promise<{ added: if (inGrace) held.add("server"); const current = evaluateHealth(snapshot, healthChecks, now, { skipServers: inGrace }); - const { added, resolved, next } = diffConditions(await loadState(), current, held); + const { added, resolved, next } = diffConditions(await loadState(), current, held, await activeSubjects(new Date(now))); await setInternalFlag(STATE_FLAG, JSON.stringify(next)); // State is tracked even when the alert toggle is off (the notify functions check it), diff --git a/server/src/services/integrationHealthMonitor.ts b/server/src/services/integrationHealthMonitor.ts index 819c9e2..1faf851 100644 --- a/server/src/services/integrationHealthMonitor.ts +++ b/server/src/services/integrationHealthMonitor.ts @@ -1,4 +1,5 @@ import { getSettings } from "./settingsStore.js"; +import { isSourceInMaintenance } from "./maintenance.js"; import { notifyIntegrationDown, notifyIntegrationRecovered } from "./notify.js"; interface SourceHealth { @@ -31,8 +32,12 @@ export async function trackIntegrationHealth(source: string, ok: boolean): Promi state.consecutiveFailures += 1; const { notifications } = await getSettings(); if (notifications.integrationFailureAlerts && !state.alerted && state.consecutiveFailures >= notifications.integrationFailureThreshold) { - state.alerted = true; - await notifyIntegrationDown(source, state.consecutiveFailures); + // During maintenance the failures keep being counted but `alerted` stays false, so if the service is + // still failing once the window ends, the very next failed call alerts — a real outage isn't swallowed. + if (!(await isSourceInMaintenance(source))) { + state.alerted = true; + await notifyIntegrationDown(source, state.consecutiveFailures); + } } health.set(source, state); } diff --git a/server/src/services/maintenance.ts b/server/src/services/maintenance.ts new file mode 100644 index 0000000..b6436ce --- /dev/null +++ b/server/src/services/maintenance.ts @@ -0,0 +1,71 @@ +import { eq, gt } from "drizzle-orm"; +import { db } from "../db/client.js"; +import { dnsProviders, integrations, maintenanceWindows, servers } from "../db/schema.js"; + +export type ActiveWindow = typeof maintenanceWindows.$inferSelect; + +/** Windows whose end is still in the future. ISO strings compare correctly as text, so this is a plain string comparison. */ +export async function listActiveWindows(now: Date = new Date()): Promise { + return db.select().from(maintenanceWindows).where(gt(maintenanceWindows.endsAt, now.toISOString())); +} + +/** + * The identity a health condition or alert belongs to, as "server:3" / + * "integration:1" / "dns_provider:2", so it can be matched against active windows. + * Derived from the condition key (which already encodes it) rather than stored, + * so conditions persisted by an earlier version still map correctly. + */ +export function subjectOfConditionKey(key: string): string | null { + let m = /^(?:offline|disk):server:(\d+)(?::|$)/.exec(key); + if (m) return `server:${m[1]}`; + m = /^disk:proxmox:(\d+):/.exec(key); + if (m) return `integration:${m[1]}`; + m = /^(?:synology-volume|synology-disk|disk:synology):(\d+):/.exec(key); + if (m) return `integration:${m[1]}`; + return null; +} + +export async function activeSubjects(now: Date = new Date()): Promise> { + return new Set((await listActiveWindows(now)).map((w) => `${w.targetType}:${w.targetId}`)); +} + +export async function isInMaintenance(targetType: ActiveWindow["targetType"], targetId: number, now: Date = new Date()): Promise { + return (await activeSubjects(now)).has(`${targetType}:${targetId}`); +} + +/** + * Integration-failure alerts are tracked per service TYPE ("proxmox", + * "cloudflare", ...), not per configured instance, so a window on any + * integration or DNS provider of that type silences that type's failure alerts. + * (With two integrations of one type, a real failure on the un-windowed one is + * silenced too while the window is open — a known consequence of that + * granularity, stated on the Maintenance page.) + */ +export async function isSourceInMaintenance(source: string, now: Date = new Date()): Promise { + const windows = await listActiveWindows(now); + if (windows.length === 0) return false; + for (const w of windows) { + if (w.targetType === "integration") { + const [row] = await db.select({ type: integrations.type }).from(integrations).where(eq(integrations.id, w.targetId)).limit(1); + if (row?.type === source) return true; + } else if (w.targetType === "dns_provider") { + const [row] = await db.select({ type: dnsProviders.providerType }).from(dnsProviders).where(eq(dnsProviders.id, w.targetId)).limit(1); + if (row?.type === source) return true; + } + } + return false; +} + +/** Display name for a window's target, or null if the target no longer exists. */ +export async function describeTarget(targetType: ActiveWindow["targetType"], targetId: number): Promise<{ name: string; kind: string } | null> { + if (targetType === "server") { + const [r] = await db.select({ name: servers.name }).from(servers).where(eq(servers.id, targetId)).limit(1); + return r ? { name: r.name, kind: "Server" } : null; + } + if (targetType === "integration") { + const [r] = await db.select({ name: integrations.name, type: integrations.type }).from(integrations).where(eq(integrations.id, targetId)).limit(1); + return r ? { name: r.name, kind: `Integration (${r.type})` } : null; + } + const [r] = await db.select({ name: dnsProviders.name, type: dnsProviders.providerType }).from(dnsProviders).where(eq(dnsProviders.id, targetId)).limit(1); + return r ? { name: r.name, kind: `DNS provider (${r.type})` } : null; +} diff --git a/server/src/services/proxmoxBackupScheduler.ts b/server/src/services/proxmoxBackupScheduler.ts index eab65c7..982f323 100644 --- a/server/src/services/proxmoxBackupScheduler.ts +++ b/server/src/services/proxmoxBackupScheduler.ts @@ -6,6 +6,7 @@ import { loadIntegrationConfig } from "../integrations/loadIntegration.js"; import { createProxmoxAdapter, guestsWithoutBackupCoverage, type ProxmoxBackupTask } from "../integrations/proxmox/adapter.js"; import { notifyProxmoxBackupFailure, notifyProxmoxUncoveredGuests } from "./notify.js"; import { getSettings, getInternalFlag, setInternalFlag } from "./settingsStore.js"; +import { isInMaintenance } from "./maintenance.js"; const LAST_RUN_FLAG = "proxmoxBackupCheckLastRunDate"; @@ -19,6 +20,8 @@ async function checkProxmoxBackups(): Promise { const uncovered: { integrationName: string; guestName: string; vmid: number; node: string }[] = []; for (const row of rows) { + // A host being worked on can't run or report backups; this check is daily, so tomorrow's pass covers it. + if (await isInMaintenance("integration", row.id)) continue; try { const loaded = await loadIntegrationConfig(row.id); if (!loaded) continue; diff --git a/web/src/App.tsx b/web/src/App.tsx index c5ee0a6..5bdaf0b 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -20,6 +20,7 @@ import Gitea from "./pages/Gitea"; import Proxmox from "./pages/Proxmox"; import Synology from "./pages/Synology"; import Generator from "./pages/Generator"; +import Maintenance from "./pages/Maintenance"; import Settings from "./pages/Settings"; import NotificationSettings from "./pages/settings/NotificationSettings"; import BadgeSettings from "./pages/settings/BadgeSettings"; @@ -88,6 +89,7 @@ export default function App() { } /> } /> } /> + } /> } /> } /> } /> diff --git a/web/src/api/client.ts b/web/src/api/client.ts index 7cc8ab4..75d238a 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -30,6 +30,26 @@ export interface SessionSummary { expiresAt: string | null; } +export type MaintenanceTargetType = "server" | "integration" | "dns_provider"; + +export interface MaintenanceWindow { + id: number; + targetType: MaintenanceTargetType; + targetId: number; + targetName: string; + targetKind: string; + reason: string | null; + startedAt: string; + endsAt: string; + createdBy: string | null; +} + +export interface MaintenanceTargets { + servers: { id: number; name: string }[]; + integrations: { id: number; name: string; type: string }[]; + dnsProviders: { id: number; name: string; providerType: string }[]; +} + export interface AuditLogEntry { id: number; actorUserId: number | null; @@ -649,6 +669,12 @@ export const api = { body: JSON.stringify({ role }), }), }, + maintenance: { + list: () => request<{ windows: MaintenanceWindow[]; targets: MaintenanceTargets }>("/api/maintenance"), + start: (data: { targetType: MaintenanceTargetType; targetId: number; minutes: number; reason?: string }) => + request<{ window: MaintenanceWindow }>("/api/maintenance", { method: "POST", body: JSON.stringify(data) }), + end: (id: number) => request(`/api/maintenance/${id}`, { method: "DELETE" }), + }, sessions: { list: () => request<{ sessions: SessionSummary[]; currentSessionId: string }>("/api/sessions"), revoke: (id: string) => request(`/api/sessions/${encodeURIComponent(id)}`, { method: "DELETE" }), diff --git a/web/src/layout/AppShell.tsx b/web/src/layout/AppShell.tsx index e3698a3..afbf1d7 100644 --- a/web/src/layout/AppShell.tsx +++ b/web/src/layout/AppShell.tsx @@ -1,5 +1,5 @@ -import { useState, type ReactNode } from "react"; -import { NavLink } from "react-router-dom"; +import { useEffect, useState, type ReactNode } from "react"; +import { Link, NavLink } from "react-router-dom"; import { IconLayoutDashboard, IconServer2, @@ -22,8 +22,10 @@ import { IconSun, IconMoon, IconWand, + IconTool, } from "@tabler/icons-react"; -import type { CurrentUser } from "../api/client"; +import { api, type CurrentUser, type MaintenanceWindow } from "../api/client"; +import { formatRemaining } from "../utils/duration"; import { getTheme, setTheme, type Theme } from "../utils/theme"; import CommandPalette from "../components/CommandPalette"; @@ -48,6 +50,7 @@ const NAV_ITEMS: NavItem[] = [ { to: "/gitea", label: "Gitea", icon: }, { to: "/integrations", label: "Integrations", icon: }, { to: "/generator", label: "Generator", icon: }, + { to: "/maintenance", label: "Maintenance", icon: }, { to: "/users", label: "Users", icon: , minRole: "admin" }, { to: "/sessions", label: "Sessions", icon: , minRole: "admin" }, { to: "/audit-log", label: "Audit Log", icon: , minRole: "operator" }, @@ -60,6 +63,28 @@ const roleRank: Record = { viewer: 0, operator: 1, export default function AppShell({ user, children }: { user: CurrentUser; children: ReactNode }) { const [sidebarOpen, setSidebarOpen] = useState(false); const [theme, setThemeState] = useState(() => getTheme()); + const [maintenance, setMaintenance] = useState([]); + + // Everyone sees what's currently silenced, so nobody is left wondering why an alert never came. + useEffect(() => { + let cancelled = false; + const load = () => + api.maintenance + .list() + .then((res) => { + if (!cancelled) setMaintenance(res.windows); + }) + .catch(() => {}); + load(); + const poll = setInterval(load, 60_000); + window.addEventListener("maintenance-changed", load); + return () => { + cancelled = true; + clearInterval(poll); + window.removeEventListener("maintenance-changed", load); + }; + }, []); + const visibleItems = NAV_ITEMS.filter( (item) => !item.minRole || roleRank[user.role] >= roleRank[item.minRole], ); @@ -126,7 +151,19 @@ export default function AppShell({ user, children }: { user: CurrentUser; childr
-
{children}
+
+ {maintenance.length > 0 && ( +
+ +
+ Maintenance — alerts silenced for{" "} + {maintenance.map((w) => `${w.targetName} (${formatRemaining(w.endsAt)} left)`).join(", ")}.{" "} + Manage +
+
+ )} + {children} +
diff --git a/web/src/pages/Maintenance.tsx b/web/src/pages/Maintenance.tsx new file mode 100644 index 0000000..fd08279 --- /dev/null +++ b/web/src/pages/Maintenance.tsx @@ -0,0 +1,249 @@ +import { useEffect, useState } from "react"; +import { api, type CurrentUser, type MaintenanceTargets, type MaintenanceTargetType, type MaintenanceWindow } from "../api/client"; +import { formatDateTime } from "../utils/date"; +import { formatRemaining } from "../utils/duration"; +import { readableError } from "../utils/errors"; + +const DURATIONS: { minutes: number; label: string }[] = [ + { minutes: 30, label: "30 minutes" }, + { minutes: 60, label: "1 hour" }, + { minutes: 120, label: "2 hours" }, + { minutes: 240, label: "4 hours" }, + { minutes: 480, label: "8 hours" }, + { minutes: 1440, label: "24 hours" }, + { minutes: 4320, label: "3 days" }, +]; + +// Tells the app shell's banner to refetch right away instead of waiting for its next poll. +function announceChange() { + window.dispatchEvent(new Event("maintenance-changed")); +} + +export default function Maintenance({ user }: { user: CurrentUser }) { + const canEdit = user.role === "admin" || user.role === "operator"; + const [windows, setWindows] = useState(null); + const [targets, setTargets] = useState(null); + const [error, setError] = useState(null); + const [now, setNow] = useState(() => Date.now()); + + const [target, setTarget] = useState(""); + const [minutes, setMinutes] = useState(60); + const [reason, setReason] = useState(""); + const [busy, setBusy] = useState(false); + + function load() { + api.maintenance + .list() + .then((res) => { + setWindows(res.windows); + setTargets(res.targets); + }) + .catch((err) => setError(readableError(err))); + } + + useEffect(() => { + load(); + const tick = setInterval(() => setNow(Date.now()), 30_000); + return () => clearInterval(tick); + }, []); + + async function start(e: React.FormEvent) { + e.preventDefault(); + const [targetType, id] = target.split(":"); + if (!targetType || !id) return; + setError(null); + setBusy(true); + try { + await api.maintenance.start({ + targetType: targetType as MaintenanceTargetType, + targetId: Number(id), + minutes, + reason: reason.trim() || undefined, + }); + setTarget(""); + setReason(""); + load(); + announceChange(); + } catch (err) { + setError(readableError(err)); + } finally { + setBusy(false); + } + } + + async function end(w: MaintenanceWindow) { + setError(null); + try { + await api.maintenance.end(w.id); + load(); + announceChange(); + } catch (err) { + setError(readableError(err)); + } + } + + return ( + <> +

Maintenance

+
+ Silence alerts about one server or integration while you work on it. Every window ends on its own — anything + still wrong when it ends alerts then, so nothing stays hidden. +
+ {error &&
{error}
} + + {canEdit && ( +
+
+

Start maintenance

+
+
+
+
+ + +
+
+ + +
+
+ + setReason(e.target.value)} + /> +
+
+
+ +
+
+
+ )} + +
+
+

Active

+
+
+ + + + + + + + {canEdit && } + + + + {windows?.map((w) => ( + + + + + + {canEdit && ( + + )} + + ))} + {windows?.length === 0 && ( + + + + )} + +
TargetReasonEndsStarted byActions
+ {w.targetName} · {w.targetKind} + {w.reason ?? "—"} + {formatRemaining(w.endsAt, now)} left + {formatDateTime(new Date(w.endsAt))} + {w.createdBy ?? "—"} + +
+ Nothing is in maintenance — all alerts are active. +
+
+
+ +
+
+

What gets silenced

+
+
+
+
+
Silenced for the target
+
    +
  • Server offline and disk-full alerts (for a server).
    • +
  • Storage, volume and disk-health alerts (for a Proxmox or Synology integration).
    • +
  • Backup-failure and uncovered-guest alerts (for a Proxmox integration).
    • +
  • + "Integration down" alerts for that type of service — these are tracked per type (e.g. all + Proxmox), not per instance, so with two Proxmox integrations a failure on the other one is silenced + too while a window is open. +
    • +
+
+
+
Not silenced
+
    +
  • Secret and certificate expiry, Tailscale key expiry, Docker update reminders.
    • +
  • DNS record change notifications.
    • +
+
When a window ends
+
+ A problem that started during it and is still there alerts on the next check (within 15 minutes). One + that was already alerted before it began and has since cleared is reported as cleared; one that + started and cleared entirely inside the window is never reported. +
+
+
+
+
+ + ); +} diff --git a/web/src/pages/Secrets.tsx b/web/src/pages/Secrets.tsx index 4484696..1387c1c 100644 --- a/web/src/pages/Secrets.tsx +++ b/web/src/pages/Secrets.tsx @@ -3,6 +3,7 @@ import { useSearchParams } from "react-router-dom"; import { api, type CurrentUser, type SecretInput, type SecretRecord, type SecretStatus } from "../api/client"; import { downloadCsv } from "../utils/csv"; import { formatDateTime } from "../utils/date"; +import { readableError } from "../utils/errors"; import { useSortable } from "../hooks/useSortable"; import SortableTh from "../components/SortableTh"; import { usePagination } from "../hooks/usePagination"; @@ -33,21 +34,6 @@ const emptyForm: SecretInput = { checkPort: 443, }; -/** The API client throws `Request failed (400): {json}` — pull the server's own message out of that when there is one. */ -function readableError(err: unknown): string { - const text = err instanceof Error ? err.message : String(err); - const match = text.match(/^Request failed \(\d+\): (.*)$/s); - if (match) { - try { - const body = JSON.parse(match[1]); - if (typeof body.message === "string") return body.message; - } catch { - // not JSON — fall through to the raw text - } - } - return text; -} - export default function Secrets({ user }: { user: CurrentUser }) { const canEdit = user.role === "admin" || user.role === "operator"; const [secrets, setSecrets] = useState(null); diff --git a/web/src/utils/duration.ts b/web/src/utils/duration.ts new file mode 100644 index 0000000..f1fe145 --- /dev/null +++ b/web/src/utils/duration.ts @@ -0,0 +1,9 @@ +/** "45 min", "2 h 10 min", "1 d 3 h" until an ISO timestamp (never negative). */ +export function formatRemaining(endsAtIso: string, now: number = Date.now()): string { + const minutes = Math.max(0, Math.ceil((new Date(endsAtIso).getTime() - now) / 60_000)); + if (minutes < 60) return `${minutes} min`; + const hours = Math.floor(minutes / 60); + if (hours < 24) return minutes % 60 ? `${hours} h ${minutes % 60} min` : `${hours} h`; + const days = Math.floor(hours / 24); + return hours % 24 ? `${days} d ${hours % 24} h` : `${days} d`; +} diff --git a/web/src/utils/errors.ts b/web/src/utils/errors.ts new file mode 100644 index 0000000..f243ad9 --- /dev/null +++ b/web/src/utils/errors.ts @@ -0,0 +1,14 @@ +/** The API client throws `Request failed (400): {json}` — pull the server's own message out of that when there is one. */ +export function readableError(err: unknown): string { + const text = err instanceof Error ? err.message : String(err); + const match = text.match(/^Request failed \(\d+\): (.*)$/s); + if (match) { + try { + const body = JSON.parse(match[1]); + if (typeof body.message === "string") return body.message; + } catch { + // not JSON — fall through to the raw text + } + } + return text; +}