Compare commits
181
Commits
5c70956f78
...
master
No files matched your search
+12
-2
@@ -1,12 +1,22 @@
|
|||||||
# Public URL the app is reachable at (used for OIDC redirect_uri and cookie behavior).
|
# Public URL the app is reachable at (used for OIDC redirect_uri and cookie behavior).
|
||||||
APP_BASE_URL=https://homelab.example.lan
|
APP_BASE_URL=https://homelab.example.lan
|
||||||
|
|
||||||
|
# Optional: make this installation look different, so a dev copy can't be taken for production.
|
||||||
|
# The sidebar takes the colour, a badge with the name is shown next to the title (and on the sign-in page),
|
||||||
|
# and the browser tab reads "[Dev] Homelab Manager". Leave both out for the normal look.
|
||||||
|
# ENVIRONMENT_COLOR is a name (red, orange, amber, green, teal, blue, purple, pink, gray) or a hex value like #b45309.
|
||||||
|
# A name without a colour gets orange.
|
||||||
|
# ENVIRONMENT_NAME=Dev
|
||||||
|
# ENVIRONMENT_COLOR=orange
|
||||||
|
|
||||||
# Random long string used to sign session cookies. Generate with:
|
# Random long string used to sign session cookies. Generate with:
|
||||||
# node -e "console.log(require('crypto').randomBytes(32).toString('hex'))"
|
# node -e "console.log(require('crypto').randomBytes(32).toString('hex'))"
|
||||||
|
# Required in production: without it the app refuses to start (the alternative is a value that is
|
||||||
|
# in the source code). Changing it signs everyone out.
|
||||||
SESSION_SECRET=change-me-to-a-random-64-char-hex-string
|
SESSION_SECRET=change-me-to-a-random-64-char-hex-string
|
||||||
|
|
||||||
# 32-byte (64 hex char) key used to encrypt stored integration API tokens, and the
|
# 32-byte (64 hex char) key used to encrypt stored integration API tokens, and the
|
||||||
# notification channels' credentials (Gotify/ntfy tokens, SMTP password, webhook
|
# notification channels' credentials (Gotify/ntfy/Telegram tokens, Pushover keys, SMTP password, webhook
|
||||||
# secret), at rest (AES-256-GCM). Generate the same way as SESSION_SECRET.
|
# secret), at rest (AES-256-GCM). Generate the same way as SESSION_SECRET.
|
||||||
# Losing/changing this key makes those stored credentials unreadable.
|
# Losing/changing this key makes those stored credentials unreadable.
|
||||||
CREDENTIALS_ENCRYPTION_KEY=change-me-to-a-random-64-char-hex-string
|
CREDENTIALS_ENCRYPTION_KEY=change-me-to-a-random-64-char-hex-string
|
||||||
@@ -28,5 +38,5 @@ AUTHENTIK_ISSUER_URL=https://authentik.example.lan/application/o/homelab-manager
|
|||||||
AUTHENTIK_CLIENT_ID=
|
AUTHENTIK_CLIENT_ID=
|
||||||
AUTHENTIK_CLIENT_SECRET=
|
AUTHENTIK_CLIENT_SECRET=
|
||||||
|
|
||||||
# Notification channels (Gotify, ntfy, SMTP, webhook) are configured in-app
|
# Notification channels (Gotify, ntfy, Telegram, Pushover, SMTP, webhook) are configured in-app
|
||||||
# under Settings, not here.
|
# under Settings, not here.
|
||||||
+312
@@ -0,0 +1,312 @@
|
|||||||
|
# Updating the agents
|
||||||
|
|
||||||
|
The agent on each server — [`agent/linux`](agent/linux) or [`agent/windows`](agent/windows) — is a copy of a
|
||||||
|
script that Homelab Manager serves, taken at the time it was installed. It does **not** update itself, and
|
||||||
|
updating Homelab Manager does not touch the agents already out on your servers. When a new version of
|
||||||
|
Homelab Manager teaches the agent something new, each server needs its agent installed again to learn it.
|
||||||
|
|
||||||
|
This page is about how to do that, safely, and how to tell it worked.
|
||||||
|
|
||||||
|
## Do I need to?
|
||||||
|
|
||||||
|
| You updated Homelab Manager to… | Agents need updating? |
|
||||||
|
|---|---|
|
||||||
|
| a version that only changed the app | No. Older agents keep working: a report is read the same way as before. |
|
||||||
|
| the version that added **pending updates** (the Updates page) | Yes, to see updates. Until then the server's row says *not reported*. Everything else about it carries on as before. |
|
||||||
|
|
||||||
|
Each agent script carries a version number and sends it in every report; Homelab Manager compares it with the
|
||||||
|
scripts it serves itself, so it can tell you which agents are behind. A change to an agent is only given a new
|
||||||
|
number when it is worth installing the agent again for.
|
||||||
|
|
||||||
|
| Agent version | What it added |
|
||||||
|
|---|---|
|
||||||
|
| 1 | Everything before the number was reported — so an agent that says no version is version 1. |
|
||||||
|
| 2 | Reports pending updates, and its own version. |
|
||||||
|
| 3 (Linux) | Asks the package manager outside the agent's systemd sandbox, so `dnf` and `zypper` hosts can answer (below). Install again to get it: the unit file changes too. |
|
||||||
|
| 3 (Windows) | Can replace itself with the newer script when automatic updates are switched on (below). |
|
||||||
|
| 4 (Linux) | Also reports failed services, disk health (SMART) and ZFS pool state; reads `pacman`; and can replace itself when automatic updates are switched on. |
|
||||||
|
| 4 (Windows) | Also reports failed services, disk health and Storage Spaces pools (below), and what the machine is (maker, model, serial number). |
|
||||||
|
| 6 (Linux), 5 (Windows) | Also says **what it couldn't collect and why** (no `smartctl`, no systemd, a package manager that didn't answer, Windows Update that timed out …), shown on the server's page with what to do about it (below). |
|
||||||
|
| 7 (Linux), 6 (Windows) | Also says **which operating system it runs** (distribution and release, or Windows product and build), so the server's page can say when that release stops being supported (below). |
|
||||||
|
| 8 (Linux), 7 (Windows) | Also says **how long the machine has been up**, shown as *Up for* in the status strip and as *Uptime* on the server's page. Older agents keep working; the figure just stays empty until the agent is installed again (or updates itself). |
|
||||||
|
| 5 (Linux) | Also says what the machine is — maker, model and serial number, read from the firmware — so the inventory can fill itself in (below). |
|
||||||
|
|
||||||
|
Reports are always read leniently — a field an older agent doesn't send is simply left alone, and a field
|
||||||
|
an agent sends that the server doesn't know yet is ignored — so there is never a rush, and a mixed fleet of
|
||||||
|
old and new agents is fine.
|
||||||
|
|
||||||
|
**How to tell which agents are old.** Homelab Manager says so: a server whose agent is behind shows an
|
||||||
|
*agent behind* badge on the **Servers** page, *update available* beside **Agent** on its own page, and *behind* in the
|
||||||
|
**Agent** column of the **Updates** page, which also counts them under *Agents to update*. (An agent that was
|
||||||
|
updated before version numbers existed reads as "v1 or older" until it has been updated once more.) On the
|
||||||
|
machine itself:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Linux — prints 1 or more if the script knows about updates, 0 if it doesn't
|
||||||
|
grep -c collect_updates /usr/local/bin/homelab-manager-agent.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
# Windows (elevated) — True if it does
|
||||||
|
Select-String -Path 'C:\ProgramData\HomelabManager\homelab-manager-agent.ps1' -Pattern 'Get-PendingUpdates' -Quiet
|
||||||
|
```
|
||||||
|
|
||||||
|
## What updating does
|
||||||
|
|
||||||
|
"Updating" is simply running the install command again. The installer is built to be run over an existing
|
||||||
|
install:
|
||||||
|
|
||||||
|
- it downloads the current agent script from your Homelab Manager and replaces the old one;
|
||||||
|
- it writes the URL and token again (the **same token keeps the same server**: its tasks, history, tags,
|
||||||
|
uptime and everything else stay exactly where they are — nothing is registered twice);
|
||||||
|
- on Linux it rewrites the systemd service and timer and reloads systemd; on Windows it replaces the
|
||||||
|
*Homelab Manager Agent* scheduled task;
|
||||||
|
- Windows sends a first report straight away so you see at once whether it worked.
|
||||||
|
|
||||||
|
It does not uninstall anything, does not touch the machine's own cron jobs, timers or tasks, and does not
|
||||||
|
install any updates itself.
|
||||||
|
|
||||||
|
Two settings go back to their defaults, so give them again if you had changed them:
|
||||||
|
|
||||||
|
- **Reporting interval** — back to 15 minutes unless you set `INTERVAL_MINUTES` on the command.
|
||||||
|
- **Windows `includeMicrosoftTasks`** — the installer rewrites `agent.json` with just the URL, token and
|
||||||
|
certificate setting. Put `INCLUDE_MICROSOFT_TASKS=true` in the environment afterwards, or add
|
||||||
|
`"includeMicrosoftTasks": true` to `agent.json` again.
|
||||||
|
|
||||||
|
## Getting the command
|
||||||
|
|
||||||
|
You need the server's URL and its token. There are two ways to have both.
|
||||||
|
|
||||||
|
### A. Reuse the token already on the machine (nothing to look up)
|
||||||
|
|
||||||
|
The token is already stored on each server, readable only by root / Administrators, so the machine can
|
||||||
|
update itself with the credentials it has. Run as root:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Linux
|
||||||
|
sudo sh -c '. /etc/homelab-manager-agent.env; curl -fsSL "$API_URL/agent/linux/install.sh" | API_URL="$API_URL" API_TOKEN="$API_TOKEN" API_INSECURE="$API_INSECURE" bash'
|
||||||
|
```
|
||||||
|
|
||||||
|
If Homelab Manager uses a **self-signed certificate**, the first download needs `-k` too (the agent's own
|
||||||
|
`API_INSECURE` is carried over from the file):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo sh -c '. /etc/homelab-manager-agent.env; curl -fsSLk "$API_URL/agent/linux/install.sh" | API_URL="$API_URL" API_TOKEN="$API_TOKEN" API_INSECURE="$API_INSECURE" bash'
|
||||||
|
```
|
||||||
|
|
||||||
|
In an **elevated** Windows PowerShell:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
$c = Get-Content 'C:\ProgramData\HomelabManager\agent.json' | ConvertFrom-Json
|
||||||
|
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
|
||||||
|
$env:API_URL = $c.apiUrl; $env:API_TOKEN = $c.apiToken
|
||||||
|
if ($c.insecure) { $env:API_INSECURE = 'true'; [Net.ServicePointManager]::ServerCertificateValidationCallback = { $true } }
|
||||||
|
iex ((New-Object Net.WebClient).DownloadString("$env:API_URL/agent/windows/install.ps1"))
|
||||||
|
```
|
||||||
|
|
||||||
|
(That form is for Windows PowerShell 5.1, the one built into Windows. See the
|
||||||
|
[Windows agent README](agent/windows/README.md) for PowerShell 7.)
|
||||||
|
|
||||||
|
### B. Rotate the token and use the command the page shows
|
||||||
|
|
||||||
|
In Homelab Manager: **Servers → Manage servers → Rotate token** on that server. The page shows a fresh
|
||||||
|
install command for it (with the self-signed-certificate box if you need it). Run it on the machine.
|
||||||
|
|
||||||
|
Use this when the token is lost or you want it replaced anyway. Know that rotating takes effect **at once**:
|
||||||
|
the old token stops working, so that server's agent can't report until the new command has been run on it —
|
||||||
|
and the server will look as if it has stopped reporting (and alert) in the meantime. Do one server at a
|
||||||
|
time, and put it under [maintenance](README.md) first if the gap could be more than a few minutes.
|
||||||
|
|
||||||
|
## Updating many servers
|
||||||
|
|
||||||
|
For more than a handful, use the tool you already have. The commands in A are plain shell and PowerShell, so
|
||||||
|
Ansible or Semaphore can run them. For example, a playbook for the Linux hosts (an example — adjust the
|
||||||
|
group name, and add `-k` to `curl` for a self-signed certificate):
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
- hosts: linux_servers
|
||||||
|
become: true
|
||||||
|
tasks:
|
||||||
|
- name: Update the Homelab Manager agent (same token as before)
|
||||||
|
ansible.builtin.shell: |
|
||||||
|
set -e
|
||||||
|
. /etc/homelab-manager-agent.env
|
||||||
|
curl -fsSL "$API_URL/agent/linux/install.sh" | API_URL="$API_URL" API_TOKEN="$API_TOKEN" API_INSECURE="$API_INSECURE" bash
|
||||||
|
args:
|
||||||
|
executable: /bin/bash
|
||||||
|
changed_when: true
|
||||||
|
```
|
||||||
|
|
||||||
|
Homelab Manager can start such a template for you, limited to one host, from the server's own page (*Run a
|
||||||
|
playbook*) — a good way to try it on one machine first, then run it on the rest from Semaphore.
|
||||||
|
|
||||||
|
## Checking it worked
|
||||||
|
|
||||||
|
1. **The Updates page.** The server's row changes from *not reported* to its figures, and its **Agent** to the
|
||||||
|
current version, within one reporting interval (15 minutes by default; on Windows the installer reports straight away, and on Linux you can
|
||||||
|
start one now — below).
|
||||||
|
2. **Last seen** on the server's page moves to *just now*.
|
||||||
|
|
||||||
|
On a Linux server:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
systemctl list-timers homelab-manager-agent.timer # when it next runs
|
||||||
|
sudo systemctl start homelab-manager-agent.service # report now
|
||||||
|
sudo journalctl -u homelab-manager-agent -n 30 # what it said (look for "Note:" lines)
|
||||||
|
sudo /usr/local/bin/homelab-manager-agent.sh --dry-run | jq .system.updates # what it would send; sends nothing
|
||||||
|
```
|
||||||
|
|
||||||
|
On a Windows server (elevated):
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
Start-ScheduledTask -TaskName 'Homelab Manager Agent' # report now
|
||||||
|
Get-ScheduledTaskInfo -TaskName 'Homelab Manager Agent' # LastRunTime, LastTaskResult (0 = fine)
|
||||||
|
powershell -NoProfile -ExecutionPolicy Bypass -File 'C:\ProgramData\HomelabManager\homelab-manager-agent.ps1' -DryRun # prints what it would send
|
||||||
|
```
|
||||||
|
|
||||||
|
## Letting agents update themselves
|
||||||
|
|
||||||
|
**Servers → "Update agents automatically"** (administrators; off by default). With it on, an agent that is behind the script
|
||||||
|
this Homelab Manager serves downloads the newer one and replaces itself at its next run — no install command to run again.
|
||||||
|
|
||||||
|
How it stays safe:
|
||||||
|
|
||||||
|
- The agent asks first (`GET /api/agent/report/update?version=N`, with its own token). The answer is yes only when the switch
|
||||||
|
is on and the agent is behind, and it carries the **SHA-256 of the script**.
|
||||||
|
- The agent downloads the script and installs it only if the checksum matches, it is newer than the running one, it says
|
||||||
|
the version that was promised, and it parses (`bash -n` on Linux, PowerShell's own parser on Windows). Anything else is thrown
|
||||||
|
away and the agent carries on as it was; a note goes to the journal (Linux) or the task output (Windows).
|
||||||
|
- The new file is written beside the old one and moved over it in one step, so there is never half a script.
|
||||||
|
- On Linux this happens in the same step that is outside the sandbox (`ExecStartPre=-+…`, below), because the sandbox keeps
|
||||||
|
`/usr` read-only for the report itself. That step runs first, so the report that follows is already the new version.
|
||||||
|
On Windows the new script runs from the next report.
|
||||||
|
- Each update is written down as a line in the server's history ("Agent updated to version 4"), and switching the setting
|
||||||
|
is in the audit log.
|
||||||
|
|
||||||
|
What it can't do:
|
||||||
|
|
||||||
|
- **Update an agent that doesn't have the update code yet.** An agent has to contain the self-update itself: Linux version 4 or
|
||||||
|
later, Windows version 3 or later. Getting there is one install command run on each host (the last one you need to run by hand);
|
||||||
|
after that, new versions arrive by themselves. A Linux host whose unit was written by version 3 keeps working: the version 4
|
||||||
|
script still understands the unit's older step name.
|
||||||
|
- **Replace the systemd unit.** Self-update replaces the script only. A release that also needs a changed unit says so in the
|
||||||
|
version table above ("install again").
|
||||||
|
- **Update an agent that isn't reporting.**
|
||||||
|
- **Choose per server.** It is one switch for all agents.
|
||||||
|
|
||||||
|
## What the Linux agent reports about the machine (version 4)
|
||||||
|
|
||||||
|
| What | Where it comes from | Notes |
|
||||||
|
|---|---|---|
|
||||||
|
| **Failed services** | `systemctl list-units --state=failed` | Read on every report (it works inside the sandbox). Shown on the server's page as *Host health* and raised as an alert of kind *Failing services*. |
|
||||||
|
| **Disk health** | `smartctl -j -H -A` for each disk it lists | Needs `smartmontools` installed (the section is simply left out without it) and the devices, so it is read in the step outside the sandbox. Taken **about once an hour** and with `-n standby`, so a disk that is asleep isn't woken: what was last known of it is kept (in `/var/lib/homelab-manager-agent`). A disk that says its SMART check **fails** is a critical alert; one that has used 90% or more of its rated life is a warning. Virtual disks have no SMART and are skipped. |
|
||||||
|
| **ZFS pools** | `zpool list` | Needs the `zpool` command. A pool in any state but ONLINE is a critical alert. |
|
||||||
|
|
||||||
|
A report that leaves one of these out leaves what was last said alone (an agent that skips a part never turns an old problem into
|
||||||
|
"all well"); an empty list is an answer ("none failed") and clears the old one.
|
||||||
|
|
||||||
|
## What the Windows agent reports about the machine (version 4)
|
||||||
|
|
||||||
|
The same three parts, from what Windows itself knows (the agent runs as SYSTEM, so nothing needs installing):
|
||||||
|
|
||||||
|
| What | Where it comes from | Notes |
|
||||||
|
|---|---|---|
|
||||||
|
| **Failed services** | `Win32_Service` | A service that starts by itself, isn't running, and stopped with an error. Services that finished normally (exit code 0), that start on a trigger (1077) or that are starting up are not failures. Some services always do this (updaters, for one): list them under `ignoreServices` in `agent.json` (or `IGNORE_SERVICES` in the environment), as names or patterns like `gupdate*` — or, easier and for either system, on the server's page (below). |
|
||||||
|
| **Disk health** | `Get-PhysicalDisk` and its reliability counters | Windows' own verdict: *Healthy* passes, *Warning* and *Unhealthy* fail, anything else says nothing. Temperature and wear come from the counters where the drive gives sane ones. Virtual disks are left out. |
|
||||||
|
| **Storage pools** | `Get-StoragePool` (Storage Spaces) | *Healthy*, *Warning* and *Unhealthy* are reported as ONLINE, DEGRADED and FAULTED, with how full the pool is, and show beside ZFS pools. A machine with no pools reports an empty list. |
|
||||||
|
|
||||||
|
## Services to watch, from the server's page
|
||||||
|
|
||||||
|
Services are **opt-in**. The agents report every service that has failed, but none of them raises an alert until you say it matters: on the server's **Host health** card an administrator
|
||||||
|
types a name or a pattern with `*` (`nginx`, `docker*`, `postgresql*`; case doesn't matter and `.service` is optional; `*` alone watches every service, which is how it
|
||||||
|
worked before), or presses **Watch** beside a service that has failed. A watched service that has failed raises the alert and counts in `/metrics`; one that is not watched still shows on the page,
|
||||||
|
marked *not watched*, with its **Watch** button. It takes effect the moment it is saved, with no agent to update. The list belongs to that server (it is kept with its own alert limits) and works
|
||||||
|
the same for Linux and Windows; the Windows agent's own `ignoreServices` (which leaves a service out of what the agent reports at all) still works as well. Only a service that has *failed* is seen:
|
||||||
|
one that was merely stopped on Linux is not reported by the agent.
|
||||||
|
|
||||||
|
## What the agent couldn't collect (Linux version 6, Windows version 5)
|
||||||
|
|
||||||
|
Each agent notes, in every report, the checks it couldn't make: nothing there to read it with (*unavailable*) or it tried and couldn't (*failed*), why in a
|
||||||
|
word, and for a failure at most a line of what the tool said. The server's page turns that into a card, **What the agent couldn't collect**, with words and a
|
||||||
|
fix for each — "Disk health isn't reported: smartctl isn't installed → install smartmontools" — so the question *why is there no disk health here?* has an
|
||||||
|
answer on the page. A tip is simply how the machine is (a virtual machine with no serial number); *look at* is something to fix. A machine with no ZFS isn't
|
||||||
|
told it has no ZFS. The card also says when the agent is behind. The wording lives in the server, so it improves without installing anything again.
|
||||||
|
|
||||||
|
On Linux, the step outside the sandbox (`--prepare`) leaves its notes with the other answers, and the report carries them; if that step didn't run at all
|
||||||
|
(an older systemd unit, or a copy run by hand) that is itself noted, because disk health and ZFS are only read there. `--check-updates` prints the same notes.
|
||||||
|
|
||||||
|
## What the machine is (Linux version 5, Windows version 4)
|
||||||
|
|
||||||
|
The agents say who made the machine, its model and its serial number (Linux reads `/sys/class/dmi/id`, where the serial is only
|
||||||
|
readable by root; Windows asks the computer system, the BIOS, and the motherboard when those say nothing — a home-built PC). What
|
||||||
|
a maker writes when it has nothing to say ("To Be Filled By O.E.M.", "Default string", all zeros) is left out, and many virtual
|
||||||
|
machines and single-board computers say nothing at all. The figures are kept on the server and used by the **Inventory**: a
|
||||||
|
linked entry gets its blank maker, model and serial number filled in, and *Refresh from agent* on the Inventory page replaces
|
||||||
|
what is written there with what the agent says, when you choose to.
|
||||||
|
|
||||||
|
To see exactly what a host finds, as root: `sudo /usr/local/bin/homelab-manager-agent.sh --check-updates` — it prints the
|
||||||
|
update figures, the failed services, the SMART readings and the ZFS pools, says whether it would update itself, and changes nothing.
|
||||||
|
|
||||||
|
## What the machine runs (Linux version 7, Windows version 6)
|
||||||
|
|
||||||
|
Each agent says which operating system it runs, in the report's `system.os`: `{ id, name, version }`.
|
||||||
|
|
||||||
|
- **Linux** reads `/etc/os-release` — `ID` (lower case, e.g. `ubuntu`, `debian`, `rocky`), `VERSION_ID` and `PRETTY_NAME`. On a **Proxmox VE** host (where `pveversion` exists) it says `proxmox-ve` and the
|
||||||
|
release instead, since the Debian underneath is not what needs watching.
|
||||||
|
- **Windows** says `windows`, the product name without "Microsoft " (e.g. "Windows Server 2022 Standard") and the build number — which is how client and server releases are told apart.
|
||||||
|
|
||||||
|
The server keeps it with the machine and compares the release with a built-in table of end-of-support dates (Ubuntu, Debian, CentOS, RHEL and its rebuilds, Fedora, Alpine, openSUSE Leap, Proxmox VE, Windows
|
||||||
|
client and server). A server's page shows it in its *System* row with *supported*, *ends in N days* or *support ended*; one past its end — or inside the warning window under Settings → Notifications (*Flag an
|
||||||
|
operating system whose support ends within*, 90 days) — is on the Alerts page, and `operatingSystem` is in the read API. A release the table does not know is shown without a badge, never guessed at. An agent
|
||||||
|
older than this version reports no OS and the server shows nothing for it.
|
||||||
|
|
||||||
|
`sudo /usr/local/bin/homelab-manager-agent.sh --check-updates` prints what it found.
|
||||||
|
|
||||||
|
## How the update check works — and why a server may stay "not reported"
|
||||||
|
|
||||||
|
The agent only *asks*; it never installs, downloads or refreshes anything.
|
||||||
|
|
||||||
|
| System | What it asks | Notes |
|
||||||
|
|---|---|---|
|
||||||
|
| Debian, Ubuntu, Raspberry Pi OS (`apt`) | a simulated upgrade (`apt-get -s upgrade`) | Uses the package lists already on the machine. If nothing refreshes them (`apt update`, or the distribution's own daily timer), the count is only as fresh as they are. Security updates are those from a `-security` archive. |
|
||||||
|
| Fedora, RHEL and relatives (`dnf`) | `dnf check-update --cacheonly` | Reads dnf's cache only; if the cache is empty the server stays *not reported* (`sudo dnf makecache` fills it, and dnf's own timer keeps it fresh). A restart is flagged by `needs-restarting -r` when that tool is installed. |
|
||||||
|
| openSUSE, SLES (`zypper`) | `zypper list-updates` without refreshing | Security figure not available (shown as unknown). |
|
||||||
|
| Arch and relatives (`pacman`) | `pacman -Qu` | What the local copy of the sync databases says could be upgraded (what `checkupdates` lists); packages held back are left out; no security figure. A restart is flagged when the running kernel is no longer the installed one. |
|
||||||
|
| Windows | Windows Update, through its own API | Searching can take a minute or two, so it is done at most **every six hours** and kept in `%ProgramData%\HomelabManager\updates.json`. It runs in a separate process with a 150-second limit (the scheduled task itself is stopped after five minutes). If the search fails or times out it isn't tried again for an hour, and that report leaves updates out. |
|
||||||
|
|
||||||
|
**Not reported** means the agent didn't give a figure. The usual reasons:
|
||||||
|
|
||||||
|
1. **It is an older agent.** Update it (above). This is by far the most common.
|
||||||
|
2. **Another package manager** — Alpine (`apk`) and others aren't read yet. (The Linux installer needs systemd, which Alpine doesn't use.)
|
||||||
|
3. **The check couldn't run.** On Linux the agent writes a line to the journal when its package manager
|
||||||
|
couldn't be read — `journalctl -u homelab-manager-agent -n 30`. The agent itself runs inside a tight systemd
|
||||||
|
sandbox (read-only system, no hardware, minimal privileges, see `install.sh`), and `dnf` and `zypper` write to
|
||||||
|
`/var` and `/run` even when only asked a question. So from agent version 3 the unit asks the package manager in
|
||||||
|
one separate step *before* the report — `ExecStartPre=-+…homelab-manager-agent.sh --prepare-updates`, where
|
||||||
|
`+` means outside the sandbox and `-` means a failure doesn't stop the report — and hands the answer to the
|
||||||
|
report through a private directory (`/run/homelab-manager-agent/updates.json`, removed when the run ends).
|
||||||
|
That step only runs the package manager's read-only questions. The report itself keeps the full sandbox.
|
||||||
|
To see what it finds on a host, as root:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo /usr/local/bin/homelab-manager-agent.sh --check-updates
|
||||||
|
```
|
||||||
|
|
||||||
|
It prints what the package manager said (errors included) and the figures that would be reported. To try the
|
||||||
|
whole path the way the timer does it: `sudo systemctl start homelab-manager-agent` and then
|
||||||
|
`journalctl -u homelab-manager-agent -n 30`.
|
||||||
|
4. **The package manager was busy** (another update running). It is tried again at the next report.
|
||||||
|
|
||||||
|
A server that says nothing keeps what it last said — an agent that skips a report never turns an old figure
|
||||||
|
into "up to date".
|
||||||
|
|
||||||
|
## If something goes wrong
|
||||||
|
|
||||||
|
- **The installer fails on download** — the machine can't reach Homelab Manager's `/agent/...` address, or the
|
||||||
|
certificate isn't trusted (use the self-signed forms above). Nothing has changed on the machine yet.
|
||||||
|
- **The installer ran but the server doesn't report** — check the token (a rotated token makes the old one
|
||||||
|
stop at once) and `API_URL`: `journalctl -u homelab-manager-agent` on Linux, the scheduled task's last
|
||||||
|
result on Windows.
|
||||||
|
- **To go back** — there is no older version to fetch; the agent a Homelab Manager serves is the one that
|
||||||
|
came with it. To remove an agent, use the uninstall command shown on the Servers page; the server's entry and
|
||||||
|
history stay until you delete it there.
|
||||||
@@ -0,0 +1,334 @@
|
|||||||
|
# Backup and restore
|
||||||
|
|
||||||
|
Everything Homelab Manager knows is in **one SQLite file**, plus a handful of settings in your
|
||||||
|
`.env`. A backup is those two things — and the part people get wrong is keeping them **apart**
|
||||||
|
(see [Keeping backups safe](#keeping-backups-safe)).
|
||||||
|
|
||||||
|
- [The built-in backups (Settings → Data → Backup)](#the-built-in-backups-settings--data--backup)
|
||||||
|
- [What to back up](#what-to-back-up)
|
||||||
|
- [Taking a backup](#taking-a-backup)
|
||||||
|
- [Checking a backup](#checking-a-backup)
|
||||||
|
- [Before an upgrade](#before-an-upgrade)
|
||||||
|
- [Keeping backups safe](#keeping-backups-safe)
|
||||||
|
- [Automating it](#automating-it)
|
||||||
|
- [Restoring](#restoring)
|
||||||
|
- [The encrypted export in Settings → Data → Backup](#the-encrypted-export-in-settings--data--backup)
|
||||||
|
- [If there is no backup](#if-there-is-no-backup)
|
||||||
|
- [What has been tested, and what hasn't](#what-has-been-tested-and-what-hasnt)
|
||||||
|
|
||||||
|
The commands assume the standard Docker setup from the README: you run them in the folder that
|
||||||
|
holds `docker-compose.yml`, `.env` and `data/`, and the service is called `homelab-manager`.
|
||||||
|
|
||||||
|
## The built-in backups (Settings → Data → Backup)
|
||||||
|
|
||||||
|
You don't need the snippets below for the everyday case: Homelab Manager takes a backup of its own
|
||||||
|
database **every day** (03:30 in the notification time zone, the newest 14 kept — both changeable, and it
|
||||||
|
can be switched off; backups you make by hand, upload or restore from are kept too: the newest ten, three and three), checks each one as it makes it, and keeps them in `data/backups/` with the same
|
||||||
|
names the snippets use. **Settings → Data → Backup** shows them and does the rest:
|
||||||
|
|
||||||
|
- **Back up now** makes one on the spot. **Download** takes a copy to your computer (the quickest way
|
||||||
|
off the machine). **Check** opens a backup and says whether SQLite finds it sound, how many users,
|
||||||
|
servers and integrations it holds, and whether it is from an older or newer version.
|
||||||
|
- **Upload a backup…** takes a file from somewhere else (another machine, the script below) and keeps it
|
||||||
|
if it checks out. A file that isn't a Homelab Manager database is refused and nothing is kept.
|
||||||
|
- **Restore…** checks the backup and *stages* it. Nothing changes until the app next starts — restart it
|
||||||
|
(`docker compose restart homelab-manager`) and, before anything opens the database, the staged file
|
||||||
|
takes the database's place. The database that was in use is **kept** in `data/backups/` as
|
||||||
|
`…-pre-restore.sqlite`, so a restore that wasn't what you wanted is undone by restoring that file. If
|
||||||
|
something stops it half way, the old database is put back. A backup from a **newer** version than the
|
||||||
|
one running is refused; an older one is fine (the start-up brings it up to date). **Cancel the restore**
|
||||||
|
until you restart if you change your mind.
|
||||||
|
- If the newest backup is more than two days old, or a scheduled one fails, there is an **alert** (and a
|
||||||
|
message through your channels), and `/metrics` has `homelab_selfbackup_last_success_timestamp_seconds`
|
||||||
|
for your own monitoring.
|
||||||
|
|
||||||
|
Three things it does **not** do, and the rest of this page is about: it does not hold
|
||||||
|
`CREDENTIALS_ENCRYPTION_KEY` (keep that separately — see [Keeping backups safe](#keeping-backups-safe)),
|
||||||
|
it does not copy anything off the machine on its own (Download, or copy `data/backups/` with your usual
|
||||||
|
tool), and it does not replace a restore drill. Access to all of it is for administrators only.
|
||||||
|
|
||||||
|
## What to back up
|
||||||
|
|
||||||
|
| What | Where | Why | If you lose it |
|
||||||
|
|---|---|---|---|
|
||||||
|
| **The database** | `data/homelab-manager.sqlite` | Everything: users and roles, servers and their agent tokens (as hashes) with their history (load, memory, disks), scheduled tasks, secrets, IP addresses, DNS caches, domains, ports, tags, notes, to-do items, runbooks, the hardware inventory, endpoint checks and heartbeats with their results, your own alert rules, maintenance windows and snoozes, dependencies, incidents, what was read from UniFi, Proxmox and Synology storage, and Home Assistant, both logs, settings, the integrations and DNS providers (with their credentials, encrypted), the alert-ticket memory | You start from nothing — see [If there is no backup](#if-there-is-no-backup) |
|
||||||
|
| **`CREDENTIALS_ENCRYPTION_KEY`** | `.env` | Decrypts every stored credential: integration tokens and passwords, DNS provider credentials, and the Gotify/ntfy/Telegram/Pushover/SMTP/webhook credentials | The database restores, but every credential in it is unreadable and has to be entered again |
|
||||||
|
| **The rest of `.env`** | `.env` | `SESSION_SECRET`, `APP_BASE_URL`, the Authentik client id and secret | `SESSION_SECRET`: everyone has to sign in again, nothing more. The Authentik values can be copied from Authentik again |
|
||||||
|
| Your `docker-compose.yml` and the image version | next to `.env` | So you can bring it back the way it was. Note which **version** each backup came from (the image tag, or the git commit) | Re-create it from the README |
|
||||||
|
|
||||||
|
You do **not** need `data/sessions/` (the sign-ins; everyone just signs in again), the Docker image
|
||||||
|
(it is rebuilt or pulled), or anything in Authentik, osTicket, Proxmox and the other systems the
|
||||||
|
app talks to — those have their own backups.
|
||||||
|
|
||||||
|
The agents on your servers keep working after a restore: each server's token is stored in the
|
||||||
|
database, as a hash, and the agent holds the token itself.
|
||||||
|
|
||||||
|
## Taking a backup
|
||||||
|
|
||||||
|
### Option A — stop, copy, start (simplest)
|
||||||
|
|
||||||
|
A few seconds of downtime, nothing to install, and nothing that can go wrong halfway:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose stop homelab-manager
|
||||||
|
sudo tar czf ~/homelab-manager-data-$(date +%F-%H%M).tar.gz --exclude=data/sessions --exclude=data/backups data
|
||||||
|
docker compose start homelab-manager
|
||||||
|
```
|
||||||
|
|
||||||
|
(`sudo` because the container runs as root, so the files in `data/` are root's.)
|
||||||
|
|
||||||
|
### Option B — while it runs
|
||||||
|
|
||||||
|
Safe to run while the app is working: SQLite makes a consistent snapshot of the file with
|
||||||
|
`VACUUM INTO`, even if something is writing at that moment. The container has no `sqlite3` tool,
|
||||||
|
so this uses Node and the library the app already has. The snapshot lands in `data/backups/` on
|
||||||
|
your host, named with the date and time:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose exec -T homelab-manager node - <<'EOF'
|
||||||
|
const fs = require("fs");
|
||||||
|
const { createClient } = require("@libsql/client");
|
||||||
|
const dir = "/app/data/backups";
|
||||||
|
fs.mkdirSync(dir, { recursive: true });
|
||||||
|
const stamp = new Date().toISOString().slice(0, 19).replace(/[:T]/g, "-");
|
||||||
|
const out = `${dir}/homelab-manager-${stamp}.sqlite`;
|
||||||
|
const c = createClient({ url: "file:/app/data/homelab-manager.sqlite" });
|
||||||
|
c.execute(`VACUUM INTO '${out}'`).then(
|
||||||
|
() => { console.log("wrote", out); c.close(); },
|
||||||
|
(e) => { console.error("backup failed:", e.message); process.exit(1); },
|
||||||
|
);
|
||||||
|
EOF
|
||||||
|
```
|
||||||
|
|
||||||
|
It takes well under a second for a database of this size. SQLite will not overwrite an existing
|
||||||
|
file, which is why the name includes the time; two runs in the same second would collide.
|
||||||
|
|
||||||
|
**Which one?** Use B for the regular, automatic backup and A before a risky change if you want
|
||||||
|
the extra certainty. Either way, the file stays on the same disk as the app until you copy it
|
||||||
|
somewhere else — see below.
|
||||||
|
|
||||||
|
Also copy `.env` — separately from the database (next section):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cp .env ~/homelab-manager-env-$(date +%F)
|
||||||
|
chmod 600 ~/homelab-manager-env-$(date +%F)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Checking a backup
|
||||||
|
|
||||||
|
A backup you have never opened is a hope, not a backup. This opens the **newest** file in
|
||||||
|
`data/backups/`, runs SQLite's own integrity check and shows what is in it:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose exec -T homelab-manager node - <<'EOF'
|
||||||
|
const fs = require("fs");
|
||||||
|
const { createClient } = require("@libsql/client");
|
||||||
|
const dir = "/app/data/backups";
|
||||||
|
const newest = fs.readdirSync(dir).filter((f) => f.endsWith(".sqlite")).sort().pop();
|
||||||
|
if (!newest) { console.error("no backups found in", dir); process.exit(1); }
|
||||||
|
const c = createClient({ url: `file:${dir}/${newest}` });
|
||||||
|
(async () => {
|
||||||
|
const one = async (sql) => Object.values((await c.execute(sql)).rows[0])[0];
|
||||||
|
console.log(newest);
|
||||||
|
console.log(" integrity check:", await one("PRAGMA integrity_check"));
|
||||||
|
console.log(" migrations applied:", await one("select count(*) from __drizzle_migrations"));
|
||||||
|
console.log(" users:", await one("select count(*) from users"), "| servers:", await one("select count(*) from servers"), "| integrations:", await one("select count(*) from integrations"));
|
||||||
|
c.close();
|
||||||
|
})();
|
||||||
|
EOF
|
||||||
|
```
|
||||||
|
|
||||||
|
You want `integrity check: ok` and numbers that look like your system. The migration count is the
|
||||||
|
version of the database: it goes up by one whenever a release changes the schema.
|
||||||
|
|
||||||
|
A tar from Option A is checked by listing it: `tar tzf homelab-manager-data-….tar.gz | head`.
|
||||||
|
|
||||||
|
Now and then — say after you first set it up, and after big changes — do a real **restore
|
||||||
|
drill**: copy a backup to a spare machine or a second folder, start a second instance on another
|
||||||
|
port with the same `.env`, and sign in. That is the only test that proves the whole chain,
|
||||||
|
including the key.
|
||||||
|
|
||||||
|
## Before an upgrade
|
||||||
|
|
||||||
|
Take a backup first, every time. A release can change the database, and the change **only goes
|
||||||
|
forward**: the new version upgrades the file the first time it starts.
|
||||||
|
|
||||||
|
- Restoring an **older** backup into a **newer** version is fine; the newer version brings it
|
||||||
|
up to date when it starts, and keeps the data.
|
||||||
|
- Restoring a **newer** backup into an **older** version is **not supported**. The older version
|
||||||
|
doesn't know the newer tables or columns and may fail in odd ways. If you roll back the
|
||||||
|
software, roll back the data with it: use the backup you took before the upgrade.
|
||||||
|
|
||||||
|
## Keeping backups safe
|
||||||
|
|
||||||
|
A database backup holds a lot: the audit log, every server's name and address, the hashed agent
|
||||||
|
tokens, and all your credentials — encrypted, which is exactly why the key matters.
|
||||||
|
|
||||||
|
1. **Keep the key apart from the database.** If the database file and `CREDENTIALS_ENCRYPTION_KEY`
|
||||||
|
are stolen together, the credentials are readable. Put the database backups where they are
|
||||||
|
convenient, and the key in a password manager or somewhere else entirely. You need both to
|
||||||
|
restore a working system, and neither alone is useful to a thief.
|
||||||
|
2. **Copy it off the machine.** `data/backups/` sits on the same disk as the app, so a dead disk
|
||||||
|
takes both. Copy the files to another machine, a NAS, or encrypted cloud storage.
|
||||||
|
3. **Restrict access.** `sudo chmod -R go-rwx data/backups` and the same for wherever you copy to.
|
||||||
|
4. **Keep a few.** The last 7 daily and 4 weekly is plenty for a system that changes this slowly.
|
||||||
|
A single backup is not enough: a problem you only notice a week later is already in last night's.
|
||||||
|
|
||||||
|
## Automating it
|
||||||
|
|
||||||
|
A script that takes the snapshot, keeps the newest 14, and (if you fill it in) copies the newest
|
||||||
|
off the machine. Save it as `/usr/local/bin/homelab-manager-backup.sh`, make it executable, and
|
||||||
|
set `APP_DIR`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
APP_DIR=/opt/homelab-manager # the folder with docker-compose.yml and data/
|
||||||
|
KEEP=14
|
||||||
|
|
||||||
|
cd "$APP_DIR"
|
||||||
|
docker compose exec -T homelab-manager node - <<'EOF'
|
||||||
|
const fs = require("fs");
|
||||||
|
const { createClient } = require("@libsql/client");
|
||||||
|
const dir = "/app/data/backups";
|
||||||
|
fs.mkdirSync(dir, { recursive: true });
|
||||||
|
const stamp = new Date().toISOString().slice(0, 19).replace(/[:T]/g, "-");
|
||||||
|
const out = `${dir}/homelab-manager-${stamp}.sqlite`;
|
||||||
|
const c = createClient({ url: "file:/app/data/homelab-manager.sqlite" });
|
||||||
|
c.execute(`VACUUM INTO '${out}'`).then(
|
||||||
|
() => { console.log("wrote", out); c.close(); },
|
||||||
|
(e) => { console.error("backup failed:", e.message); process.exit(1); },
|
||||||
|
);
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# keep the newest $KEEP, delete the older ones (other files in the folder are left alone)
|
||||||
|
ls -1t data/backups/homelab-manager-*.sqlite | tail -n +$((KEEP + 1)) | xargs -r rm --
|
||||||
|
|
||||||
|
# copy the newest one off this machine — edit, or remove if you do that another way
|
||||||
|
# rsync -a "$(ls -1t data/backups/homelab-manager-*.sqlite | head -1)" backuphost:/backups/homelab-manager/
|
||||||
|
```
|
||||||
|
|
||||||
|
Run it nightly from root's crontab (`sudo crontab -e`):
|
||||||
|
|
||||||
|
```
|
||||||
|
17 3 * * * /usr/local/bin/homelab-manager-backup.sh >> /var/log/homelab-manager-backup.log 2>&1
|
||||||
|
```
|
||||||
|
|
||||||
|
A script that fails quietly is the usual way backups stop. Look at the log now and then, or add
|
||||||
|
a line that pings something you watch (Uptime Kuma's push monitors are good at this: no ping,
|
||||||
|
you get told).
|
||||||
|
|
||||||
|
## Restoring
|
||||||
|
|
||||||
|
You need: a backup of the database, and the `CREDENTIALS_ENCRYPTION_KEY` that was in use when it
|
||||||
|
was taken (from your `.env` backup).
|
||||||
|
|
||||||
|
1. **Stop the app.**
|
||||||
|
```bash
|
||||||
|
docker compose stop homelab-manager
|
||||||
|
```
|
||||||
|
2. **Keep what is there**, in case the restore isn't what you wanted:
|
||||||
|
```bash
|
||||||
|
sudo mv data "data.before-restore-$(date +%F-%H%M)"
|
||||||
|
mkdir data
|
||||||
|
```
|
||||||
|
(If your backups live in `data/backups/`, they are now in the folder you just moved. Take the
|
||||||
|
one you want from there.)
|
||||||
|
3. **Put the database in place**, under the name the app expects:
|
||||||
|
```bash
|
||||||
|
sudo cp /path/to/homelab-manager-2026-10-07-03-17-00.sqlite data/homelab-manager.sqlite
|
||||||
|
```
|
||||||
|
From an Option A tar instead: `sudo tar xzf homelab-manager-data-….tar.gz` in this folder.
|
||||||
|
4. **Check `.env`.** On the same machine it is probably still right. On a new machine, restore it
|
||||||
|
from your backup. The one value that must match is `CREDENTIALS_ENCRYPTION_KEY`.
|
||||||
|
5. **Start it.** The first start brings the database up to the current version by itself.
|
||||||
|
```bash
|
||||||
|
docker compose up -d
|
||||||
|
docker compose logs --tail 30 homelab-manager
|
||||||
|
curl -fsS http://localhost:3000/health
|
||||||
|
```
|
||||||
|
6. **Sign in** and look around: the Servers page should show your servers (as "last seen" at the
|
||||||
|
time of the backup, until their agents report again, within 15 minutes), and Integrations
|
||||||
|
should list everything.
|
||||||
|
|
||||||
|
### If the key doesn't match
|
||||||
|
|
||||||
|
Symptoms: an integration or a DNS provider fails with *"Unsupported state or unable to
|
||||||
|
authenticate data"*, and notification channels that worked before don't. The database is fine;
|
||||||
|
only the stored credentials can't be read. If you can't find the old key, enter the credentials
|
||||||
|
again (Integrations → edit, DNS providers, Settings → Notifications). Nothing else is lost.
|
||||||
|
**Don't** change `CREDENTIALS_ENCRYPTION_KEY` on a running system to "fix" this: it makes
|
||||||
|
everything stored so far unreadable.
|
||||||
|
|
||||||
|
### After a restore
|
||||||
|
|
||||||
|
- Servers whose agents were installed **after** the backup was taken have no token in the
|
||||||
|
restored database. Press **Rotate token** on that server's row on the Servers page: it shows the new token together with a ready-to-copy install command, to run on that server.
|
||||||
|
- Anything changed since the backup is gone: new users, edits, audit entries. The audit log will
|
||||||
|
not show the gap, so it is worth noting it somewhere.
|
||||||
|
- Everyone has to sign in again if you restored without `data/sessions/`. That is expected.
|
||||||
|
|
||||||
|
## The encrypted export in Settings → Data → Backup
|
||||||
|
|
||||||
|
A different tool, for a different job. **Settings → Data → Backup** downloads one file, encrypted with a
|
||||||
|
passphrase you choose (AES-256-GCM, key derived with scrypt), holding:
|
||||||
|
|
||||||
|
- the settings (notification channels, display, log retention, alert tickets, and so on),
|
||||||
|
- the integrations and DNS providers, with their credentials.
|
||||||
|
|
||||||
|
Because it is encrypted with your passphrase and not with `CREDENTIALS_ENCRYPTION_KEY`, it is
|
||||||
|
**portable**: it restores onto a fresh install that has a different key. Importing merges the
|
||||||
|
settings over the current ones and adds integrations and providers that don't exist yet, matched
|
||||||
|
by type and name; it never overwrites an existing one.
|
||||||
|
|
||||||
|
It does **not** contain servers (and their agent tokens), tasks, secrets, IP addresses, domains,
|
||||||
|
ports, tags, maintenance windows, the users, or either log — nor the other data the app keeps: endpoint
|
||||||
|
checks, heartbeats, alert rules, notes and to-do items, the inventory, dependencies, the Home Assistant
|
||||||
|
watch list, incidents or any history. So it is the right thing for moving
|
||||||
|
your *configuration* to a new instance, or as a second copy of the part that is tedious to
|
||||||
|
re-enter. It is **not** a backup of the system: use the database backup for that.
|
||||||
|
|
||||||
|
Keep the passphrase somewhere safe; it can't be recovered.
|
||||||
|
|
||||||
|
## If there is no backup
|
||||||
|
|
||||||
|
What you can get back, from least to most work:
|
||||||
|
|
||||||
|
- **You have a Settings → Data → Backup export:** a fresh install plus an import restores the settings,
|
||||||
|
integrations and DNS providers. Everything else is re-created.
|
||||||
|
- **You only have the `.env`:** a fresh install with the same file works, and everything in the
|
||||||
|
database starts empty. Sign in (the first account becomes admin), re-add the integrations,
|
||||||
|
register each server again on the Servers page — it shows a ready-to-copy install command with each
|
||||||
|
new token — and run that on the server. IP addresses, secrets, domains, ports and tags
|
||||||
|
are typed in again; DNS records, IPAM syncs and the Tailscale/Proxmox/Docker views come back by
|
||||||
|
syncing.
|
||||||
|
- **You have neither:** the same, plus new integration credentials and a new key.
|
||||||
|
|
||||||
|
## What has been tested, and what hasn't
|
||||||
|
|
||||||
|
**The built-in backups** were tested with the app's own tests (a backup holds what was there and not what came
|
||||||
|
after; two in the same second don't collide; garbage, someone else's database and a newer version's
|
||||||
|
database are refused; the newest N are kept; a staged restore is checked, can be cancelled and puts nothing
|
||||||
|
half-copied in place; a restore that can't complete leaves the database exactly as it was) and by a real
|
||||||
|
restore: a backup was taken, a user added, the restore staged, the actual built app (`node dist/index.js`)
|
||||||
|
restarted on the same data folder — the user added after the backup was gone, the previous database was
|
||||||
|
kept as `…-pre-restore.sqlite`, and the next daily backup was scheduled. **Not** tested: running it inside the
|
||||||
|
real Docker container, and a very large database (the copy is made on this machine's disk and takes about as
|
||||||
|
long as copying the file).
|
||||||
|
|
||||||
|
The snippet-based way below was tested on a scratch database made with the real migrations:
|
||||||
|
|
||||||
|
- The snapshot (`VACUUM INTO`) taken from a second process **while the app was writing**: it
|
||||||
|
completed in tens of milliseconds, passed the integrity check, held all migrations, and an
|
||||||
|
encrypted credential in it decrypted with the same key.
|
||||||
|
- The snippets above, run as written from standard input (`node -`), including a repeat run (which
|
||||||
|
is why the name has the time in it), the check snippet, and the "keep the newest N" line.
|
||||||
|
- An **older version's** database restored into the current version: it upgraded itself on start and
|
||||||
|
kept its data.
|
||||||
|
- A stored credential with the **wrong** key fails with the message quoted above.
|
||||||
|
|
||||||
|
**Not** tested here: running the commands through `docker compose exec` in the real container
|
||||||
|
(there was no Docker in the test environment — the snippets use only Node and the app's own
|
||||||
|
database library, which the container has, and the app's working folder `/app` is where `require`
|
||||||
|
looks), the cron line, the off-machine copy, and a full restore drill on your hardware. Do that
|
||||||
|
drill once; it is the one check that covers all of them.
|
||||||
+1292
-30
File diff suppressed because it is too large.
Load diff
+468
-14
@@ -6,13 +6,42 @@ Homelab Manager (Integrations → Add integration, or DNS → Add
|
|||||||
provider). All credentials are entered in-app and encrypted at rest —
|
provider). All credentials are entered in-app and encrypted at rest —
|
||||||
nothing is read from environment variables.
|
nothing is read from environment variables.
|
||||||
|
|
||||||
Every adapter listed here performs write actions (starting/stopping
|
Every adapter listed here except UniFi, Nginx Proxy Manager, Home Assistant and the UPS performs write actions (starting/stopping
|
||||||
things, editing DNS records, etc.), not just reads — a read-only
|
things, editing DNS records, etc.), not just reads — a read-only
|
||||||
credential will fail as soon as you use one of those actions, even if
|
credential will fail as soon as you use one of those actions, even if
|
||||||
the dashboard views themselves load fine.
|
the dashboard views themselves load fine. (UniFi, Nginx Proxy Manager, Home Assistant and the UPS only ever read.)
|
||||||
|
|
||||||
## Integrations
|
## Integrations
|
||||||
|
|
||||||
|
### Reading one now
|
||||||
|
|
||||||
|
Every integration has a **Read now** button on the Integrations page (operators and administrators; the pages of Media managers, Download clients, Hetzner Robot, Hetzner Cloud, Game servers, AdGuard Home, Technitium DNS, Grafana and Speedtest Tracker have it too, on each card). `POST /api/integrations/{id}/read-now` answers `{ kind, ok, error, at, latencyMs }`:
|
||||||
|
|
||||||
|
- **`kind: "stored"`** — the integration has a reading that is kept (everything but the ones below): it is read the way its schedule reads it, stored, and the alerts are worked out again at once. A read that fails says why and leaves what was stored as it was.
|
||||||
|
- **`kind: "live"`** — Proxmox, Synology, Tailscale, Gitea, Semaphore, Uptime Kuma, Proxmox Backup Server, osTicket, phpIPAM and Nginx Proxy Manager keep nothing: their pages and the health pass read them live every time. Here it asks whether the integration answers (the question *Test* asks, with what is saved) and brings its Working/Failing mark up to date; the alerts, which read it live, are worked out again at the next look.
|
||||||
|
- **Not too often:** the same integration is not read again within **20 seconds** (HTTP 429, `Retry-After`) — some systems limit how often they may be asked. One that is switched off cannot be read (HTTP 400); one that is not there is a 404.
|
||||||
|
- Which kinds have a stored reading is one list in `server/src/services/integrationSyncs.ts`, which also decides what happens when an integration is added, changed (a new address or credentials, or switched on) or deleted: it is read at once, and what was read is forgotten on delete. A new kind with a schedule is added there.
|
||||||
|
|
||||||
|
### History of the figures
|
||||||
|
|
||||||
|
The numbers an integration reports — queries a day and how many were blocked (AdGuard Home, Technitium), items in the queue and wanted-but-missing (Sonarr, Radarr, Prowlarr), download and upload speed (qBittorrent, SABnzbd), photos, videos, library size and disk (Immich), documents and the inbox (Paperless-ngx), users, files and free space (Nextcloud), devices and addresses (NetBox), failed runs (n8n), rules firing (Grafana), targets down and alerts (Prometheus), failed sign-ins (Authentik), the library (Jellyfin) and the estimated monthly cost and server count (Hetzner Cloud) — are kept over time, so a page can show how a figure has moved instead of only what it is now. Each of those pages has a **Show the history** button on every integration's card: a small chart for each figure with where it stands, how it changed over the stretch (*up 2,398*, *down 1.5 points*), its lowest and highest, over 24 hours, 7, 30 or 90 days or a year.
|
||||||
|
|
||||||
|
- **How:** every five minutes the stored reading of each enabled integration (the one its sync already keeps) is looked at, and about every quarter of an hour a reading of each figure is kept — for five weeks; beyond that its hours (four months) and days (about two years) as an average with a lowest and highest, the way the servers' trends are kept. Nothing is asked of the integration for this.
|
||||||
|
- **A gap is a gap:** a reading that is more than 45 minutes old (the integration could not be read) is not recorded, so a stretch when it was unreachable is empty on the chart, not a flat line. A figure the integration could not give — a number that is null, not zero — is left out.
|
||||||
|
- **Not covered:** the integrations that already keep a history of their own (UPS, Home Assistant, speedtest, who is streaming from Jellyfin, the storage of Proxmox and Synology) and those that are only read live when a page is opened (Proxmox, Synology, Tailscale, Gitea, Semaphore, Uptime Kuma, Proxmox Backup Server, osTicket, phpIPAM, Nginx Proxy Manager): there is nothing stored to take figures from. Asked for history of one of those, the endpoint says `kept: false`.
|
||||||
|
- **Starting:** history begins when this was added; nothing is made up for the time before. A figure appears after its first reading.
|
||||||
|
- **API:** `GET /api/integrations/:id/figures?days=7` (any signed-in user; 1 to 365 days) answers `{ kept, days, figures: [{ key, label, unit, latest, readAt, points, first, low, high }] }`. The extractors, one for each kind, are in `server/src/services/integrationFigures.ts`.
|
||||||
|
|
||||||
|
### Checking what an integration can read
|
||||||
|
|
||||||
|
On the Integrations page an administrator has an **Inspect** button on each integration that reads through the shared HTTP helper (Authentik, Immich, Jellyfin, NetBox, Nextcloud, Paperless-ngx, Prometheus, Hetzner Cloud, Hetzner Robot, Speedtest Tracker, Fragnet, Sonarr, Radarr, Prowlarr, Lidarr, Readarr, qBittorrent, SABnzbd, AdGuard Home, Technitium DNS, Grafana, n8n and Home Assistant). It runs the integration's reader once and shows:
|
||||||
|
|
||||||
|
- **what was asked** — each request, its path (with the *names* of the query parameters, not their values), the status, how long it took and how many bytes;
|
||||||
|
- **what came back** — the *shape* of each JSON answer to three levels: the names of the fields and the kind of each (`string`, `number`, `array(12) of { … }`), with a `?` on a field that only some items of a list have. Never a value, never a token, so the report can be copied into a message;
|
||||||
|
- **what the reader made of it**, in the same way, and **where that is empty** — a field that is null, an empty text, an empty list, or null in every item of a list. A reader that expects a field the system names differently ends up here.
|
||||||
|
|
||||||
|
It is for the readers written from documentation (see "Not verified" on each integration): put what came back beside what is empty and a mismatch is plain. Something being empty is often right — no containers, nothing wrong — so it is a list to look at, not a verdict. Each inspection is in the audit log; nothing is stored. Only an administrator can run it, and it times out after 90 seconds.
|
||||||
|
|
||||||
### Tailscale
|
### Tailscale
|
||||||
|
|
||||||
- **Config fields:** Tailnet, API key
|
- **Config fields:** Tailnet, API key
|
||||||
@@ -24,9 +53,10 @@ the dashboard views themselves load fine.
|
|||||||
|
|
||||||
- **Config fields:** Proxmox URL, API token ID, API token secret, allow self-signed certificate
|
- **Config fields:** Proxmox URL, API token ID, API token secret, allow self-signed certificate
|
||||||
- **Auth:** `PVEAPIToken=<tokenId>=<tokenSecret>` header
|
- **Auth:** `PVEAPIToken=<tokenId>=<tokenSecret>` header
|
||||||
- **Actions used:** list nodes/VMs/LXCs, read guest config and live status, read node host stats and storage usage, start/stop/reboot/shutdown a guest, read backup job schedules and recent `vzdump` task history
|
- **Actions used:** list nodes/VMs/LXCs, read guest config and live status, read node host stats and storage usage, start/stop/reboot/shutdown a guest, **list, take, roll back to and delete a guest's snapshots**, read backup job schedules and recent `vzdump` task history
|
||||||
- **Required access:** A dedicated API token (Datacenter → Permissions → API Tokens) with a role granting **VM.Audit + VM.PowerMgmt** (e.g. `PVEVMAdmin`) on the VMs/LXCs to manage, **plus Sys.Audit and Datastore.Audit** on the node(s) for the host CPU/RAM/storage cards to populate. A token scoped to VM management only will still work for the guest list and power actions — it'll just show an error on the per-node hardware card instead of failing outright. Backup job schedules and run history need that same **Sys.Audit** — no separate permission to grant if the node-stats card already works.
|
- **Required access:** A dedicated API token (Datacenter → Permissions → API Tokens) with a role granting **VM.Audit + VM.PowerMgmt** (e.g. `PVEVMAdmin`) on the VMs/LXCs to manage, **plus Sys.Audit and Datastore.Audit** on the node(s) for the host CPU/RAM/storage cards to populate. A token scoped to VM management only will still work for the guest list and power actions — it'll just show an error on the per-node hardware card instead of failing outright. Backup job schedules and run history need that same **Sys.Audit** — no separate permission to grant if the node-stats card already works.
|
||||||
- **Notes:** self-signed certificates are supported via the "Allow self-signed certificate" checkbox — common for an internal PVE host.
|
- **Snapshots** (Infrastructure → Proxmox → click a guest → *Snapshots*): everyone signed in sees a guest's snapshots (oldest first, a branch indented, *with memory* marked); an **operator** can take one (a name — or one made from the time, `hlm-YYYYMMDD-HHMMSS` — a description, and for a running VM whether to keep its memory so a rollback brings it back running); **rolling back and deleting are an administrator's**, each behind a confirmation, because a rollback throws away everything the guest has done since and Proxmox removes the snapshots taken after the one chosen. Proxmox does each as a task: the page waits up to 20 s and says how it went, or that it is still going and looks again. Every one is in the audit log and the change log (*A snapshot was taken of guest 100 on pve1*). The token needs **VM.Snapshot** (take, delete) and **VM.Snapshot.Rollback** (roll back) as well as VM.Audit — the `PVEVMAdmin` role has them; a token without them lists the snapshots and gets Proxmox's own refusal when it tries the rest. Checked against Proxmox's published API description and a pretend Proxmox, not a live host.
|
||||||
|
- **Notes:** self-signed certificates are supported via the "Allow self-signed certificate" checkbox — common for an internal PVE host. An address that says `http://` (a reverse proxy in front of Proxmox) is spoken to over plain HTTP.
|
||||||
|
|
||||||
### Synology DSM
|
### Synology DSM
|
||||||
|
|
||||||
@@ -40,7 +70,7 @@ the dashboard views themselves load fine.
|
|||||||
|
|
||||||
- **Config fields:** Semaphore URL, API token
|
- **Config fields:** Semaphore URL, API token
|
||||||
- **Auth:** Bearer token
|
- **Auth:** Bearer token
|
||||||
- **Actions used:** list projects, list templates, run a template (creates a task)
|
- **Actions used:** list projects, list templates, run a template (creates a task — optionally with a `limit` of hosts and a note saying who started it from Homelab Manager), read a task's status
|
||||||
- **Required access:** A user API token with access to every project you want visible, and **permission to run templates** in those projects — a viewer/guest-level project role can list templates but will fail on "Run", so the account needs at least the operator-equivalent role Semaphore's own project permissions define.
|
- **Required access:** A user API token with access to every project you want visible, and **permission to run templates** in those projects — a viewer/guest-level project role can list templates but will fail on "Run", so the account needs at least the operator-equivalent role Semaphore's own project permissions define.
|
||||||
|
|
||||||
### Gitea
|
### Gitea
|
||||||
@@ -56,10 +86,15 @@ the dashboard views themselves load fine.
|
|||||||
- **Auth:** Bearer token against Dockhand's own API
|
- **Auth:** Bearer token against Dockhand's own API
|
||||||
- **Actions used:** list environments/containers, check for image updates, start/stop/restart a container
|
- **Actions used:** list environments/containers, check for image updates, start/stop/restart a container
|
||||||
- **Required access:** A Dockhand API token belonging to a user with access to every environment (Docker host) you want visible, with permission to **start/stop/restart containers and trigger update checks** in each — not just view them.
|
- **Required access:** A Dockhand API token belonging to a user with access to every environment (Docker host) you want visible, with permission to **start/stop/restart containers and trigger update checks** in each — not just view them.
|
||||||
|
- **Suggested checks:** each container's image and published ports are also used to suggest an endpoint check on the app's own health address ([MONITORING.md](docs/MONITORING.md#endpoint-checks-and-heartbeats)). Ports are read from whatever shape Dockhand's container list carries them in (Docker's own, `"0.0.0.0:8080->80/tcp"` strings or inspect's map) — **not checked against a live Dockhand**; if none is read, the Docker host's listening ports from its agent stand in.
|
||||||
|
- **Container health:** on the 15-minute health pass the app also reads every container's state and status text and raises alerts for what is broken — see [Docker containers that are broken](docs/MONITORING.md#docker-containers-that-are-broken). A Docker host that Dockhand
|
||||||
|
cannot read (its Hawser agent is down) is an alert too, and its containers' alerts are held, not cleared. The Docker page shows each container's problem and has a **Leave alone** button.
|
||||||
|
- **Vulnerability scans:** every fifteen minutes the app also reads what Dockhand's scans have stored — `GET /api/vulnerabilities/count?env=` for each environment, and `GET /api/vulnerabilities?env=&severity=critical&limit=100` only where that count is above zero (needs the *images: view* permission). Nothing is scanned from here. **Critical** findings are a warning per Docker host — and so are **high** ones once a host has as many as a number you set (*Also warn when a Docker host has this many high vulnerabilities*, off by default) — (a setting turns the critical one off — see [MONITORING.md](docs/MONITORING.md#docker-containers-that-are-broken)); the Docker page lists them. **Written from Dockhand's source and OpenAPI file, not tried against a live Dockhand** — its published docs show the lists at the top level of the answer, its code puts the counts under `summary`; the reader follows the code and reads nothing it does not find as *unknown*.
|
||||||
|
- **Image age:** the same pass reads each Docker host's images and containers (`GET /api/images?env=`, `GET /api/containers?env=&all=true`) and keeps, for every image a *running* container uses, when it was built (Docker's own `Created`). Matched by the name the container gives (Docker Hub's `docker.io/library/` and a missing `:latest` are not a difference) or by id. With *Warn about running Docker images that were built more than … days ago* set (Settings → Notifications; **off by default**), a host with such images is a warning — whatever the scans of them said. The Docker vulnerabilities view shows each host's oldest running images either way. **Written from Dockhand's documented list (`Id`, `RepoTags`, `Created`), not tried against a live Dockhand**; an image whose date cannot be found is not judged.
|
||||||
|
|
||||||
### Uptime Kuma
|
### Uptime Kuma
|
||||||
|
|
||||||
- **Config fields:** Uptime Kuma URL, username (leave blank — see below), API key
|
- **Config fields:** Uptime Kuma URL, username (leave blank — see below), API key, and the optional *Login username* and *Login password* (for mirroring maintenance windows) with a checkbox to keep maintenance windows in step with Uptime Kuma's schedule
|
||||||
- **Auth:** HTTP Basic, with the API key as the password and the username left empty. Uptime Kuma has no
|
- **Auth:** HTTP Basic, with the API key as the password and the username left empty. Uptime Kuma has no
|
||||||
conventional REST API — the dashboard talks to it over Socket.IO — so this integration reads its
|
conventional REST API — the dashboard talks to it over Socket.IO — so this integration reads its
|
||||||
Prometheus-metrics endpoint (`GET /metrics`) instead and parses that. On installs from before the API-key
|
Prometheus-metrics endpoint (`GET /metrics`) instead and parses that. On installs from before the API-key
|
||||||
@@ -74,14 +109,75 @@ the dashboard views themselves load fine.
|
|||||||
with no single network target (groups, push monitors, keyword checks with a complex URL) are shown
|
with no single network target (groups, push monitors, keyword checks with a complex URL) are shown
|
||||||
unmatched rather than guessed at. Uptime Kuma's tags aren't read, since the metrics endpoint doesn't
|
unmatched rather than guessed at. Uptime Kuma's tags aren't read, since the metrics endpoint doesn't
|
||||||
reliably distinguish a tag from any other label.
|
reliably distinguish a tag from any other label.
|
||||||
- **Maintenance import:** the Maintenance page can read which monitors are currently in maintenance in Uptime
|
- **Maintenance import ("Import now"):** the Maintenance page can read which monitors are currently in
|
||||||
Kuma and start (or extend) a maintenance window here for the matching server, for a duration you pick — the
|
maintenance in Uptime Kuma and start (or extend) a maintenance window here for the matching server, for a
|
||||||
metrics endpoint only exposes current status, not a monitor's scheduled start/end time, so this reflects
|
duration you pick — the metrics endpoint only exposes current status, not a window's schedule.
|
||||||
what's in maintenance right now rather than mirroring Uptime Kuma's own schedule.
|
- **Maintenance schedule ("Sync schedule"):** Uptime Kuma shares its maintenance windows only over the live
|
||||||
|
Socket.IO connection its own dashboard uses, and that connection takes a **login username and password** —
|
||||||
|
an API key is not accepted there. Fill in the optional *Login username* and *Login password* on the
|
||||||
|
integration to use it. Be aware that a Uptime Kuma login can change anything in Uptime Kuma (it has no
|
||||||
|
read-only accounts), so create a **dedicated account** for this. An account with two-factor authentication
|
||||||
|
cannot be used. The password is stored encrypted like every other credential here, and is used for nothing
|
||||||
|
but this. The login is sent as Uptime Kuma's own `login` event over `ws://` or `wss://` to the same address
|
||||||
|
as the integration, so use an `https://` address if it crosses anything but a trusted network.
|
||||||
|
- For each window Uptime Kuma reports (its running time slot and the next one), the app finds the
|
||||||
|
monitors it covers and matches each monitor's address to something here: a **server** (the monitor's host
|
||||||
|
is one of its IP addresses, its hostname or its short name — the same matching the Uptime Kuma page
|
||||||
|
uses), an **integration** or a **DNS provider** (the monitor's host is the host of the address it was
|
||||||
|
added with, as the same name or the same IP). It then makes a maintenance window on each match. A window
|
||||||
|
that hasn't started shows as *Scheduled* and silences nothing yet.
|
||||||
|
- Matching is exact. Nothing is worked out through DNS, so a website that only resolves to the same address
|
||||||
|
as a reverse proxy does not put the proxy into maintenance. A monitor that matches nothing is listed in the
|
||||||
|
result as `name (address)` so you can see why. The usual cause is that Uptime Kuma watches a thing by a different
|
||||||
|
address than the one it was added with here (a Proxmox host added by its LAN address but monitored over Tailscale).
|
||||||
|
Add that address under **Address aliases** on the Maintenance page, saying which server, integration or DNS
|
||||||
|
provider it belongs to; matching stays exact, and an alias applies to every sync. Monitors that are not in
|
||||||
|
Uptime Kuma's metrics (paused ones) and hosted services outside your network can't be matched and are left alone.
|
||||||
|
- A window on an integration silences that integration's alerts, including "integration down" for that
|
||||||
|
type of service — see the Maintenance page for the full list.
|
||||||
|
- Tick *Keep maintenance windows in step with Uptime Kuma's schedule* to run this automatically every
|
||||||
|
15 minutes. Without it, the *Sync schedule* button does the same once.
|
||||||
|
- Windows are capped at 31 days, and only slots starting within two weeks are mirrored. A window Uptime
|
||||||
|
Kuma holds in maintenance "until switched off" (the manual kind) is mirrored 45 minutes at a time and
|
||||||
|
renewed on each pass, so it lapses by itself if Uptime Kuma becomes unreachable.
|
||||||
|
- A window you end or cancel here is not brought back by a later sync. One that is removed or switched
|
||||||
|
off in Uptime Kuma is removed here (a running one is ended at once).
|
||||||
|
- Monitors that match no server (groups, push monitors, hosts the app doesn't know) are listed as
|
||||||
|
unmatched in the result and silence nothing. A window on a *group* monitor does not extend to its
|
||||||
|
children here.
|
||||||
|
- Verified against the Uptime Kuma 1.23 and 2.0 source and against a stand-in server speaking the same
|
||||||
|
protocol; not yet run against a live Uptime Kuma.
|
||||||
|
|
||||||
|
### Duplicati
|
||||||
|
|
||||||
|
Duplicati is different from the other integrations: **this app never calls Duplicati**, and no password for it is kept. Duplicati can tell a URL how each operation went (its own `send-http-url` option), so each machine running it is a *source* with a secret address, and its reports come in.
|
||||||
|
That works on every Duplicati version, needs no login handshake and leaves nothing to leak from this side.
|
||||||
|
|
||||||
|
- **Set up** (Backups → Duplicati → *Add a machine*, administrators): give the machine a name and, if you like, the server it runs on. You are shown, **once**, an address of the form `https://<this app>/api/duplicati-report/hlm_d_…` and the three options to add in Duplicati under
|
||||||
|
**Settings → Default options → Advanced options** (so every job on that machine reports):
|
||||||
|
|
||||||
|
```
|
||||||
|
--send-http-url=https://<this app>/api/duplicati-report/hlm_d_…
|
||||||
|
--send-http-result-output-format=Json
|
||||||
|
--send-http-level=All
|
||||||
|
```
|
||||||
|
|
||||||
|
`send-http-level=All` matters: by default Duplicati only reports problems, and a backup that quietly stops reporting could not be told from one that works. The plain-text format is also read, but JSON carries the name of the job and the figures reliably.
|
||||||
|
**How Duplicati posts it does not matter:** by default it sends a **form-encoded** body (`message=<the text, URL-encoded>`, the message beginning *"Duplicati Backup report for <job>"*), which is unwrapped — the field may have another name (`send-http-message-parameter-name`) and have others beside it; the text or JSON posted as the body as it is (`send-http-json-urls`, a message set by hand) is read too. The job's name and the operation are taken from that first line when the result does not say them. In the text form the warnings and errors are read from their `Warnings: [ … ]` and `Errors: [ … ]` lists.
|
||||||
|
For one job only, add the same options to that job's advanced options instead.
|
||||||
|
- **What is read** from each report: the operation (only *Backup* counts — a compact, repair or test is set aside), the result (`Success`, `Warning`, `Error`, `Fatal`), the job's name, when it began and ended, the duration, the size of what was examined, the files added, changed and deleted, the number of
|
||||||
|
warnings and errors, the first message and **what the warnings said** (JSON format only; up to twenty, each cut at 500 characters, kept with the run). A job is learned from its first report, by name; the same name on another machine is another job.
|
||||||
|
- **The address is the access.** It is a long random token (stored only as a SHA-256 hash, shown once); all it allows is to say how a backup went. It answers a wrong token with "not found", limits requests per address, and takes POST or PUT. Removing the machine ends it.
|
||||||
|
- **Not verified against every Duplicati version** — the report fields are read defensively from Duplicati's published output (the results at the top or under `Data`, the job's name among `Extra`). A report the app cannot read is answered "OK (not a report this app can read)" and the page does not learn from it.
|
||||||
|
- **Why a machine has no jobs.** "Last heard" only says something arrived on the address. Each machine's card on the page also says **what the last report was** — a backup (with the job and how it ended), another operation that was set aside (compact, repair, test … only a *Backup* makes a job), or one that **could not be read** with the reason (no `ParsedResult`, not valid JSON …) — and how many of each have come since the machine was added. Administrators can open **What was sent** to see the first 600 characters of an unreadable report, to see its form; others see the reason but not the text. A machine with no job yet and only set-aside reports is told to check `--send-http-level=All`.
|
||||||
|
- **Warnings in a row.** One run with warnings is common; the page and an alert say when the **last three runs in a row** all finished with warnings (*Warnings 4 runs in a row*; alert key `duplicati-streak:<job>`). It is on for every job and a button turns it off for one that always has a few (*Stop streak alerts*); it is not raised for a job that already alerts on every warning run, nor for one that has failed or is overdue (their own alerts say that).
|
||||||
|
- **How much room the backups take.** Duplicati says how big the backups are where they are kept (`BackendStatistics.KnownFileSize`) and, for some kinds of storage, how much room there is (`TotalQuotaSpace`). Both are read (the plain-text form too) and the size is kept with every run. The *Stored* column on the page shows it with the growth over the last 180 days — *+3.4 GB a month*, *not growing*, *size goes up and down* — only once there are five readings over a week, and nothing is claimed from less. Where the room is known it also says *full in about N days*, and when that is within the **disk forecast window** (Settings → Alerts, 14 days by default; 0 switches it off) and the growth is steady (a straight line fits at least 0.8) it is an alert (`duplicati-growth:<job>`; critical at three days or less). A storage that does not say how much room it has is never forecast to fill: the growth is shown, nothing more. Compacting makes the size drop now and then, which lowers the fit and keeps a jagged size from raising a false alarm.
|
||||||
|
- What it does with them — alerts, `/metrics`, the history — is in [MONITORING.md](docs/MONITORING.md#duplicati-backups).
|
||||||
|
|
||||||
### Proxmox Backup Server
|
### Proxmox Backup Server
|
||||||
|
|
||||||
- **Config fields:** Proxmox Backup Server URL, API token ID, API token secret, allow self-signed certificate
|
- **Config fields:** Proxmox Backup Server URL, API token ID, API token secret, allow self-signed certificate
|
||||||
|
- **Where it shows:** **Backups → Proxmox backups → *Proxmox Backup Server*** tab, next to the *Backup coverage* tab (`/backups?view=pbs`; the old `/pbs` address goes there).
|
||||||
- **Auth:** `PBSAPIToken=<tokenId>:<tokenSecret>` header — note the **colon** between the token id and secret; Proxmox
|
- **Auth:** `PBSAPIToken=<tokenId>:<tokenSecret>` header — note the **colon** between the token id and secret; Proxmox
|
||||||
VE's own token header uses `=` there instead, so a PVE token/secret pair copied verbatim into this integration's
|
VE's own token header uses `=` there instead, so a PVE token/secret pair copied verbatim into this integration's
|
||||||
fields will still format correctly (the adapter supplies the colon itself) as long as the id/secret values
|
fields will still format correctly (the adapter supplies the colon itself) as long as the id/secret values
|
||||||
@@ -116,8 +212,8 @@ the dashboard views themselves load fine.
|
|||||||
reading osTicket's own MySQL/MariaDB database directly, the same way osTicket's own admin panel does internally.
|
reading osTicket's own MySQL/MariaDB database directly, the same way osTicket's own admin panel does internally.
|
||||||
This makes it the only integration in this app that isn't a REST API.
|
This makes it the only integration in this app that isn't a REST API.
|
||||||
- **Actions used:** read-only — a single `SELECT` joining the ticket, status, priority, department, staff, team,
|
- **Actions used:** read-only — a single `SELECT` joining the ticket, status, priority, department, staff, team,
|
||||||
and requester tables for every ticket whose status is in the "open" state. Nothing here can create, update, or
|
and requester tables for every ticket whose status is in the "open" state. Nothing in this part can create,
|
||||||
close a ticket.
|
update, or close a ticket — opening tickets is the separate, optional API described below.
|
||||||
- **Required access:** a MySQL/MariaDB user with **read-only (`SELECT`) access to the osTicket database only** —
|
- **Required access:** a MySQL/MariaDB user with **read-only (`SELECT`) access to the osTicket database only** —
|
||||||
never reuse osTicket's own application database user, which has full read/write access. Create one with
|
never reuse osTicket's own application database user, which has full read/write access. Create one with
|
||||||
something like:
|
something like:
|
||||||
@@ -148,6 +244,56 @@ the dashboard views themselves load fine.
|
|||||||
a field looks wrong, tell us what happened and we'll adjust — this one has had less real-world exposure than
|
a field looks wrong, tell us what happened and we'll adjust — this one has had less real-world exposure than
|
||||||
every other integration listed here.
|
every other integration listed here.
|
||||||
|
|
||||||
|
**Optional: opening tickets for alerts** (Alert settings → Alert tickets)
|
||||||
|
|
||||||
|
- **Config fields:** Ticket API address (optional), Ticket API key (optional, stored encrypted like the database
|
||||||
|
password), Allow self-signed certificate (ticket API). Leave them empty and the integration works as before.
|
||||||
|
- **Auth:** osTicket's own HTTP API: `POST <address>/api/tickets.json` with the key in an `X-API-Key` header.
|
||||||
|
- **Make the key in osTicket:** Admin Panel → Manage → API Keys → Add. Tie it to the IP address osTicket sees this
|
||||||
|
app's requests come from (behind Docker or a reverse proxy that is the host's or proxy's address, not the
|
||||||
|
container's — check osTicket's log if it refuses) and tick **Can create tickets**. osTicket refuses a key from any
|
||||||
|
other address, which is the most common reason a test ticket fails.
|
||||||
|
- **What is sent:** one ticket per problem — name and email you set under Alert settings → Alert tickets, a subject, the
|
||||||
|
problem's text and a link back to this app, optionally a help topic and priority id — with `autorespond` off (the
|
||||||
|
"user" is this app, so an acknowledgement would only bounce) and osTicket's own staff alert on or off as you choose.
|
||||||
|
- **What the API can't do:** close, reopen, or comment on a ticket — osTicket's API has no such endpoint. Closing one is
|
||||||
|
a separate, optional thing, described next, and it does not use the API.
|
||||||
|
- **Checked against a real osTicket for a test ticket** (the API key with its IP restriction, creating the ticket, and the
|
||||||
|
message arriving exactly as written, line breaks and non-ASCII characters included). It was built from osTicket's
|
||||||
|
published API documentation and tested against a stand-in server too, and the real one showed two mistakes the stand-in
|
||||||
|
hid: the text first arrived with every space as `%20`, then as base64. osTicket takes the message's type from a
|
||||||
|
`data:` prefix but uses everything after the first comma exactly as it is, decoding nothing, so the text is sent raw.
|
||||||
|
A ticket opened for a real alert has been seen working too; a summary ticket (many problems at once) has not been seen on a real osTicket yet. If osTicket rejects a ticket, "Send a test
|
||||||
|
ticket" shows the reason it gave.
|
||||||
|
|
||||||
|
**Optional: closing the ticket when you close the alert** (the *Also close the osTicket ticket* tick box in the Close drawer on the Alerts page)
|
||||||
|
|
||||||
|
osTicket's API cannot close a ticket, so this is the one place in the app that **writes to another application's database**. Read the next paragraphs before switching it on.
|
||||||
|
|
||||||
|
- **Config fields:** Database username for closing tickets (optional) and its password (optional, stored encrypted). They are **separate from the read-only login above**, on purpose: closing is granted to one account and never widens what reading may do.
|
||||||
|
Leave them empty and nothing here ever runs.
|
||||||
|
- **Make the login in MySQL/MariaDB**, with only what closing needs (replace `osticket` and the prefix if yours differ):
|
||||||
|
```sql
|
||||||
|
CREATE USER 'homelab_close'@'%' IDENTIFIED BY 'a-strong-password';
|
||||||
|
GRANT SELECT ON osticket.ost_ticket_status TO 'homelab_close'@'%';
|
||||||
|
GRANT SELECT ON osticket.ost_thread TO 'homelab_close'@'%';
|
||||||
|
GRANT SELECT ON osticket.ost_event TO 'homelab_close'@'%';
|
||||||
|
GRANT SELECT, UPDATE (status_id, closed, lastupdate, updated, isanswered, isoverdue) ON osticket.ost_ticket TO 'homelab_close'@'%';
|
||||||
|
GRANT INSERT ON osticket.ost_thread_entry TO 'homelab_close'@'%';
|
||||||
|
GRANT INSERT ON osticket.ost_thread_event TO 'homelab_close'@'%';
|
||||||
|
FLUSH PRIVILEGES;
|
||||||
|
```
|
||||||
|
- **What a close writes** — what osTicket's own *Close* does, in **one transaction** (all or none): the ticket's status becomes a closed one (the one called *Closed*, else the first closed status), its *closed* and *last update* times are set and it is no longer marked answered or overdue; a **note** in the ticket's thread says
|
||||||
|
*Closed from Homelab Manager by <name>* with the reason you gave; and a **"closed" event** is recorded so the ticket's history reads like any other. Nothing else is touched, and nothing is ever deleted.
|
||||||
|
- **What it checks first:** that the tables and columns it writes to are there (written for **osTicket 1.14–1.18**); a different osTicket is refused with the list of what is missing, before anything is written. It closes only a ticket that is **open**, and only if its status is still what it was when it was read — a ticket somebody
|
||||||
|
else closes at the same moment is left as they left it. A ticket that was opened as part of a **summary ticket** (many problems at once) is not closed from here, because it covers other alerts.
|
||||||
|
- **Test closing** (Alert settings → Alert tickets, next to *Send a test ticket*; administrators): signs in with the login, checks the tables and columns, and then **tries every write a close makes inside a transaction that is rolled back** — so a missing grant is found there and not on the first real ticket. Nothing stays (a counter or two
|
||||||
|
inside the database moves on).
|
||||||
|
- **Closing several alerts at once:** pick them on the Alerts page and choose *Close*; when some of them have a ticket that can be closed from here, the drawer offers *Also close N osTicket tickets* (with their numbers), and it is closed one ticket after another, each written on its alert's incident and in the audit log as above. One that cannot be closed does not stop the others: the alerts are closed all the same and the message says how many tickets were closed, were already closed, or could not be (with the first reason). Summary tickets and an osTicket with no closing login are counted and left open. **Undo opens the alerts again but cannot reopen the tickets** — the message says so.
|
||||||
|
- **What the alert shows:** the ticket badge says *(closed)* once it has been closed from here, and the Close drawer says so instead of offering it again. If the ticket cannot be closed, the alert is **still closed** and you are told why; the reason is also written on the incident and in the audit log (`alert_ticket/close_ticket`, `close_ticket_failed`).
|
||||||
|
- **Not verified against a live osTicket — and this is the least verified part of the app.** There was no MySQL/MariaDB server to run it on; it is built from osTicket's published schema and tried only against a pretend database that answers the exact statements it sends. For opening tickets, a real osTicket showed two mistakes that a
|
||||||
|
stand-in server had hidden, and the same can happen here (a column that is called something else in your version, a status or event that is set up differently). That is what *Test closing* is for: **run it first**, and try the first real close on a ticket that does not matter. If something is wrong the error says what; tell me and I'll adjust.
|
||||||
|
|
||||||
### phpIPAM
|
### phpIPAM
|
||||||
|
|
||||||
- **Config fields:** phpIPAM URL, API app ID, App token
|
- **Config fields:** phpIPAM URL, API app ID, App token
|
||||||
@@ -168,6 +314,307 @@ the dashboard views themselves load fine.
|
|||||||
run it against a real phpIPAM yet. If a sync comes back empty or with the wrong fields, tell us what your
|
run it against a real phpIPAM yet. If a sync comes back empty or with the wrong fields, tell us what your
|
||||||
instance actually returned and we'll adjust.
|
instance actually returned and we'll adjust.
|
||||||
|
|
||||||
|
### UniFi (Network application)
|
||||||
|
|
||||||
|
- **Config fields:** controller URL; **either** an API key, **or** a username and password; optionally one site; two checkboxes — *self-hosted Network application* and *allow self-signed certificate*.
|
||||||
|
- **Two ways in:**
|
||||||
|
- **API key** (UniFi Network 9 or later). In the controller, **Settings → Control Plane → Integrations → Create API key**. Used with the official Integration API: on a UniFi OS console (UDM, Cloud Gateway,
|
||||||
|
Cloud Key Gen2+) at `https://<console>/proxy/network/integration/v1/…`, on a self-hosted Network application at `https://<host>:8443/integration/v1/…`. The form is found by asking, so the URL is
|
||||||
|
just the console's address; tick *self-hosted* only to skip the first guess.
|
||||||
|
- **Username and password** (any version, including ones from before API keys). Create a **local** user in the controller with **read-only** access to the site (a Ubiquiti cloud/SSO account, or one with
|
||||||
|
two-factor, cannot be used). The classic endpoints are used (`/proxy/network/api/s/<site>/…`, or `/api/s/<site>/…` self-hosted), and the session is logged out after each read.
|
||||||
|
- **Required access:** read only — and, unlike most integrations here, **nothing is ever written** to the controller.
|
||||||
|
- **What it does:** every five minutes (and on **Read now**) the controller is read and kept: each device's name, model, kind (gateway, switch, access point), state, IP address, MAC address, uptime, firmware (and
|
||||||
|
whether an update is waiting), and the device it is connected through; and the networks (VLAN, address range) and Wi-Fi names (security, which network) it has set up. Wi-Fi passphrases are never read into the app.
|
||||||
|
It all appears on **Network → Network devices**.
|
||||||
|
- A device is a **node in the dependency map**, like a server: the *connected through* link is made into a dependency by itself (an access point needs its switch, the switch needs the gateway), and you can write
|
||||||
|
others by hand (a server needs the switch it is plugged into).
|
||||||
|
- A device the controller has not seen online for longer than **Settings → Notifications → Alert when a network device is offline for** (10 minutes) is a critical **offline** alert about it. What needs it, and what
|
||||||
|
it is connected through if that is down too, is folded under it, like a guest under its host. It can be silenced with a maintenance window, and has runbooks like any target.
|
||||||
|
- **Servers plugged in:** the controller's list of what is connected is matched to servers by IP address (exact, and only when one server has that address). A server is then *connected through* the switch or access point
|
||||||
|
it is on — a dependency made by itself, shown on the server's page and on the Connections map. A key or account that may not read clients costs only this; devices, networks and Wi-Fi names are unaffected.
|
||||||
|
- If the controller cannot be read for twenty minutes its devices are neither raised nor cleared, and the Alerts page says it could not check; the integration itself is raised as failing like any other.
|
||||||
|
- **Not verified against a live controller** — built from Ubiquiti's published Integration API and the long-documented classic API (and tried against a pretend controller that answers the way those documents say),
|
||||||
|
but nobody has run it against real hardware yet. Every field is read defensively, so one a version renames is left blank rather than breaking the read. If a read fails or comes back empty, the **Diagnostic Log** has
|
||||||
|
what the controller answered; tell us and we'll adjust.
|
||||||
|
|
||||||
|
### Nginx Proxy Manager
|
||||||
|
|
||||||
|
- **Config fields:** NPM's admin URL (the address of its web page, usually port 81), an email and a password, optionally a second **email and password for managing** proxy hosts (below), and a checkbox to allow a self-signed certificate.
|
||||||
|
- **Required access:** NPM has no API keys: its API is the one its own web page uses, so the app signs in with an email and password (each read signs in again; nothing is kept). Make a **dedicated user** under
|
||||||
|
*Users* and, under its *Permissions*, give it **View only** on *Proxy Hosts* and *SSL Certificates* — that is all that is read, and this user **never writes**: reading NPM sends nothing but `GET` requests.
|
||||||
|
- **What it does:** when **Network → Proxy hosts** is opened (and for the alerts and the suggested checks, at most once a minute) NPM is asked for its proxy hosts and certificates. Nothing is stored in the database:
|
||||||
|
what is shown is what NPM said a minute ago.
|
||||||
|
- Each host shows its names, where it forwards to — and the server that is, when the address (or name) is one of a server's own, exactly — its certificate and whether it is working.
|
||||||
|
- A certificate that an *enabled* host uses and that ends within 14 days (critical within 3) or has ended is an **expiry** alert; a host NPM says could not be set up (its nginx did not accept the configuration)
|
||||||
|
is a **monitoring** alert. A host that is switched off is left alone. Both link to the Proxy hosts page and can be silenced with maintenance on the integration.
|
||||||
|
- The hosts it serves are offered as **Endpoints → Suggested** checks (a web check on the name people use; https when there is a certificate), linked to the server it forwards to.
|
||||||
|
- If NPM cannot be read, the Alerts page says it could not check, and the integration is raised as failing like any other.
|
||||||
|
- **Managing proxy hosts (optional, administrators only):** with a second user in *Email of a user that may manage* and *Password of that user* — a user with **Manage** on *Proxy Hosts*, **without two-factor sign-in** (NPM's API can't complete that from here); give it only if you want this — **Network → Proxy hosts** gets **New proxy host** and, on each row, **Edit**, **Switch off / on** and **Delete**:
|
||||||
|
- **Make / change** (`POST /api/nginx/proxy-hosts`, `PUT …/{id}`): the **names**, where it **forwards to** (http or https, a host or address, a port), an **existing certificate**, and what is switched on — Force HTTPS, HTTP/2, HSTS (and for subdomains), block common exploits, cache assets, websockets. Names are written lower case and must be names NPM takes; a name another host has is NPM's own refusal, shown as it was said. Getting a *new* certificate is done in NPM.
|
||||||
|
- **Only what differs is sent** when a host is changed, and a host's **access list, locations and custom nginx configuration are never sent**, so they stay as they are. NPM quietly turns Force HTTPS and HTTP/2 off for a host with no certificate (and HSTS when HTTPS is not forced), so those combinations are refused here with the reason, and the form does not let them be ticked.
|
||||||
|
- **Switch off / on** (`POST …/{id}/disable` and `…/enable`): off, NPM stops serving the names; nothing is lost. **Delete** (`DELETE …/{id}`): NPM stops serving them and keeps the record out of sight; **the host's first name has to be typed**, and the server checks it against the host as NPM has it now, so a page that is minutes old or a wrong id cannot delete another host.
|
||||||
|
- Every call **signs in again** as the managing user (nothing is kept); the user that reads is never used to write. What this app has read of NPM is read again after a change. *Stop managing* is under Integrations → edit (removing the two fields, or `DELETE /api/npm/{id}/manage`, which keeps the reading user). Written in the audit log: who made, changed, switched or deleted which host (its names and what it forwards to, and which settings changed — never a password).
|
||||||
|
- **Not verified against a live NPM** — built from the API its own web page uses (`/api/tokens`, `/api/nginx/proxy-hosts?expand=certificate`, `/api/nginx/certificates`, and for managing `/api/nginx/proxy-hosts` with `POST`, `PUT`, `DELETE` and `…/enable`, `…/disable`) and tried against a pretend one. Every
|
||||||
|
field is read defensively. If a read fails or comes back empty, the **Diagnostic Log** has what NPM answered.
|
||||||
|
|
||||||
|
### Home Assistant
|
||||||
|
|
||||||
|
- **Config fields:** Home Assistant's address (with the port, usually 8123), a long-lived access token, and a checkbox to allow a self-signed certificate.
|
||||||
|
- **Required access:** read only. Make the token on a user's profile page (*Security → Long-lived access tokens*); the user needs no more rights than a viewer's. Only `GET /api/` and `GET /api/states` are used —
|
||||||
|
**nothing is ever sent** to Home Assistant.
|
||||||
|
- **What it does:** you choose which entities to watch on **Apps → Home Assistant** (search by name, then *Watch*; at most 60 per Home Assistant). Every five minutes (and on **Read now**) Home Assistant's
|
||||||
|
entities are read in one call and the watched ones are kept: their state, and for a number a reading for the chart (one at most every two minutes, kept 35 days).
|
||||||
|
- An entity that has been *unavailable* or *unknown* — or has disappeared from Home Assistant — for 15 minutes is an alert. A number past a limit you set (*alert when under / over*) is an alert. Both are in the
|
||||||
|
*Monitoring* category, link to the page and can be silenced with maintenance on the integration.
|
||||||
|
- If Home Assistant cannot be read for twenty minutes the watched entities are neither raised nor cleared, the Alerts page says it could not check, and the integration is raised as failing like any other.
|
||||||
|
- A service reached at a server's address (the integration's URL) is offered as a suggested dependency on that server.
|
||||||
|
- **Checked against a live Home Assistant** (Inspect): `/api/states` answered with 1,392 entities (about 600 KB) and every field the reader uses had a value. It was not checked with an entity that is `unavailable` or with a wrong token; those follow the published API and a pretend server. Every field is read defensively. If a read fails, the **Diagnostic Log** has what it answered.
|
||||||
|
|
||||||
|
### UPS (NUT or SNMP)
|
||||||
|
|
||||||
|
- **Config fields:** the **host** (the machine running NUT's `upsd`, or the UPS's own network card), a checkbox to read it **over SNMP instead of NUT**, and optionally the **port** (3493 for NUT, 161 for SNMP), the UPS's **name** on the NUT
|
||||||
|
server (the first one it lists when blank) and the SNMP **read community** (`public` when blank; stored encrypted).
|
||||||
|
- **Required access:** read only. **NUT:** `upsd` must listen on an address this server can reach (`LISTEN` in `upsd.conf`; the default is only the machine itself) and, if `upsd.users`/`ACL` limit who may ask, allow
|
||||||
|
this host. Only `LIST UPS` and `LIST VAR <ups>` are sent — no command, no login. **SNMP:** version 2c, a `GET` of the standard UPS-MIB (RFC 1628, `1.3.6.1.2.1.33`), which most UPSs and their network cards answer;
|
||||||
|
APC-only or other private objects are not used. **Nothing is ever set.** The host is checked like an endpoint check's: not this machine itself, not the cloud metadata address; your own network is fine.
|
||||||
|
- **What it does:** every **minute** the UPS is read and stored — its state, and a point for the charts (every five minutes, and every minute while on battery; kept 35 days): battery charge, runtime left, load, mains
|
||||||
|
voltage in and out, model and maker. **Monitoring → UPS** shows it, with the history and the times mains power was lost.
|
||||||
|
- **Alerts** (category *Monitoring*, source *UPS*; announced and recovered like the other health problems, held back by maintenance on the integration): **mains power has failed — running on battery** (critical;
|
||||||
|
NUT `OB`, SNMP output source *on battery*), **the battery is nearly empty** (critical; NUT `LB`, SNMP battery low or depleted), **the UPS asks for its battery to be replaced** (warning; NUT `RB`), and **a load of 90 %
|
||||||
|
or more, or an overload** (warning; NUT `OVER`).
|
||||||
|
- **It is a node in the dependency map** (`integration:<id>`): say what it powers under **Map → Dependencies**. While it runs on battery it counts as *down*, so servers that go offline with it are folded under its
|
||||||
|
alert (which says "also takes down …") instead of each being announced. The UPS page lists what needs it in the **order to shut it down** — what nothing else needs first, then what that needed — so nothing is
|
||||||
|
switched off while something that depends on it is still running.
|
||||||
|
- If the UPS cannot be read for five minutes its alerts are neither raised nor cleared, the Alerts page says it could not check, and the integration is raised as failing like any other.
|
||||||
|
- `/metrics` has `homelab_ups_on_battery`, `homelab_ups_battery_charge_percent`, `homelab_ups_runtime_seconds`, `homelab_ups_load_percent` and `homelab_ups_input_volts` for each UPS that was read lately.
|
||||||
|
- **Not verified against a real UPS** — NUT's protocol and the UPS-MIB are as published, and both readers are tried against pretend servers. Some UPSs leave out objects (the page then shows a dash), and some
|
||||||
|
network cards number the load or voltage tables differently; if a value is missing, the **Diagnostic Log** has what was asked and answered.
|
||||||
|
|
||||||
|
### Jellyfin
|
||||||
|
|
||||||
|
- **Config fields:** the **address** of Jellyfin (`http://host:8096`), an **API key** (Jellyfin → Dashboard → API Keys → add one; stored encrypted) and, for an https address with a certificate you made yourself, *insecure*.
|
||||||
|
- **Required access:** read only. The API key is Jellyfin's own, which is administrator-level in Jellyfin; this app only ever sends GET requests for the server's information, its sessions, its scheduled tasks, the item counts and the activity log. **Nothing is ever changed or played.**
|
||||||
|
- **What it does:** every **two minutes** Jellyfin is read and stored — its version, whether it **wants a restart** or has an update, **who is streaming what** and whether it is being **transcoded**, how its **scheduled tasks** (a library scan) last ended, how many films, series, episodes and songs there are, and how many
|
||||||
|
**sign-ins failed** in the last day. **Apps → Jellyfin** shows it, with a chart of the most that were streaming at once (readings kept 35 days).
|
||||||
|
- **Alerts** (category *Monitoring*, source *Jellyfin*; held back by maintenance on the integration): **waiting to be restarted** (warning), **a scheduled task failed** (warning; the task's name and error), **10 or more sign-ins failed in a day** (warning).
|
||||||
|
- **Somebody is watching.** If the Jellyfin runs on a server this app knows (the Jellyfin's address is that server's address or host name), then **before an update run restarts that machine** the update drawer says who is streaming and asks again, and the server answers `409 watching` to a run started without saying it should go ahead anyway.
|
||||||
|
A **patch plan** that comes to that server while people are streaming **waits for them, up to two hours**, then goes ahead and says so in its message; the server's row in the run reads *Waiting: 2 people are streaming from Cinema (Ann: Dune, …)*. Only a reading from the last ten minutes counts: if Jellyfin cannot be read, nobody is assumed to be watching.
|
||||||
|
- If Jellyfin cannot be read for ten minutes its alerts are neither raised nor cleared, the Alerts page says it could not check, and the integration is raised as failing like any other.
|
||||||
|
- `/metrics` has `homelab_jellyfin_streams`, `homelab_jellyfin_transcodes` and `homelab_jellyfin_pending_restart` for each Jellyfin that was read lately.
|
||||||
|
- **Checked against a live Jellyfin** (Inspect): `/System/Info`, `/Sessions`, `/ScheduledTasks` (19 tasks, with their last result), `/Items/Counts` and `/System/ActivityLog/Entries` all answered as the reader expects. Two things are empty there and right to be: the operating system (this Jellyfin leaves `OperatingSystemDisplayName` and `OperatingSystem` blank, as newer versions do, so the page shows no OS) and the failed-task message (no task had failed). **Not yet seen with something in them:** a playing stream (there were no active sessions) and failed sign-ins in the activity log (there were none); those are as Jellyfin publishes them and tried against a pretend server. Every field is read defensively, and a part that cannot be read (older versions
|
||||||
|
differ) is left out without failing the rest; the **Diagnostic Log** has what was asked and answered.
|
||||||
|
|
||||||
|
### Immich
|
||||||
|
|
||||||
|
- **Config fields:** the **address** of Immich (`http://host:2283`), an **API key** (Immich → Account Settings → API Keys; stored encrypted) and, for an https address with a certificate you made yourself, *insecure*.
|
||||||
|
- **Required access:** read only. Give the key just these permissions: `server.about`, `server.storage`, `server.versionCheck`, `server.statistics` and `job.read` (`queue.read` on newer versions) — statistics and jobs need a key **made by an administrator user** (the permissions alone are not enough; Immich answers “Forbidden”). Without one of them that part is left out (and **Inspect** shows the reason Immich gave, such as *Missing required permission: server.versionCheck*). This app only ever sends GET requests; **nothing is changed, and no job is started, paused or cleared**.
|
||||||
|
If one of those is refused, that part is left out and the rest still works.
|
||||||
|
- **What it does:** every **five minutes** Immich is read and stored — its version (and whether a newer one is out), how full the **disk under the library** is, how many **photos and videos** there are and what they take, and the **background job queues** with how many jobs **failed** in each.
|
||||||
|
**Apps → Immich** shows it.
|
||||||
|
- **Alerts** (source *Immich*; held back by maintenance on the integration): **the disk under the library is 90 % full** (warning; **95 %**, critical) and **10 or more background jobs have failed**, all queues together (warning; the worst queues are named — thumbnails or metadata that were never made.
|
||||||
|
Failed jobs stay counted until you run them again or clear them in Immich, so the alert stays until you do).
|
||||||
|
- If Immich cannot be read for 20 minutes its alerts are neither raised nor cleared, the Alerts page says it could not check, and the integration is raised as failing like any other. A part that could not be asked is *unknown*, never *none*.
|
||||||
|
- `/metrics` has `homelab_immich_photos`, `homelab_immich_videos`, `homelab_immich_library_bytes`, `homelab_immich_disk_used_percent` and `homelab_immich_failed_jobs` for each Immich that was read lately.
|
||||||
|
- **Checked against a live Immich** (Inspect, with a key made by an administrator): `/api/server/about`, `/server/storage`, `/server/version-check`, `/server/statistics` (photos, videos, usage, and usage for each user) and `/api/jobs` (19 queues, with active, waiting, failed and paused) all answered as the reader expects, and nothing the reader uses was empty. Without an administrator's key, statistics and the job queues are refused with a plain “Forbidden” (a missing permission reads “Missing required permission: …” instead), and the page says so where those figures would be. This Immich answers on `/jobs`; `/queues` (newer versions) has **not been seen live**. `/api/queues` is as Immich publishes it, and the reader is tried against a pretend server that answers both ways. Immich renames
|
||||||
|
these now and then; the **Diagnostic Log** has what was asked and answered if something comes back empty. Whether a *waiting* backlog is a problem is not judged: a big import has one for days.
|
||||||
|
|
||||||
|
### Nextcloud
|
||||||
|
|
||||||
|
- **Config fields:** the **address** of Nextcloud (`https://cloud.example.lan`); optionally an **administrator's name and an app password** (Personal settings → Security → Devices & sessions → *Create new app password*; stored encrypted — never the account's own password); and, for an https address with a certificate you made
|
||||||
|
yourself, *insecure*.
|
||||||
|
- **Required access:** read only. Without a login only `/status.php` is read, which Nextcloud answers to anyone. With one, the *Monitoring* app (`serverinfo`, installed by default) is also asked, which needs an **administrator**; if it is refused or the app is off, the status is still read and the page says so. This
|
||||||
|
app only ever sends GET requests, and **nothing is changed**.
|
||||||
|
- **What it does:** every **five minutes** Nextcloud is read and stored — its version, whether it is in **maintenance mode**, whether it **needs its database upgrading**, and (with the login) the **room left where the data is kept**, the users, the files, who was active in the last day and how many **app updates** wait. **Apps → Nextcloud** shows it.
|
||||||
|
- **Alerts** (source *Nextcloud*; held back by maintenance on the integration): **the database needs upgrading** (critical — nobody can use it until `occ upgrade`), **maintenance mode is on** (warning — nobody can sign in), and **less than 20 GB is left for the data** (warning; under **5 GB**, critical), **the background jobs are not running** (warning — Nextcloud's own *Cron last run* check says error, which it does when the last run was more than an hour ago; see below) and **Nextcloud's own checks report an error** (warning; one alert naming them).
|
||||||
|
- If Nextcloud cannot be read for 20 minutes its alerts are neither raised nor cleared, the Alerts page says it could not check, and the integration is raised as failing like any other.
|
||||||
|
- `/metrics` has `homelab_nextcloud_maintenance_mode`, `homelab_nextcloud_free_bytes`, `homelab_nextcloud_active_users_24h` and `homelab_nextcloud_app_updates` for each Nextcloud that was read lately.
|
||||||
|
- **Not verified against a live Nextcloud** — `/status.php` and the Monitoring app's `/ocs/v2.php/apps/serverinfo/api/v1/info` are as Nextcloud publishes them and the reader is tried against a pretend server. **Cron** is read a different way, because the Monitoring app has no field for it (its source was read to be sure): with the same login, Nextcloud's own **setup checks** are asked (`GET /settings/setupcheck`, **Nextcloud 28 and later**; no administrator is needed), and the one that is Nextcloud's *Cron last run* — found by its class
|
||||||
|
name, not by its text, which is in the language Nextcloud is set to — gives the alert above and the *Background jobs* line on the page. Any other of those checks that reports an **error** is one alert, and ones that report a **warning** (HTTP headers, a PHP setting …) are only listed on the page, as in Nextcloud's own admin overview.
|
||||||
|
**Before Nextcloud 28** there is no such page: cron and the checks are then *not known* (and say so, never *fine*). The same read gives the version Nextcloud itself says is available, shown on the page. With **Nextcloud AIO** the admin pages are the mastercontainer's, but the Nextcloud behind it is an ordinary one and is read the same way (use its own address, not the AIO admin address).
|
||||||
|
|
||||||
|
### Sonarr, Radarr and Prowlarr
|
||||||
|
|
||||||
|
- **Config fields:** the application's **URL** and its **API key** (in the application: Settings → General → Security → API Key; stored encrypted), and *Allow self-signed certificate* if it is on HTTPS with its own certificate. Add each application as its own integration, as the kind it is — the *Test* button says if the address turns out to be another one. The key is sent as `X-Api-Key`; the app only ever sends GET requests.
|
||||||
|
- **Lidarr and Readarr** (music and books) are the same: add them as their own kinds. They answer under `/api/v1` (as Prowlarr does), and ask for their own unknown items in the queue (`includeUnknownArtistItems`, `includeUnknownAuthorItems`). The *Test* button says if an address is another one of the family. Readarr's own project has been retired; the integration reads whatever install you still run.
|
||||||
|
- **Required access:** the application's address reachable from the app. Sonarr and Radarr answer under `/api/v3`, Prowlarr under `/api/v1`. A read is five to six requests, every five minutes.
|
||||||
|
- **What it does:** every **five minutes** each application is read — its version, what its own checks found (`/health`), its **download queue** (`/queue`: the first 100, and how many there are), its **disks** (`/diskspace`), for Sonarr and Radarr how many **wanted** items have no file (`/wanted/missing`), and for Prowlarr its **indexers** and which it has switched off after failures (`/indexer`, `/indexerstatus`). **Apps → Media managers** shows them. Nothing is searched, grabbed, removed or changed.
|
||||||
|
- **Alerts** (sources *Sonarr*, *Radarr*, *Prowlarr*; held back by maintenance on the integration): what the application **says is wrong** with itself (a finding of type warning or error — a download client that cannot be reached, a root folder gone, indexers unavailable; a *notice*, such as an update being available, is shown but not raised); **downloads that are stuck** (in the queue for two hours or more and judged warning or error, failed, or blocked from importing — one alert for the lot, naming the first three and why); a **disk nearly full** (under 10% free, critical under 5%; folders on one disk count once, and a disk under 2 GB is not a media disk). See [MONITORING.md](docs/MONITORING.md#media-managers-sonarr-radarr-prowlarr).
|
||||||
|
- If an application cannot be read for 20 minutes its alerts are neither raised nor cleared, the Alerts page says it could not check, and the integration is raised as failing like any other. A part that could not be read (the queue, say) is shown as *could not be read*, never as empty.
|
||||||
|
- `/metrics` has `homelab_arr_health_issues`, `homelab_arr_queue_items`, `homelab_arr_queue_stuck_items`, `homelab_arr_wanted_missing` and `homelab_arr_disk_free_bytes`.
|
||||||
|
- **Not verified against live installs** — the endpoints and fields are as the applications' published OpenAPI files and source have them (HealthResource, QueueResource, DiskSpaceResource, IndexerStatusResource), and the reader is tried against pretend ones. A finding is raised under a key made from the check and its words (numbers left out), so a finding whose wording changes is raised anew. Lidarr, Readarr and Bazarr are not read.
|
||||||
|
|
||||||
|
### qBittorrent and SABnzbd
|
||||||
|
|
||||||
|
- **Config fields:** for **qBittorrent**, the **Web UI URL** (Tools → Options → Web UI) and, if it asks for a sign-in, the **user name** and **password** (stored encrypted); leave them empty for one that lets this address in without signing in (*Bypass authentication for clients in whitelisted IP subnets*). For **SABnzbd**, its **URL** and its **API key** (Config → General → Security → API Key — the full API key, not the NZB key; stored encrypted). *Allow self-signed certificate* for either on HTTPS with its own certificate. Add each as its own integration.
|
||||||
|
- **Required access:** the address reachable from the app. qBittorrent is read through its Web API v2 (`/api/v2/…`): the app signs in with the form (`POST /auth/login`), sends the session cookie and its own address as the `Referer` (qBittorrent's protection against forged requests wants that), reads, and signs out. SABnzbd is read through `/api?mode=…&output=json&apikey=…`. A read is three to five requests, every five minutes. Nothing is sent that adds, pauses, resumes, removes or changes anything — the sign-in is the only POST.
|
||||||
|
- **What it does:** every **five minutes** each client is read — qBittorrent: its version, every torrent (`/sync/maindata`, or `/torrents/info` where that is not answered) with its state, progress, speed and last activity, the speeds and whether it can reach the network (`connection_status`: connected, firewalled or disconnected) and the free space of its download folder; SABnzbd: its version, the queue (`mode=queue`: up to 200 slots, paused or not, the speed, and the free and total space of the incomplete and complete folders), what failed (`mode=history`: the last 50 entries, the ones that failed) and its own warnings (`mode=warnings`). **Apps → Download clients** shows them, the items that need a look first; items that are finished and idle (seeding, stopped) are counted, not listed (at most 100 are kept).
|
||||||
|
- **Alerts** (sources *qBittorrent*, *SABnzbd*; held back by maintenance on the integration): **downloads stuck** (in an error state — qBittorrent's `error` and `missingFiles` — or *stalled* with nothing moving for a day; one alert for the lot, naming the first three); **no connection** (qBittorrent, `disconnected` with torrents loaded for 15 minutes — a VPN that dropped; *firewalled* is shown but not raised, since it still downloads); **paused and forgotten** (SABnzbd's queue paused for six hours with downloads waiting); **downloading nothing** (SABnzbd, not paused, work waiting, and no speed for an hour — a news server that does not answer or refuses the login); a **download disk nearly full** (SABnzbd says how big the disks are: under 10% free, critical under 5%; qBittorrent says only what is free: under 20 GB, critical under 5 GB). Failures in SABnzbd's history are shown for the last day but are not an alert, since a failed download is normal on Usenet and the *arr applications try another release. See [MONITORING.md](docs/MONITORING.md#download-clients-qbittorrent-sabnzbd).
|
||||||
|
- "Since when" (disconnected, paused, idle) is what this app saw: the time of the first read that showed it.
|
||||||
|
- If a client cannot be read for 20 minutes its alerts are neither raised nor cleared, the Alerts page says it could not check, and the integration is raised as failing like any other. A part that could not be read (the history, say) is shown as *could not be read*, never as empty.
|
||||||
|
- `/metrics` has `homelab_download_speed_bytes`, `homelab_download_items`, `homelab_download_stuck_items`, `homelab_download_paused`, `homelab_download_connected`, `homelab_download_failed_recent` and `homelab_download_disk_free_bytes`.
|
||||||
|
- **A wrong password is retried every five minutes**, and qBittorrent bans an address for an hour after a few wrong ones (*Web UI → Ban client after consecutive failures*). If the integration says it was refused with HTTP 403, fix the password and wait out the ban (or restart qBittorrent).
|
||||||
|
- **Not verified against live installs** — the endpoints and fields are as the two applications' published API documentation has them, and the reader is tried against pretend ones. qBittorrent 5 renamed some states (`pausedDL` became `stoppedDL`); both are understood. SABnzbd's figures arrive as text, in megabytes and gigabytes, and are converted. Torrents are not paused, removed or added from here, and no NZB is added.
|
||||||
|
|
||||||
|
### Hetzner Robot
|
||||||
|
|
||||||
|
- **Config fields:** the **webservice user name** (it looks like `#ws+abc123…`) and its **password** (stored encrypted), made in Hetzner Robot → Settings → Webservice and app settings. That is not the login to Robot itself. There is no address to give: Robot is always `https://robot-ws.your-server.de`. This is the *dedicated servers and storage boxes* side of Hetzner; the cloud servers are the **Hetzner Cloud** integration, with a project token.
|
||||||
|
- **Required access:** the webservice user, reachable from the app over HTTPS. HTTP Basic is sent with every request, and the app only ever sends GET requests.
|
||||||
|
- **What it does:** every **fifteen minutes** it reads `GET /server` (number, name, product, data centre, addresses, traffic plan, status, cancelled, paid until), `GET /server/{number}` for each server (whether its **rescue system** is on — it is not in the list) and `GET /storagebox` (product, location, cancelled, locked, paid until, the linked server). **Infrastructure → Hetzner Robot** shows them. Nothing is reset, nothing is activated, ordered or cancelled.
|
||||||
|
- **Fifteen minutes, and 25 servers:** Robot allows **200 requests an hour**, and a read is a list, one request per server and a list of storage boxes — so ten servers is 12 requests a read, 48 an hour. Only the first 25 servers (by number) are asked about their rescue system; any more are listed, counted on the page, and left without a rescue state.
|
||||||
|
- **Alerts** (source *Hetzner Robot*; held back by maintenance on the integration): a **server or storage box that is cancelled and ends within 30 days** (critical within 7; the date is the last day it runs, and the data goes with it); a **storage box that is locked** (critical); a **rescue system on for two hours** (the server's next boot is into the rescue system); a server **still "in process" a day** after the app first saw it so. See [MONITORING.md](docs/MONITORING.md#hetzner-robot).
|
||||||
|
- "Since when" (rescue on, not ready) is what this app saw: the time of the first read that showed it.
|
||||||
|
- If the account cannot be read for 45 minutes its alerts are neither raised nor cleared, the Alerts page says it could not check, and the integration is raised as failing like any other. A part that could not be read (the storage boxes, a server's rescue state) is shown as *could not be read*, never as none or off.
|
||||||
|
- `/metrics` has `homelab_hetzner_robot_servers`, `homelab_hetzner_robot_rescue_active`, `homelab_hetzner_robot_days_left` and `homelab_hetzner_robot_storage_box_locked`.
|
||||||
|
- **A wrong password is retried every fifteen minutes**, and Robot blocks an address for a while after a few wrong ones (HTTP 403). If the integration says it was refused, fix the password and wait.
|
||||||
|
- **Not verified against a live account** — the endpoints and fields are as Hetzner's published webservice documentation has them, and the reader is tried against a pretend one. An account with no servers (or no storage boxes) is answered with a 404 and a code ending in `NOT_FOUND`, read as none. Whether Robot switches the rescue system off by itself after a while is not known here, so the alert is for one that has been on two hours; traffic used is not read (Robot gives it per address, and the plan only).
|
||||||
|
|
||||||
|
### Hetzner Cloud
|
||||||
|
|
||||||
|
- **Config fields:** the project's **API token** (Hetzner Console → the project → Security → API tokens → *Generate API token*; stored encrypted). **Read** permission is enough — and recommended: the app only ever sends GET requests. A token belongs to **one project**, so add one integration for each project you want to watch.
|
||||||
|
- **Required access:** `api.hetzner.cloud` reachable from the app. The API allows 3600 requests an hour per project; a read is about ten requests (plus one for each further page of 50 servers), every five minutes — well inside it.
|
||||||
|
- **What it does:** every **five minutes** the project's servers, volumes, snapshots and backups, primary IPs and Hetzner's price list are read and stored. The **Hetzner Cloud** page shows them with a monthly cost estimate; a server's own page gets a card when it is one of them (matched by address, else by name).
|
||||||
|
- **Alerts** (source *Hetzner*; held back by maintenance on the integration): a server that **was running and is off** (warning; left alone after a week), one **stuck** migrating or rebuilding for half an hour, **traffic** at 80% of the month's quota (critical at 100%), **rescue mode** left on for an hour, and **backups** enabled but late (the same limit as Proxmox guests' backups, 48 hours by default). See [MONITORING.md](docs/MONITORING.md#hetzner-cloud).
|
||||||
|
- If Hetzner cannot be read for 20 minutes its alerts are neither raised nor cleared, the Alerts page says it could not check, and the integration is raised as failing like any other. A list that could not be asked (the volumes, say) is shown as *could not be read*, never as empty.
|
||||||
|
- **Load balancers, floating IPs and certificates** are read with the rest (a list that cannot be asked is shown as *could not be read*, never as none): the Hetzner page lists each **load balancer** with the health of its targets, each **floating IP** and what it points at, and each **certificate** with when it ends. Their prices (a load balancer's type, a floating IP's kind and place) are now in the cost estimate. Alerts: a **load balancer with an unhealthy target** (warning; critical when none of its targets is healthy), a **certificate** that is in use and ends within 14 days (critical within 3, or past its end) or that Hetzner could not issue or renew, and a **floating IP that points at no server** (it is billed all the same). A certificate nothing uses is not raised.
|
||||||
|
- **Traffic forecast:** from the month's traffic the app writes down about every six hours it works out how the month is going — by the last week's rate, or, with too little of it, by the month's average (after three days) — and warns, below the 80% alert, when a server is **on course to pass its quota before the month ends**: "at this rate web will pass its 20.0 TB traffic quota on 24 June". The page shows the date, or what the month will come to, under each server's traffic bar. The month is the calendar month in UTC; a counter that goes down starts the history over.
|
||||||
|
- `/metrics` has `homelab_hetzner_server_running`, `homelab_hetzner_traffic_outgoing_bytes`, `homelab_hetzner_traffic_included_bytes` and `homelab_hetzner_monthly_cost`.
|
||||||
|
- **Checked against a live project** (Inspect, a project with two servers, snapshots and primary IPs): `/servers`, `/images?type=snapshot` and `type=backup`, `/primary_ips`, `/pricing` and the empty lists of `/volumes`, `/load_balancers`, `/floating_ips` and `/certificates` all answered as the reader expects, and every field it reads for servers, snapshots, primary IPs and prices had a value. **Not yet seen with something in them:** volumes, load balancers (and their target health), floating IPs, certificates and servers on a private network — those are as Hetzner's published OpenAPI file has them, and the reader is tried against a pretend server. **Traffic** is read as bytes and shown in binary units (Hetzner's 20 TB is 20 TiB); a server's *included* traffic and the *outgoing* traffic of the current billing month come from the server itself. The **cost** is the list price including VAT (the `gross` prices): check it against Hetzner's invoice the first time. Dedicated (Robot) servers and Hetzner Storage Boxes are a different API and are not read.
|
||||||
|
|
||||||
|
### Fragnet game servers
|
||||||
|
|
||||||
|
- **Config fields:** the account's **API key** (panel.fragnet.net → your profile → *API Credentials* → create one; it starts with **`ptlc_`** and is shown only once; stored encrypted). It must be the account's own key — an *Application* key (`ptla_`) is for panel administrators and does not work, and the app says so. Optional: the **panel address** (`https://panel.fragnet.net` when left out — fill it in for another Pterodactyl panel, since Fragnet's is a fork of Pterodactyl and this speaks its *client* API), **warn when the newest backup is older than** *n* days (7; 0 turns backups off), **warn when disk or memory is fuller than** *n* % (90), **servers whose backups are not judged** (names, comma-separated) and three tick boxes that switch one kind of alert off for the whole account — **do not alert about disk space**, **memory** or **backups** (the page still shows the figures). A key may be limited to certain addresses in the panel; if so, allow the app's.
|
||||||
|
- **Required access:** `panel.fragnet.net` (HTTPS) reachable from the app. The panel limits the requests per minute (429 when over); a read is four requests for each game server, every five minutes.
|
||||||
|
- **What it does:** every **five minutes** each game server of the account (also those shared with it) is read — the game (the panel's *egg*, for example *Minecraft: Paper*), every IP address and port, SFTP, the plan's memory, disk and CPU limits and its room for backups and databases, whether it is running, what it uses, its backups, schedules and databases. **Infrastructure → Game servers** shows them; a row opens to show every address (click to copy), the backups, the schedules and a link to the server in the game panel.
|
||||||
|
- **Power:** operators can **start, stop, restart and kill** a server from the page (a confirmation first; kill says it loses unsaved progress). It sends exactly that signal to the panel and writes it to the audit log (`start_game_server` …). The page reads again a few seconds later. Beside backups, it is the only thing this integration ever sends but reads, and only a server the last read found can be signalled.
|
||||||
|
- **Back up now:** operators can ask for a backup of a server whose plan has room for backups (*Back up* on its row, or *Back up now* beside its backups; a confirmation first, which says how many the plan allows and how many are made). It is `POST /api/client/servers/{id}/backups` with a name that says it was asked for here (`Homelab Manager — 2026-11-04 14:30 UTC`) and nothing else — no file is left out, nothing is locked. The panel makes it in the background; the page reads the account again after a few seconds and then every twenty seconds until the backup has finished. It is written to the audit log (`backup_game_server`, with the backup's id). **Refused here before the panel is asked:** a plan with no room for backups, a server that is suspended or being installed, one for which a backup is already being made (one that has been "being made" for more than two hours is taken to be lost and does not count), and the same server asked again within 30 seconds (HTTP 429). **Refused by the panel, and said in its words:** a server at the number of backups its plan allows (delete one in the panel first — the app never deletes a backup), the panel's own limit of a couple of backups in ten minutes (429), an account that may not make backups on a shared server (403 — it needs the *Backups → Create* permission) and a server that is busy (409). A refusal costs no wait before the next try.
|
||||||
|
- **Alerts** (source *Fragnet*; held back by maintenance on the integration): a server whose plan allows backups and whose **newest successful backup is older** than the limit (or has none, once the app has known it for that long; a backup that failed does not count, and the message says when the newest attempt failed); **disk** over the limit (critical from 98%); **memory** over the limit while it runs. A server that is **stopped** is not an alert. (When one is turned on or off, a notification says so: [NOTIFICATIONS.md](NOTIFICATIONS.md).)
|
||||||
|
- **Power history.** Every change a read sees (turned on, turned off) is written down, and so is each Start, Stop, Restart or Kill sent from here with who sent it, so a row's details have a **Power history**: the share of the last 30 days (or of what has been watched, when less) that the server was on, how many times it started and stopped, a bar for the last seven days (green on, grey off), and the latest changes — *"Turned on — start by Ada"* when the change followed a signal sent from here in the 20 minutes before. It is what this app saw: the account is read every five minutes, so every time is to the nearest five, and nothing is known of the time before a server was first read. Kept 180 days; it goes with the integration. See [MONITORING.md](docs/MONITORING.md#game-servers-fragnet).
|
||||||
|
- If the panel cannot be read for 20 minutes its alerts are neither raised nor cleared, the Alerts page says it could not check, and the integration is raised as failing like any other. A part that cannot be read for one server (a sub-user without the *database* permission, say) is shown as *could not be listed*, never as none.
|
||||||
|
- `/metrics` has `homelab_fragnet_server_running`, `homelab_fragnet_disk_used_bytes`, `homelab_fragnet_disk_limit_bytes`, `homelab_fragnet_memory_used_bytes`, `homelab_fragnet_memory_limit_bytes` and `homelab_fragnet_newest_backup_timestamp_seconds`.
|
||||||
|
- **Reads checked against a live account** (Inspect, four servers: the server list with its egg and allocations, `…/resources`, `…/backups`, `…/schedules` and `…/databases` all answered as the reader expects; the fields it leaves empty — description, panel status, allocation notes — were empty in the answer too). **Not checked live:** `POST …/power` and `POST …/backups`, which are as Pterodactyl's own source and Fragnet's documentation have them. A server can carry a second kind of schedule (every so often, or once, with `interval_value`/`interval_unit`/`run_at`, not a cron): it reads "every 6 hours" or "once, …" instead of a cron. Fragnet's fork may differ: the game is left out (shown as the Docker image) if the panel does not send the egg, and a panel that refuses the extra `include` is asked again plainly. Disk and memory limits are in MB as the panel counts them and shown in binary units. What the servers cost is not in the API: add it under Records → Costs.
|
||||||
|
|
||||||
|
### AdGuard Home
|
||||||
|
|
||||||
|
- **Config fields:** the **URL** of its web interface and, if it has a sign-in, the **user name** and **password** (HTTP Basic; stored encrypted). Optional: the **DNS server address** (the host of the URL unless told — fill it in if a reverse proxy sits in front of the web interface), the **name to test DNS with** (`example.com`) and *Allow self-signed certificate*.
|
||||||
|
- **Required access:** the web interface reachable from the app over HTTP(S), and — for the DNS test — its DNS port (read from its status, usually 53) reachable from the app **over UDP**. If the app sits where UDP 53 to AdGuard Home is blocked, the test will say so (no answer) and DNS will be reported down; allow it or set the DNS server address to one the app can reach.
|
||||||
|
- **What it does:** every **two minutes** it reads `GET /control/status` (running, protection, version, DNS port), `GET /control/stats` (queries, blocked, average time, top names and clients) and `GET /control/filtering/status` (the lists and when each was updated) and puts **one DNS question** — the address of the test name — to its DNS server. An answer, or "no such name", means the resolver works; a server failure or a refusal does not. **Network → AdGuard Home** shows it all. Nothing is changed: protection is never switched, no list refreshed.
|
||||||
|
- **Alerts** (source *AdGuard Home*; held back by maintenance on the integration): the DNS server **not running** (critical); the test question **unanswered in every read for ten minutes** (critical — one lost packet is not an outage); **protection switched off for an hour** (warning); **filter lists** switched on and **not updated for a week** (warning, one for all). See [MONITORING.md](docs/MONITORING.md#adguard-home).
|
||||||
|
- If it cannot be read for ten minutes its alerts are neither raised nor cleared, the Alerts page says it could not check, and the integration is raised as failing like any other. A part that could not be read (the statistics, the lists) is shown as *could not be read*, never as none.
|
||||||
|
- `/metrics` has `homelab_adguard_running`, `homelab_adguard_protection_enabled`, `homelab_adguard_dns_up`, `homelab_adguard_dns_probe_seconds`, `homelab_adguard_queries`, `homelab_adguard_blocked` and `homelab_adguard_avg_processing_seconds`.
|
||||||
|
- **Not verified against a live install** — the endpoints and fields are as AdGuard Home's published OpenAPI file has them (`ServerStatus`, `Stats`, `FilterStatus`), and the reader (and the DNS question, against a pretend DNS server) is tried against pretend ones. The statistics are over AdGuard Home's own period (a day unless set otherwise). A DNS question is asked of the host in the URL: with a proxy in front, set *DNS server address*.
|
||||||
|
|
||||||
|
### Technitium DNS
|
||||||
|
|
||||||
|
- **Config fields:** the **URL** of its web console (`http://dns.example.lan:5380`) and an **API token** — a non-expiring one, made under Administration → Sessions → *Create Token*; stored encrypted — of a user who may **view** the dashboard, the settings and the zones (the DHCP server, for the DHCP counts, and *administration*, for the cluster — a token without it still works; the cluster is just not shown). Optional: the **DNS server address** (the host of the URL unless told — fill it in if a reverse proxy sits in front of the console), the **name to test DNS with** (`example.com`) and *Allow self-signed certificate*. Zones and records are connected separately, as a **DNS provider** on the DNS page, with a token of its own; this integration only says how the server is doing.
|
||||||
|
- **Required access:** the console reachable from the app over HTTP(S) and — for the DNS test — the DNS port (the first of the server's listening endpoints, usually 53) reachable from the app **over UDP**. The token goes in an `Authorization: Bearer` header, which Technitium **15.0 and later** require; an older server answers "token not accepted". If the app sits where UDP 53 to it is blocked, the test says so (no answer) and DNS is reported down; allow it or set the DNS server address to one the app can reach.
|
||||||
|
- **What it does:** every **two minutes** it reads `GET /api/settings/get` (version, when it started, whether blocking is on, the block lists and when they are next updated, the cache limit, recursion, DNSSEC, the DNS port), `GET /api/dashboard/stats/get?type=LastDay` (the last day's queries and how each ended, clients, cache entries, zone and list counts, the clients and names asked about and blocked most), `GET /api/zones/list` (which secondary zones have expired or cannot sync; the server's own internal zones and switched-off zones are not judged), `GET /api/admin/cluster/state` (if it is part of a **cluster**, Technitium 14 and later: the nodes, which is primary, which of them it can reach and when it last took the configuration from the primary; it needs the token's user to be allowed to *view administration* — without that the rest is read as before and the page says what is missing; a server in no cluster shows nothing) and `GET /api/dhcp/scopes/list` and `…/leases/list` (scopes and lease counts) — and puts **one DNS question** to its DNS server, as for AdGuard Home. **Network → Technitium DNS** shows it all, and the dashboard has a **widget** for it in its Network section (whether DNS answers and how fast, the last day's queries and the share blocked, how they were answered, and what is wrong). Nothing is changed: blocking is never switched, no block list refreshed, no zone touched.
|
||||||
|
- **Alerts** (source *Technitium DNS*; held back by maintenance on the integration): the test question **unanswered in every read for ten minutes** (critical — one lost packet is not an outage); a **secondary zone that has expired** (critical — it no longer answers), or zones that **cannot sync** (warning), one alert for all the zones; **blocking switched off for an hour** while block lists are set up (warning — with none, off is how it is); **block lists whose update was due more than six hours ago** (warning); a **cluster node that has been unreachable for ten minutes** (warning for a secondary, critical for the primary — one alert for each node); a **secondary that has not taken the configuration from the primary** for three of its refresh periods and at least half an hour (warning). See [MONITORING.md](docs/MONITORING.md#technitium-dns).
|
||||||
|
- If it cannot be read for ten minutes its alerts are neither raised nor cleared, the Alerts page says it could not check, and the integration is raised as failing like any other. A part that could not be read (the statistics, the zones, DHCP) is shown as *could not be read*, never as none; the settings are the one part that must work, since they say it is there and the token is accepted.
|
||||||
|
- `/metrics` has `homelab_technitium_dns_up`, `homelab_technitium_dns_probe_seconds`, `homelab_technitium_blocking_enabled`, `homelab_technitium_uptime_seconds`, `homelab_technitium_block_lists_overdue`, `homelab_technitium_queries`, `homelab_technitium_blocked`, `homelab_technitium_cached`, `homelab_technitium_recursive`, `homelab_technitium_server_failures`, `homelab_technitium_clients`, `homelab_technitium_cache_entries`, `homelab_technitium_zones`, `homelab_technitium_zone_problems` and `homelab_technitium_dhcp_leases`, and for a clustered server `homelab_technitium_cluster_nodes`, `homelab_technitium_cluster_nodes_unreachable` and `homelab_technitium_cluster_config_synced_timestamp_seconds`.
|
||||||
|
- **No CPU, memory or disk.** Technitium's API has no figures for the machine it runs on (its "resource" numbers are the DNS server's own: queries, cache, zones). Put a **server agent** on that machine for those.
|
||||||
|
- **Not verified against a live install** — the endpoints and fields are as Technitium's own API documentation (`APIDOCS.md`) has them, and the reader (and the DNS question, against a pretend DNS server) is tried against pretend ones. The statistics are over the **last day**; the zones are read up to 1000 at a time; the page shows the share of failed queries in yellow from 20% (with at least 200 queries), which is not an alert. A DNS question is asked of the host in the URL: with a proxy in front, set *DNS server address*. Cluster state and the version-behind comparison of the Versions page are not read.
|
||||||
|
|
||||||
|
### Grafana
|
||||||
|
|
||||||
|
- **Config fields:** the **URL** and a **service account token** (Administration → Users and access → Service accounts → add a service account with the *Viewer* role → add a token; shown once; stored encrypted), and *Allow self-signed certificate*. To also test the data sources the service account needs permission to read and query them (the Viewer role usually has it); without it they are shown as *could not be listed* and the rules still work.
|
||||||
|
- **Required access:** Grafana's address reachable from the app. A read is a few requests plus one for each data source, every five minutes.
|
||||||
|
- **What it does:** every **five minutes** the alert rules (`GET /api/prometheus/grafana/api/v1/rules` — Grafana-managed alerting's Prometheus-compatible view: which are firing, pending, or cannot be evaluated) and the data sources with Grafana's own health check of each (`GET /api/datasources`, `GET /api/datasources/uid/{uid}/health`) are read and stored. **Monitoring → Grafana** shows them. Nothing is silenced, paused or changed.
|
||||||
|
- **Alerts** (source *Grafana*; held back by maintenance on the integration): a rule **firing** (critical when it carries the label `severity` = `critical` or `page`, a warning otherwise; a pending rule is shown, not raised), a rule **Grafana cannot evaluate**, and a **data source that fails its health check**. Grafana can notify about its own alerts too — with both switched on you hear twice; this is for having them in one list, or for the ones that reach you nowhere else. See [MONITORING.md](docs/MONITORING.md#grafana).
|
||||||
|
- If Grafana cannot be read for 20 minutes its alerts are neither raised nor cleared, the Alerts page says it could not check, and the integration is raised as failing like any other.
|
||||||
|
- `/metrics` has `homelab_grafana_rules`, `homelab_grafana_rules_firing`, `homelab_grafana_rules_error` and `homelab_grafana_datasources_failing`.
|
||||||
|
- **Not verified against a live Grafana** — the endpoints and fields are as Grafana's HTTP API documentation has them, and the reader is tried against a pretend one. Only **Grafana-managed** rules are read (rules of a Prometheus or Loki data source that Grafana merely shows are not). A data source whose type has no health check is *not tested*, which is not the same as healthy.
|
||||||
|
|
||||||
|
### Speedtest Tracker
|
||||||
|
|
||||||
|
- **Config fields:** the **address** of Speedtest Tracker (`http://speedtest.example.lan`) and an **API token** (the profile menu → *API tokens* → create one with the **results:read** ability; stored encrypted — that is all it needs). Optional: **the download and upload you pay for** (Mbit/s) and **the share below which to warn** (60% unless you say), which the alerts use, and, for an https address with a certificate of its own, *insecure*.
|
||||||
|
- **Required access:** read only. The app asks for `/api/healthcheck` (no login) and `/api/v1/results` and only ever sends GET requests — **no speed test is started** (that would be `POST /api/v1/speedtests/run`, which is never called).
|
||||||
|
- **What it does:** every **ten minutes** the newest results are read (three pages of 30 the first time, so there is a past to chart) and each is stored once for 90 days: when, download and upload, ping. The **Speedtest** page shows the latest test, the middle of the last three and charts of speed and ping over 1, 7, 30 or 90 days.
|
||||||
|
- **Alerts** (source *Speedtest*, category *Monitoring*; held back by maintenance on the integration): **the internet is slow** — the *middle* of the newest three completed tests is under the share (60%) of what you pay for, judged separately for download and upload and only for one whose number you gave; and **the speed tests are failing** — the newest three all failed (the internet is down, or Speedtest Tracker cannot reach Ookla). One slow test is not an alert: the middle of three is what is judged.
|
||||||
|
- If Speedtest Tracker cannot be read for 45 minutes its alerts are neither raised nor cleared, and the Alerts page says it could not check.
|
||||||
|
- On a **status page** the integration reads as the internet: *Working*, *Having trouble* when it is slow, *Not working* when the tests are failing.
|
||||||
|
- `/metrics` has `homelab_speedtest_download_bits_per_second`, `homelab_speedtest_upload_bits_per_second` and `homelab_speedtest_ping_seconds` for the newest completed test of each.
|
||||||
|
- **Not verified against a live Speedtest Tracker** — the endpoints and fields are read from its own source code (`routes/api/v1/routes.php`, the results controller and resource) and the reader is tried against a pretend server. `download` and `upload` are bytes per second there (the bits fields are used when present); a result's time is the one Ookla gives in its data, else `created_at`, taken as UTC. Speedtest Tracker may allow fewer than 30 results a page; the reader asks again without a size when it says so.
|
||||||
|
|
||||||
|
### NetBox
|
||||||
|
|
||||||
|
- **Config fields:** the **address** of NetBox (`https://netbox.example.lan`), an **API token** (your profile → API Tokens; stored encrypted; **write enabled off** is enough — both the older `Token` tokens and NetBox 4.5's `nbt_…` tokens work) and, for an https address with a certificate you made yourself, *insecure*.
|
||||||
|
- **Required access:** read only. `/api/status/`, and the counts and addresses of devices, virtual machines, prefixes and IP addresses. This app only ever sends GET requests; **nothing is changed**.
|
||||||
|
- **What it does:** every **five minutes** NetBox is read and stored — its version, **how many background workers are running** (they run its webhooks, scripts, jobs and reports; in the Docker image they are a separate container that can be down while the pages still load), how many devices, VMs, prefixes and addresses it holds, and the addresses
|
||||||
|
themselves (up to 5000). **Network → NetBox** shows it.
|
||||||
|
- **Alert** (source *NetBox*; held back by maintenance on the integration): **no background worker is running** (warning).
|
||||||
|
- **Not in NetBox**: the page lists the servers this app manages **none of whose addresses are in NetBox** (an address that is the same on every host, like Docker's bridge, is left out as on the Consistency page; loopback and link-local addresses say nothing). That is a **list, not an alert** — documentation that is behind is not an outage. If NetBox
|
||||||
|
holds more than 5000 addresses the page says what is missing may be in the part that was not read.
|
||||||
|
- If NetBox cannot be read for 20 minutes its alert is neither raised nor cleared, and the Alerts page says it could not check.
|
||||||
|
- `/metrics` has `homelab_netbox_workers`, `homelab_netbox_devices` and `homelab_netbox_ip_addresses` for each NetBox that was read lately.
|
||||||
|
- **Not verified against a live NetBox** — the endpoints and fields are as NetBox publishes them and the reader is tried against a pretend server that answers the addresses in pages. The **Diagnostic Log** has what was asked and answered.
|
||||||
|
|
||||||
|
### n8n
|
||||||
|
|
||||||
|
- **Config fields:** the **address** of n8n (`http://host:5678`), an **API key** (Settings → n8n API → *Create an API key*; stored encrypted) and, for an https address with a certificate you made yourself, *insecure*.
|
||||||
|
- **Required access:** read only. `GET /api/v1/workflows` and `GET /api/v1/executions` — the public API has to be switched on (it is on by default in current versions). This app only ever sends GET requests: **no workflow is started, changed or switched off**.
|
||||||
|
- **What it does:** every **five minutes** n8n is read and stored — the workflows and whether each is switched on, and, from the **newest 250 runs**, each workflow's **last outcome** and how many of its runs failed in the last day. **Automation → n8n** shows it.
|
||||||
|
- **Alert** (source *n8n*, category *Automation*; held back by maintenance on the integration): **a workflow that is switched on failed on its last run** (warning; one alert for each workflow, saying when and, if more than one run failed, how many in the last day). It **clears by itself** when the workflow next succeeds. A workflow that is switched off, or
|
||||||
|
whose failure is more than a week old, is not raised.
|
||||||
|
- If n8n cannot be read for 20 minutes its alerts are neither raised nor cleared, and the Alerts page says it could not check. If the runs could not be read (but the workflows could), the last outcomes are *unknown* and nothing is claimed.
|
||||||
|
- `/metrics` has `homelab_n8n_workflows`, `homelab_n8n_workflows_active` and `homelab_n8n_workflows_failing` for each n8n that was read lately.
|
||||||
|
- **Checked against a live n8n** (Inspect: the workflow list and the run list both answered as the reader expects, with nothing empty). **Archived workflows** (`isArchived`) are left out. It is also tried against a pretend server that pages with a cursor. A workflow that runs less often than 250 other runs go by may show no run (and so no alert); the page says it is judged from the newest 250.
|
||||||
|
|
||||||
|
### Paperless-ngx
|
||||||
|
|
||||||
|
- **Config fields:** the **address** of Paperless-ngx (`http://host:8000`), an **API token** (your profile → *API Auth Token*, or Auth Tokens in the admin pages; stored encrypted) and, for an https address with a certificate you made yourself, *insecure*.
|
||||||
|
- **Required access:** read only, and a **superuser's token** is best: the system status (the worker, Redis, the index, the disk) is only given to a superuser. With an ordinary user's token the document counts and the failed imports are still read, and the page says the status is unknown. This app only ever sends GET
|
||||||
|
requests; **nothing is changed, and no failed task is dismissed**.
|
||||||
|
- **What it does:** every **five minutes** Paperless is read and stored — its version (and whether a newer one is out), the status of the **database, Redis, the worker that consumes documents, the search index, the classifier and the sanity check**, the **disk**, the **document and inbox counts** and the **imports that failed**
|
||||||
|
in the last week and that nobody has dismissed. **Apps → Paperless** shows it.
|
||||||
|
- **Alerts** (source *Paperless-ngx*; held back by maintenance on the integration): **the database, Redis or the worker in error** (critical — nothing can be consumed or found), **the index or the classifier in error** (warning), **the sanity check found problems** (warning), **documents failed to import** (warning; the files are named and the
|
||||||
|
latest error shown — it clears when you dismiss them in Paperless, or after a week), and **the disk under the documents is 90 % full** (warning; **95 %**, critical). A classifier that simply has not been trained yet (Paperless's *warning*) is not an alert.
|
||||||
|
- If Paperless cannot be read for 20 minutes its alerts are neither raised nor cleared, the Alerts page says it could not check, and the integration is raised as failing like any other. A part that could not be asked is *unknown*, never *none*.
|
||||||
|
- `/metrics` has `homelab_paperless_documents`, `homelab_paperless_inbox`, `homelab_paperless_failed_imports` and `homelab_paperless_component_ok` for each Paperless that was read lately.
|
||||||
|
- **Not verified against a live Paperless** — the endpoints (`/api/status/`, `/api/statistics/`, `/api/tasks/`, `/api/remote_version/`) and their fields are as Paperless-ngx publishes them and the reader is tried against a pretend server. The **Diagnostic Log** has what was asked and answered.
|
||||||
|
|
||||||
|
### Prometheus
|
||||||
|
|
||||||
|
- **Config fields:** the **address** of Prometheus (`http://host:9090`); optionally a **bearer token**, if something in front of Prometheus asks for one (stored encrypted); and, for an https address with a certificate you made yourself, *insecure*.
|
||||||
|
- **Required access:** read only, and nothing needs to be installed or configured in Prometheus. This app only ever sends GET requests: `/api/v1/status/buildinfo`, `/api/v1/targets` and `/api/v1/alerts`.
|
||||||
|
- **What it does:** every **two minutes** Prometheus is read and stored — its version, every active **scrape target** with whether its last scrape worked (and the error), and the **alerts its own rules have firing**. **Monitoring → Prometheus** shows targets by job, the ones not answering and the alerts. So whatever you already
|
||||||
|
measure there — a node exporter on each host, a service's own metrics — is part of the one list of what is wrong.
|
||||||
|
- **Alerts** (category *Monitoring*, source *Prometheus*; held back by maintenance on the integration): **targets not answering** — one alert for each *job*, naming the instances and the scrape error, and only once a target has been down for **five minutes** (one failed scrape is not news; the time is kept from one read to the next),
|
||||||
|
and **an alert that Prometheus's rules are firing** — one for each alert name, **critical** if any of them has the label `severity="critical"` (or `page`), a warning otherwise. Pending alerts, `info` ones and the by-design heartbeats (`Watchdog`, `DeadMansSwitch`, `InfoInhibitor`) say nothing.
|
||||||
|
- If Prometheus cannot be read for ten minutes its alerts are neither raised nor cleared, the Alerts page says it could not check, and the integration is raised as failing like any other. A list Prometheus did not answer (targets, or alerts) is *unknown*, never *none*.
|
||||||
|
- Nothing is sent back to `/metrics`: this app already exposes its own (see [PROMETHEUS-GRAFANA.md](docs/PROMETHEUS-GRAFANA.md)), and exporting what Prometheus said into Prometheus would only go round in a circle.
|
||||||
|
- **Not verified against a live Prometheus** — the endpoints and fields are as Prometheus documents them and the reader is tried against a pretend server. Alerts that **Grafana manages itself** (Grafana-managed alert rules) are not in Prometheus's list and are not seen; alerts that Prometheus's own rules send to Alertmanager are, while they are active.
|
||||||
|
|
||||||
|
### Authentik
|
||||||
|
|
||||||
|
- **Config fields:** the **address** of Authentik (`https://auth.example.lan`), an **API token** (Authentik → Directory → Tokens and App passwords → *Create*, intent **API Token**, for a user who may read; stored encrypted), optionally a **token for making and managing users** (below), for an https address with a certificate you made yourself, *insecure*, and a tick box, **do not alert about certificates**, for an Authentik you are not using at the moment: the certificate alerts stop (and any that were raised clear), while the page still lists the certificates and `/metrics` still has `homelab_authentik_certificate_expiry_days`. The other alerts (outposts, tokens, failed sign-ins, configuration errors) are not affected.
|
||||||
|
- **Required access:** read only. This app only ever sends GET requests: the version, the workers, the outposts and their health, the certificates, the tokens and the events log. **Nothing is ever changed.** Give the token a user with just enough to read these (an administrator, or a user in a role that may view them);
|
||||||
|
if one of the lists is refused, that part is left out and the rest still works.
|
||||||
|
- **Making and managing users (optional, administrators only):** all of it is on its own tab, **Apps → Authentik → Users and groups**, which only an administrator sees; **Status** (the default) stays what it was. with a second, separate token in *Token for making and managing users* — an API token of an account that may add, change and delete users, which is close to an administrator of Authentik, so give it only if you want this — **Apps → Authentik → Users and groups → New user** makes a user
|
||||||
|
(user name, name, an email if there is one, and groups, which are read from Authentik when the form opens; a group that makes members administrators of Authentik is marked) and then a **one-time recovery link** for them. They open the link and **choose their own password**: this app never sets, sees or keeps one.
|
||||||
|
The link is in the answer to that one request and nowhere else (not stored, not in the audit log); the audit log has *who made which user* and *who made a link for whom*, never the link or a token. If the user is made and the link is not (Authentik needs a **recovery flow** on the brand — System → Brands),
|
||||||
|
the page says so and the link is made in Authentik (the user → *Create recovery link*). Authentik builds the link from the address it was called at, so the integration's address should be the one people use. The token for making users is only used for this; reading Authentik never uses it,
|
||||||
|
and *Stop allowing this* on that tab removes it again (the reading token stays).
|
||||||
|
**Users** on the same tab, read when the tab opens, lists the people (service accounts — what an outpost or an integration signs in with — are left out), searchable, with whether each is on, whether they are an administrator and when they last signed in, and lets an administrator **edit a user's name, email and groups** (the user name is shown but not changeable: it is how a person is known elsewhere, and renaming could make them a new person here; Authentik is sent only what differs, as a `PATCH`, and groups as what goes in and what comes out on top of the groups the user has *now*, so a membership somebody gave them meanwhile is kept; a user whose groups Authentik does not say can be renamed but not moved between groups), **switch a user off** (a `PATCH` of `is_active` and nothing else;
|
||||||
|
they can no longer sign in, nothing of theirs is lost, *Switch on* brings it back) or **delete** one (a `DELETE`, for good: the user name has to be typed, and is checked again by the server against the user as Authentik has them now). **Refused, by the server, with the reason:** a service account, the account the token
|
||||||
|
itself belongs to, and the last active administrator (so nobody can be locked out, or the sign-in in front of everything else cut off — for an edit, only when it would take them out of every group that makes them one); do those in Authentik. Each is in the audit log (*who edited — which fields, never their new values — switched off / on / deleted which user*). Switching off or deleting someone in Authentik does not end a session they
|
||||||
|
already have in this app: end it under Users → *Signed in now*. Lists are read 200 at a time.
|
||||||
|
**Groups** on the same tab, also read when it opens, lists the groups with how many are in each and lets an administrator **make** one (a name and nothing else, a `POST /core/groups/` with `is_superuser` always false: it starts empty and is never a group that makes administrators; a name Authentik already has is its own refusal), **rename** one, **delete** one (a `DELETE`, for good — its members stay, just not in it: the group's name has to be typed and is checked again by the server against the group as Authentik has it now; **a group that makes its members administrators of Authentik is refused**, since deleting it would take that from all of them at once, possibly from the last administrator) and **put users in and take them out** (a `PATCH` of `name` and nothing else; `POST /core/groups/{id}/add_user/` and `remove_user/`, one user each,
|
||||||
|
so somebody who joined meanwhile is never dropped by a whole list replacing the members). Everything asked is checked first and **nothing is changed unless all of it may be**: a service account, the token's own account, the last active administrator
|
||||||
|
being taken out of the group that makes them one, and someone whose groups Authentik did not say (when taking them out of such a group) are refused with the reason. If Authentik fails part-way, the answer says how many of the changes had been made, and the audit log
|
||||||
|
(*who made, renamed or deleted which group — a deleted one with how many members it had and whether it decided a role here — and who was put in and taken out*) has what was done. Whether a group **makes its members administrators of Authentik** is shown and is **not** changed or set from here, and no group that makes administrators is deleted here either. A group whose name is one of the
|
||||||
|
*admin* or *operator* groups of **Settings → Sign-in** is marked *decides admins / operators here* and asks before it is saved: renaming it, or changing who is in it, changes people's roles in this app at their next sign-in. Making a group with such a name says so too (it is empty, so nothing changes until someone is put in it); deleting one asks first and says that people in it lose that role here at their next sign-in. Without it nothing about the page changes for anyone but an administrator, who sees where to add one. The calls (`GET /core/groups/`, `POST /core/users/`, `POST /core/users/{id}/recovery/`, `GET /core/users/`, `GET /core/users/{id}/`, `GET /core/users/me/`, `PATCH` (`is_active`, or `name`, `email` and `groups`) and `DELETE /core/users/{id}/`, `GET /core/groups/{id}/`, `POST /core/groups/`, `PATCH` and `DELETE /core/groups/{id}/` and `POST /core/groups/{id}/add_user/` and `remove_user/`; asking for a link that lasts a day, which a version of
|
||||||
|
Authentik that does not take that sets for itself) are **not verified against a live Authentik**; the tests use a pretend one.
|
||||||
|
- **What it does:** every **five minutes** Authentik is read and stored — its version (and whether a newer one is out), whether a **worker** is connected, each **outpost** (the proxy outpost that guards your apps, an LDAP one) with how many instances reported in during the last five minutes and whether they run the right version, the
|
||||||
|
**certificates** (those Authentik does not renew by itself) with their expiry, the **API tokens and app passwords** that expire, and how many **sign-ins failed** and **configuration errors** happened in the last day. **Apps → Authentik** shows it, with what runs out soonest first.
|
||||||
|
- **Alerts** (source *Authentik*; held back by maintenance on the integration): **an outpost that serves applications has not reported in** (critical — the apps behind it stop letting anyone in; an outpost with no applications is left alone), **an outpost runs another version** (warning),
|
||||||
|
**a certificate runs out** (warning within 14 days, critical within 3 and once it has), **an API token or app password runs out** (warning within 7 days, critical within 2 — the token this app uses to read Authentik is among them), **no worker is connected** (warning; mail, outpost changes and scheduled tasks stop),
|
||||||
|
**20 or more sign-ins failed in a day** (warning; shown as "100 or more" when the log's page is full) and **3 or more configuration errors in a day** (warning; a flow or stage that is set up wrongly, with the newest message).
|
||||||
|
- If Authentik cannot be read for 20 minutes its alerts are neither raised nor cleared, the Alerts page says it could not check, and the integration is raised as failing like any other. A part that could not be asked (no workers endpoint on an older version) is not a problem: only *none* is.
|
||||||
|
- `/metrics` has `homelab_authentik_outpost_up`, `homelab_authentik_workers`, `homelab_authentik_failed_logins_24h` and `homelab_authentik_certificate_expiry_days` for each Authentik that was read lately.
|
||||||
|
- **Checked against a live Authentik** (Inspect): the version, the outposts with their health, the certificates, the tokens and the events log all answered as the reader expects. **`/admin/workers/` answered HTTP 404 (a web page, not JSON)** on that Authentik — it has no such endpoint — so the number of workers is *unknown* there: the page leaves it out, `homelab_authentik_workers` is not written, and the *no worker is connected* alert is never raised on a guess (only a real count of 0 raises it). A token is listed only if it **expires** and is an API token or app password, so an Authentik whose tokens are all set not to expire shows none, which is right. Everything else is as Authentik publishes it and tried against a pretend server. Lists are read one page at a time
|
||||||
|
(200 items; 100 events), and the newest events come first. The **Diagnostic Log** has what was asked and answered. Flows that fail for one user are not counted — only the configuration errors Authentik itself logs; what the app sees of "failing flows" is that.
|
||||||
|
|
||||||
## DNS providers
|
## DNS providers
|
||||||
|
|
||||||
### Cloudflare
|
### Cloudflare
|
||||||
@@ -209,5 +656,12 @@ the dashboard views themselves load fine.
|
|||||||
|
|
||||||
- **Config fields:** Technitium URL, API Token
|
- **Config fields:** Technitium URL, API Token
|
||||||
- **Auth:** Bearer token
|
- **Auth:** Bearer token
|
||||||
- **Actions used:** list zones, read zone records, add/delete records (an "update" is implemented as delete + add)
|
- **Actions used:** list zones, **make a zone**, **edit a zone** (switch it off and on, its options), **delete a zone**, read zone records, add/delete records (an "update" is implemented as delete + add)
|
||||||
- **Required access:** A token tied to a Technitium user account with **Zones: View + Modify** permission — view-only will fail on record changes since updates are add/delete calls under the hood.
|
- **Making a zone** (DNS page → pick the provider → **New zone**, administrators only — it makes the server an authority for a name; it is the one DNS provider that can): a **Primary** zone (a domain name such as `home.lan`, or an address or network such as `192.168.1.0/24`, which Technitium turns into the reverse `…in-addr.arpa` zone), a **Secondary** zone (the servers that hold it, comma-separated) or a **Forwarder** zone (the DNS server to pass questions to — an address, a name, a URL or `this-server` — the protocol, UDP unless told, and whether to check DNSSEC, off unless told). It is `POST /api/zones/create` with the name and the type (and, for a Secondary, `primaryNameServerAddresses`; for a Forwarder, `initializeForwarder`, `forwarder`, `protocol` and `dnssecValidation`). The name is written lower case without a trailing dot and must be ASCII (a name with other letters in its `xn--` form); anything else is refused before Technitium is asked, and what Technitium says when it refuses (a zone that is there already) is shown as it was said. The new zone opens when it is made; a Primary zone has the records Technitium makes for it and the rest are added as usual. It is written in the audit log (who made which zone, of which kind).
|
||||||
|
- **Editing a zone** (DNS page → open the zone → **Edit zone**, administrators only; the server's own internal zones are never touched):
|
||||||
|
- **Switch off / on** — `POST /api/zones/disable` and `/api/zones/enable`. Off, the server stops answering for the zone; its records stay, and on brings it all back. A switched-off zone is marked *off* in the zone list, and the Technitium integration does not judge it.
|
||||||
|
- **Who may ask the zone** (`queryAccess`: everyone, nobody, only private networks, only the zone's name servers, only the networks listed, or the name servers and the networks listed), **who may copy it** (`zoneTransfer`, the same without the private-network choice, and the TSIG keys the server has that a transfer may be limited to) and **who is told when it changes** (`notify`: nobody, the zone's name servers, only the servers listed, or both). A choice that goes by a list needs it: networks are addresses or CIDR networks, one to a line and in order, with `!` in front to refuse one; servers are addresses. It is `GET /api/zones/options/get` (with the TSIG key names) and `POST /api/zones/options/set`.
|
||||||
|
- **What a kind of zone has:** a Primary or Secondary zone has all three; a Forwarder or Stub or Catalog zone only who may ask it, and a secondary catalog none; anything else is refused with the reason, before Technitium is asked. What is not changed is not sent: only the groups that differ go, each whole (its mode and the list it uses), because the documentation does not say what Technitium does with a setting that is left out of the call. **So afterwards the zone is read again, and anything else it says about the zone that moved (dynamic updates, its catalog …) is named on the page and in the audit log** — a warning, not an undo. Written in the audit log: who switched which zone off or on, and which groups of options were changed (never the lists). The options call needs the token's user to have **Zones: Modify** (and, as the documentation words it, **Zone: Delete** on that zone). **Not verified against a live install.**
|
||||||
|
- **Deleting a zone** (DNS page → open the zone → **Delete zone**, administrators only): the zone **and every record in it** go, for good — `POST /api/zones/delete` with the zone's name. The zone's name has to be typed, and the server checks it again: it must be the name the zone has now and the provider must list the zone, so a page that is minutes old, a wrong id, or one of the server's own internal zones (which are not listed) cannot be deleted by mistake; an id that is not a zone name as written (lower case, no trailing dot) is refused. What this app has cached of the zone and its records is dropped with it (the registration of the domain on the *Domain names* tab is not touched). What Technitium says when it refuses — a token that may not delete zones — is shown, and then nothing is changed here. It is written in the audit log (who deleted which zone, and how many of its records were cached). **Not verified against a live install**: the calls are as Technitium's API documentation has them.
|
||||||
|
- **Required access:** A token tied to a Technitium user account with **Zones: View + Modify** permission — view-only will fail on record changes (updates are add/delete calls under the hood) and on making a zone. **Deleting a zone** needs **Zones: Delete**, which is a permission of its own.
|
||||||
|
- **This is the DNS provider** (zones and records, on the DNS page). How the *server* is doing — queries, blocking, zone health, a real DNS test, alerts — is a separate integration with its own token: see [Technitium DNS](#technitium-dns) under the integrations above.
|
||||||
+262
-15
@@ -2,11 +2,18 @@
|
|||||||
|
|
||||||
Every notification in Homelab Manager is sent through the same pipe
|
Every notification in Homelab Manager is sent through the same pipe
|
||||||
(`server/src/services/notify.ts`) to whichever channels are enabled under
|
(`server/src/services/notify.ts`) to whichever channels are enabled under
|
||||||
**Settings → Notifications**: Gotify, ntfy, SMTP (email), and a generic
|
**Settings → Notifications**: Gotify, ntfy, Telegram, Pushover, SMTP (email), a generic
|
||||||
JSON webhook. All four fire for every notification below — there's no
|
JSON webhook, and push to phones and browsers. All seven fire for every notification below unless a
|
||||||
per-event channel routing, only a per-event on/off toggle (the "Notify
|
**routing rule** (see *Who hears what* below) says otherwise; each event also has its on/off toggle and, for the
|
||||||
on" list on that same page) and, for the daily ones, one shared
|
daily ones, one shared time/timezone.
|
||||||
time/timezone.
|
|
||||||
|
The settings page has four tabs, with one Save button for all of them (it
|
||||||
|
shows "Unsaved changes" until you save, and the browser asks before the page
|
||||||
|
is closed with changes unsaved): **Channels** (the seven channels, each a row
|
||||||
|
you open to fill in and send a test), **What to send** (the event toggles
|
||||||
|
grouped by topic, each group's thresholds beside its switches), **Who hears what** (routing rules and escalation) and **When**
|
||||||
|
(the time zone, the daily reminder time, quiet hours and the weekly digest).
|
||||||
|
The address keeps the tab, e.g. `/settings/notifications?tab=when`.
|
||||||
|
|
||||||
Two settings apply to *every* notification regardless of what triggered
|
Two settings apply to *every* notification regardless of what triggered
|
||||||
it:
|
it:
|
||||||
@@ -16,7 +23,11 @@ it:
|
|||||||
instead of sent immediately, then delivered as a single digest at the
|
instead of sent immediately, then delivered as a single digest at the
|
||||||
window's end time. This matters most for the real-time alerts below —
|
window's end time. This matters most for the real-time alerts below —
|
||||||
the daily reminders already fire once at a time you pick, usually
|
the daily reminders already fire once at a time you pick, usually
|
||||||
outside the window anyway.
|
outside the window anyway. *Still send critical alerts* (off by default)
|
||||||
|
lets a **critical** one — a server down, a disk full, mains power lost —
|
||||||
|
through at once, so only the warnings wait for the morning.
|
||||||
|
- **Away** (Settings → Notifications → When, and a banner on every page while
|
||||||
|
it is on): "I'm away until Sunday". See [Away](#away).
|
||||||
- **Maintenance windows** (the Maintenance page): opening one for a
|
- **Maintenance windows** (the Maintenance page): opening one for a
|
||||||
server, integration, or DNS provider silences the alerts that target
|
server, integration, or DNS provider silences the alerts that target
|
||||||
it specifically (offline/disk-full for a server; storage, backup, and
|
it specifically (offline/disk-full for a server; storage, backup, and
|
||||||
@@ -27,6 +38,94 @@ it:
|
|||||||
Where a trigger has a configurable threshold, that's noted in its row
|
Where a trigger has a configurable threshold, that's noted in its row
|
||||||
below; several are fixed and can't be changed from the UI.
|
below; several are fixed and can't be changed from the UI.
|
||||||
|
|
||||||
|
## Notification history
|
||||||
|
|
||||||
|
**Records → Notifications** (operators and administrators; also linked from Settings → Notifications) lists everything that was sent, newest first, and answers "why did I not get it?". Each entry says what it said and how it went through **each channel** — a ✓ or ✕ beside Gotify, ntfy, Telegram, Pushover, Email, Webhook or Push, with what the channel answered when it refused — and one of these:
|
||||||
|
|
||||||
|
| Status | Meaning |
|
||||||
|
|---|---|
|
||||||
|
| **Sent** | Every channel it went to took it |
|
||||||
|
| **Partly sent** | Some took it, some refused or could not be reached |
|
||||||
|
| **Failed** | No channel took it |
|
||||||
|
| **Nowhere to send it** | No channel was switched on for it |
|
||||||
|
| **Held** | Quiet hours or away mode held it for the digest (the digest is its own entry) |
|
||||||
|
| **Not sent (rule)** | A routing rule says nobody should be told about that kind |
|
||||||
|
|
||||||
|
Filter by status or search the title and message. A notification about alerts also carries **which alerts it was about**, so each alert's **Timeline** (Alerts → History → the incident) says who was told and when, with the same ✓/✕ detail — including a message that was held or not sent. (A held message is listed against its alerts when it was held; the digest that later carries it is not.) **Send again** (**Send now** for a held one) sends the entry once more to every channel that is on, writes that down as a new entry and is in the change log; it does not carry the buttons an alert message had. Messages are kept 30 days (at most 1000 entries), cut at 4000 characters, and an error is cleaned of every token, key and password in the settings before it is kept, since a channel may say them back. The *Send test* buttons in Settings are not listed.
|
||||||
|
|
||||||
|
## Who hears what
|
||||||
|
|
||||||
|
Under **Settings → Notifications → Who hears what**.
|
||||||
|
|
||||||
|
**Routing rules.** By default every notification goes to every channel that is switched on. A rule narrows that for what it matches, on any of:
|
||||||
|
the **kind** of problem (server down, disk and storage, failing services, backups, updates, expiry, automation, integration failing …), how
|
||||||
|
**bad** it is (warning or worse, critical only) and the **tags** of the server it is about (any of them). **The first rule that matches decides** which
|
||||||
|
channels get it — put the specific rules first — and a rule with no channels tells nobody (the Alerts page still shows the problem, and webhooks for
|
||||||
|
events still fire). A notification no rule matches goes to every channel that is on, as before. Rules can be switched off and reordered.
|
||||||
|
|
||||||
|
- A message that holds problems for different routes is split: a disk warning can go to ntfy while the server outage in the same check goes to
|
||||||
|
everything, as two messages. When all of them take the same route it is one message, as ever.
|
||||||
|
- A recovery follows the route its alert took (the health check remembers how bad each problem was).
|
||||||
|
- What a rule can see: health and automation problems carry a kind, a severity and (for servers) the server; secret, domain and Tailscale expiry, backup
|
||||||
|
checks, Docker image updates, update runs and integration failures carry a kind and a severity. DNS changes, the weekly digest and similar carry
|
||||||
|
neither, so only rules that ask about nothing in particular match them.
|
||||||
|
- Notifications held for quiet hours keep who they were for: the digest at the end goes only to the channels their rule picked.
|
||||||
|
|
||||||
|
**Each browser's own wishes.** Under *Phone and browser*, every subscribed browser can say it only wants warnings or worse, or only critical alerts,
|
||||||
|
and only some kinds of alert — a phone for the critical things, a desktop for everything. It applies after routing: a rule that sends something to
|
||||||
|
*Phone and browser* still doesn't reach a browser that turned that kind down. A notification that says nothing about its kind or severity reaches all.
|
||||||
|
|
||||||
|
**Escalation.** When an alert has been raised for a while and **nobody has acknowledged it**, it is told again, in up to three steps — for example *after
|
||||||
|
30 minutes: push; after 2 hours: email the neighbour who holds the key*. Each step names its time, the least severity it applies to, the channels to use
|
||||||
|
(chosen freely — routing rules don't apply to escalations), an optional extra email address to tell as well (sent through the email channel's server) and
|
||||||
|
whether to send during quiet hours and while away (otherwise it waits for them to end like any other). Every step is sent once per alert; an acknowledgement stops the
|
||||||
|
rest, and the alert starts over if it goes away and comes back. Problems under maintenance, held back because something they depend on is down, or only
|
||||||
|
informational never escalate. It is looked at every five minutes, so a step can arrive a few minutes after its time. Every escalation is written in the audit
|
||||||
|
log (category *alert*, action *escalate*).
|
||||||
|
|
||||||
|
## Endpoint checks and restore tests
|
||||||
|
|
||||||
|
An **endpoint check** that fails several times in a row (the check's own number, 2 by default) is announced like any other server problem — under the *server offline,
|
||||||
|
disk full …* switch (`healthAlerts`) — once when it starts, with a recovery message when it answers again; a certificate that is within its warning time is announced
|
||||||
|
the same way. For routing they count as *Monitoring* (a check that fails) and *Expiry* (a certificate), and carry the tags of the server the check is linked to; maintenance on that
|
||||||
|
server holds them back. A **restore test** that fails, or one that is overdue (when a limit is set), is on the Alerts page and in the incident record but is not sent as a
|
||||||
|
notification of its own — it is something to look at when you are looking at backups.
|
||||||
|
|
||||||
|
## Telegram and Pushover
|
||||||
|
|
||||||
|
Two more ways to a phone, for those who already live in one of them. Both are services with one address each (`api.telegram.org`, `api.pushover.net`), so nothing typed into the settings decides where a message is sent. Each has its own *Send test* in
|
||||||
|
**Settings → Notifications → Channels**, which sends with what is typed there (saved or not) and shows what the service said if it refuses.
|
||||||
|
|
||||||
|
**Telegram** sends through a bot of your own, to one chat. Make the bot by messaging [@BotFather](https://t.me/BotFather) (`/newbot`) and paste its token; the chat id is a number (a group's is negative) or `@yourchannel` — send the bot a message (or add it to the
|
||||||
|
group or channel and post there), then open `https://api.telegram.org/bot<token>/getUpdates` and read `"chat":{"id":…}`. A bot cannot message a person who has not written to it first. The message is the title, a blank line and the text, as plain text
|
||||||
|
(nothing in it is read as formatting), cut to Telegram's 4096-character limit. *Send silently* sends without a sound or a buzz.
|
||||||
|
|
||||||
|
**Pushover** needs an application token (pushover.net → *Create an Application/API Token*) and your user key (or a group's), both from your Pushover account; a device name limits it to one of your devices. **Priority** runs from *lowest* (no
|
||||||
|
notification) to *emergency*, which repeats every minute for an hour until it is acknowledged in the Pushover app. The message is cut to Pushover's 1024 characters and the title to 250.
|
||||||
|
|
||||||
|
The token and the keys are stored encrypted with the same key as the other credentials, and are never put in an error message or the audit log (a change to them is recorded as the field's name). A message that is refused is logged with what the service
|
||||||
|
said; it never stops the other channels. Both take part in routing rules like any channel.
|
||||||
|
|
||||||
|
## Push to a phone or browser
|
||||||
|
|
||||||
|
The fifth channel, **Phone and browser**, sends the same messages as the others to browsers that have asked for them — a phone with
|
||||||
|
this page added to its home screen, a desktop browser — with no app, account or extra service. An administrator switches it on under
|
||||||
|
Settings → Notifications → Channels (the optional *contact* is given to the browsers' push services, as the standard asks), and then
|
||||||
|
each person opens the same row and presses **Notify this browser**; the browser asks permission and the row lists their browsers,
|
||||||
|
with a button to send a test and one to remove each.
|
||||||
|
|
||||||
|
- **It needs https.** Browsers only allow push for pages served over https (or from `localhost`), so a copy opened over plain http
|
||||||
|
on a LAN address shows why it can't be used instead of the button.
|
||||||
|
- **How it travels.** The message is encrypted for each browser (RFC 8291) and signed with a key pair this server makes the first time
|
||||||
|
it is needed (RFC 8292, "VAPID"), then handed to the browser's own push service — Google's, Mozilla's or Apple's — which delivers
|
||||||
|
it. The push service sees that a message is being sent and its size, not what it says. The address of each browser's push
|
||||||
|
service is checked before use: https only, and never an address on a private network.
|
||||||
|
- **Browsers' own wishes** (a minimum severity, which kinds) are set per browser under the channel's row.
|
||||||
|
- **Quiet hours** hold push messages like every other channel, and each event's on/off toggle applies. A tap on a notification opens
|
||||||
|
the Alerts page.
|
||||||
|
- **A browser that has gone** (the site's permission was removed, the app uninstalled) is dropped automatically when its push
|
||||||
|
service says so. If the server's key pair is ever lost (the credentials key changed), everyone has to press the button again.
|
||||||
|
|
||||||
## Daily reminders
|
## Daily reminders
|
||||||
|
|
||||||
These six checks share one schedule: **Settings → Notifications → Daily
|
These six checks share one schedule: **Settings → Notifications → Daily
|
||||||
@@ -45,7 +144,11 @@ until the next scheduled time to catch up.
|
|||||||
| Secret expiry | *Secret expiry reminder* | Any tracked secret (API token, SSL cert, password, generic) is expired or within its own configured warning window. SSL certificates with a host:port are re-checked live first, so this reflects the real current expiry, not a stale saved date. A certificate that couldn't be re-checked live is mentioned separately, so the expiry shown may be stale. | Per-secret warning threshold, set when adding/editing that secret |
|
| Secret expiry | *Secret expiry reminder* | Any tracked secret (API token, SSL cert, password, generic) is expired or within its own configured warning window. SSL certificates with a host:port are re-checked live first, so this reflects the real current expiry, not a stale saved date. A certificate that couldn't be re-checked live is mentioned separately, so the expiry shown may be stale. | Per-secret warning threshold, set when adding/editing that secret |
|
||||||
| Domain expiry | *Domain registration expiring or expired (daily reminder)* | Any tracked domain registration is expired or within the warning window; every domain is re-looked-up against its registry first. A domain whose registry lookup has been failing for 3+ days is also mentioned, separately, as "may be stale." Registries that don't publish an expiry (e.g. `.de`, `.eu`) are tracked but never trigger this. | Settings → Notifications → Health checks → **Domain expiry warning** (default 30 days) |
|
| Domain expiry | *Domain registration expiring or expired (daily reminder)* | Any tracked domain registration is expired or within the warning window; every domain is re-looked-up against its registry first. A domain whose registry lookup has been failing for 3+ days is also mentioned, separately, as "may be stale." Registries that don't publish an expiry (e.g. `.de`, `.eu`) are tracked but never trigger this. | Settings → Notifications → Health checks → **Domain expiry warning** (default 30 days) |
|
||||||
| Tailscale key expiry | *Tailscale key expiry reminder* | Any device's node key (across every enabled Tailscale integration) is within 30 days of expiring, or already expired. Devices with key expiry disabled are skipped. | Fixed at 30 days |
|
| Tailscale key expiry | *Tailscale key expiry reminder* | Any device's node key (across every enabled Tailscale integration) is within 30 days of expiring, or already expired. Devices with key expiry disabled are skipped. | Fixed at 30 days |
|
||||||
|
| Docker containers broken | *Warn when a Docker container is broken* | A container keeps restarting, is unhealthy, stopped unexpectedly or never started, or a Docker host cannot be reached, as Dockhand reports on the 15-minute pass — announced once with the server problems (*Server offline, disk nearly full …*, `healthAlerts`), with a recovery message when it runs again. See [MONITORING.md](docs/MONITORING.md#docker-containers-that-are-broken). | Its own switch; *Containers to leave alone* |
|
||||||
| Docker image updates | *Docker image update available* | Any container (across every enabled Dockhand integration) has an image update available, per Dockhand's own cached update-check results — this doesn't trigger a fresh registry lookup, just reads what the Docker page itself would show. | Not configurable (reflects Dockhand's own check interval) |
|
| Docker image updates | *Docker image update available* | Any container (across every enabled Dockhand integration) has an image update available, per Dockhand's own cached update-check results — this doesn't trigger a fresh registry lookup, just reads what the Docker page itself would show. | Not configurable (reflects Dockhand's own check interval) |
|
||||||
|
| Proxmox backups — gone stale | *(same toggle as the Proxmox backup check)* | Once a day with the other backup messages: a guest whose newest backup (on Proxmox or a Backup Server) is older than the limit. Guests under maintenance (on the server or the Proxmox integration) are left out. | Settings → Notifications → **A guest's backup is too old after** (default 48 h; 0 off); a server can have its own limit on its page |
|
||||||
|
| Game server turned on or off | *A game server (Fragnet) was turned on or off* | A read of a Fragnet account shows a game server **on** (running or starting) when the one before showed it off, or **off** (stopped or stopping) when it showed it on — however it came about: someone pressed Start or Stop, the game crashed, the host stopped it. One message for what changed in one read, a line for each server (name, game, account); *starting* counts as on, so a start is said once. A server met for the first time, one whose state could not be read and one being installed or moved say nothing. Goes through the routing rules as a *services* notification and waits for quiet hours like any other that is not critical; the accounts are read every five minutes, so it can be that late. On unless switched off. | Settings → Notifications → What to send |
|
||||||
|
| Update run finished | *An update run you started finished, or didn't* | An update run started from the Updates page ended: what is still waiting and whether a restart is due (or, for a run that restarts the server afterwards, that it was restarted and is back), or why it failed. A restart started on its own says "restarted" or "Restart of … didn't finish". Sent at once (not held for the daily check). | Settings → Notifications → What to send |
|
||||||
| Proxmox backups | *Proxmox backup failed or a guest has no coverage* | Two independent conditions, both under this one toggle: (1) a node's most recent `vzdump` backup task (across every enabled Proxmox integration) didn't succeed; (2) a VM/LXC isn't covered by any enabled backup job at all. | Not configurable |
|
| Proxmox backups | *Proxmox backup failed or a guest has no coverage* | Two independent conditions, both under this one toggle: (1) a node's most recent `vzdump` backup task (across every enabled Proxmox integration) didn't succeed; (2) a VM/LXC isn't covered by any enabled backup job at all. | Not configurable |
|
||||||
| Proxmox Backup Server verification | *Proxmox Backup Server snapshot failed verification* | Across every enabled PBS integration: any datastore has at least one stored snapshot whose verification state is "failed", or a datastore couldn't be read at all (e.g. a permissions problem). This is distinct from the Proxmox check above — PVE only knows a backup *ran*, PBS is the only place that knows whether the stored data still verifies. | Not configurable |
|
| Proxmox Backup Server verification | *Proxmox Backup Server snapshot failed verification* | Across every enabled PBS integration: any datastore has at least one stored snapshot whose verification state is "failed", or a datastore couldn't be read at all (e.g. a permissions problem). This is distinct from the Proxmox check above — PVE only knows a backup *ran*, PBS is the only place that knows whether the stored data still verifies. | Not configurable |
|
||||||
|
|
||||||
@@ -66,6 +169,7 @@ starting baseline rather than announced all at once.
|
|||||||
| Check | "Notify on" toggle | Fires when | Configured at |
|
| Check | "Notify on" toggle | Fires when | Configured at |
|
||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
| Health — server offline | *Server offline, disk nearly full, or Synology volume/disk problem* | A server's agent hasn't reported for longer than the offline threshold. Held for the first 20 minutes after this app restarts, since agents haven't had a chance to report yet. | Settings → Notifications → Health checks → **Server offline after** (default 60 min) |
|
| Health — server offline | *Server offline, disk nearly full, or Synology volume/disk problem* | A server's agent hasn't reported for longer than the offline threshold. Held for the first 20 minutes after this app restarts, since agents haven't had a chance to report yet. | Settings → Notifications → Health checks → **Server offline after** (default 60 min) |
|
||||||
|
| Health — host health | *(same toggle as the others in this group)* | A Linux agent says a service has failed, a disk's SMART check fails (or it has used 90% or more of its rated life), or a ZFS pool is not ONLINE. A server that is currently offline isn't also judged on these (its figures are stale). Silenced by a maintenance window on the server like the others. | The same health-check settings; there is no separate threshold. |
|
||||||
| Health — disk usage | *(same toggle)* | A server disk, a Proxmox node's root filesystem or any of its storages, or a Synology volume, is at or above the usage threshold. A server that's currently offline isn't also judged on disk usage (its figures are stale). | Settings → Notifications → Health checks → **Disk usage alert at** (default 90%) |
|
| Health — disk usage | *(same toggle)* | A server disk, a Proxmox node's root filesystem or any of its storages, or a Synology volume, is at or above the usage threshold. A server that's currently offline isn't also judged on disk usage (its figures are stale). | Settings → Notifications → Health checks → **Disk usage alert at** (default 90%) |
|
||||||
| Health — Synology status | *(same toggle)* | A Synology volume's status isn't "normal", or a disk's status/SMART result isn't normal, or it's over the bad-sector or under the remaining-life threshold. | Not configurable |
|
| Health — Synology status | *(same toggle)* | A Synology volume's status isn't "normal", or a disk's status/SMART result isn't normal, or it's over the bad-sector or under the remaining-life threshold. | Not configurable |
|
||||||
| Automation — failed run | *Semaphore template or Gitea workflow run failed* | A Semaphore template's, or a Gitea repo's, most recent run status is a clear failure. A run that's still going, was cancelled, or was stopped by hand doesn't count as failed or as a recovery — it's left exactly as it was. For Gitea this follows the repo's most recent run on any workflow or branch. | Not configurable |
|
| Automation — failed run | *Semaphore template or Gitea workflow run failed* | A Semaphore template's, or a Gitea repo's, most recent run status is a clear failure. A run that's still going, was cancelled, or was stopped by hand doesn't count as failed or as a recovery — it's left exactly as it was. For Gitea this follows the repo's most recent run on any workflow or branch. | Not configurable |
|
||||||
@@ -80,7 +184,7 @@ schedule.
|
|||||||
| DNS record added | *DNS record added* | A DNS record is created against any DNS provider through this app (manually, or via any automated action that writes one). |
|
| DNS record added | *DNS record added* | A DNS record is created against any DNS provider through this app (manually, or via any automated action that writes one). |
|
||||||
| DNS record updated | *DNS record updated* | An existing DNS record is edited. |
|
| DNS record updated | *DNS record updated* | An existing DNS record is edited. |
|
||||||
| DNS record deleted | *DNS record deleted* | A DNS record is deleted. |
|
| DNS record deleted | *DNS record deleted* | A DNS record is deleted. |
|
||||||
| Integration/DNS provider down | *Integration/DNS provider failing repeatedly* | Any integration or DNS provider adapter's calls fail a configurable number of times **in a row**. Tracked per adapter *type* (e.g. "proxmox"), not per individual integration row — with two Proxmox integrations, a streak of failures on either one counts toward the same total. Only fires once per failure streak (not on every failure past the threshold), and is skipped entirely if that type is currently in a maintenance window. | Settings → Notifications → **Alert after** (default 3 consecutive failures) |
|
| Integration/DNS provider down | *Integration/DNS provider failing repeatedly* | Any integration or DNS provider adapter's calls fail a configurable number of times **in a row**. Tracked per adapter *type* (e.g. "proxmox"), not per individual integration row — with two Proxmox integrations, a streak of failures on either one counts toward the same total. Only fires once per failure streak (not on every failure past the threshold), and is skipped entirely if that type is currently in a maintenance window, **or if it is explained by something it needs that is down** (see *Dependencies* in the README): a service that starts failing while the server it runs on is offline is not announced, and if it is still failing once that server is back, the next failed call alerts. | Settings → Notifications → **Alert after** (default 3 consecutive failures) |
|
||||||
| Integration/DNS provider recovered | *(same toggle)* | The next call for that source succeeds, after a "down" alert was already sent for the current streak. |
|
| Integration/DNS provider recovered | *(same toggle)* | The next call for that source succeeds, after a "down" alert was already sent for the current streak. |
|
||||||
|
|
||||||
## Digest
|
## Digest
|
||||||
@@ -89,28 +193,165 @@ schedule.
|
|||||||
|---|---|
|
|---|---|
|
||||||
| "Notifications from quiet hours" | Once, at quiet hours' configured end time, **only if enabled and only if at least one notification was held** during the window. Bundles every held notification's title and message into one message, then clears the queue. |
|
| "Notifications from quiet hours" | Once, at quiet hours' configured end time, **only if enabled and only if at least one notification was held** during the window. Bundles every held notification's title and message into one message, then clears the queue. |
|
||||||
|
|
||||||
|
## Backups of Homelab Manager itself
|
||||||
|
|
||||||
|
A scheduled backup of the app's own database that fails sends one message through the channels ("Homelab Manager — its own backup failed", with the reason), under the same toggle as server and disk alerts (*Server offline, disk nearly full…*). The same problem is also an alert on the Alerts page of the Backups kind — "the newest backup of Homelab Manager's own database is … old" once it is more than two days old, critical after a week, and "no backup of its own yet" with the reason when the very first one fails. No alert is raised for a new install that hasn't tried yet, or when daily backups are switched off. See [BACKUP.md](BACKUP.md).
|
||||||
|
|
||||||
|
## Monthly report
|
||||||
|
|
||||||
|
| Notification | Fires when |
|
||||||
|
|---|---|
|
||||||
|
| "Homelab monthly report — <month>" | On the chosen day of the month (1–28) and time, in the notification time zone, **only if switched on** under Settings → Notifications. One plain-text message through every channel that is on, about the month just over; quiet hours hold it like any other notification. |
|
||||||
|
|
||||||
|
Availability of servers and endpoints, incidents (how many, how long they took, the longest), updates and patch plan runs, restore tests and the state of the backups, and what was renewed — a topic only where there
|
||||||
|
is something to say, now with a **Service levels** topic (how each target did against its error budget). The same report is on the Reports page. *Preview last month* and *Send last month now* are on the settings page.
|
||||||
|
|
||||||
|
With **Also send a quarterly report** on, "Homelab quarterly report — Q3 2026" follows the monthly one after the month that ends a quarter (March, June, September, December), and with **Also send an annual report** on, "Homelab annual report — 2026" follows
|
||||||
|
the December one. Both are the same report over three or twelve months, sent the same way; both are off until switched on.
|
||||||
|
|
||||||
|
## Service levels
|
||||||
|
|
||||||
|
A service-level target (Alert settings → Service levels) is announced like the other server problems — under *Server offline, disk nearly full …* (`healthAlerts`), once, with a recovery message when it clears — when **80 %** of the month's error budget is used
|
||||||
|
(a warning: "Gitea: 90% of the error budget for its 99.9% target is used up for October 2026 — down 40 min of 45 min allowed, 5 min left") and again when it is **overspent** (critical: "Gitea has missed its 99.9% target for October 2026: down 50 min, 45 min allowed").
|
||||||
|
It is *Monitoring* for routing, and it ends when the month turns over. Maintenance does not hold it back: planned downtime counts. Each target has its own switch.
|
||||||
|
|
||||||
|
## Network devices
|
||||||
|
|
||||||
|
A switch, access point or gateway that a UniFi controller has not seen online for longer than the limit (Settings → Notifications → *Servers and disks*, 10 minutes) is announced like a server that stopped reporting, under
|
||||||
|
*Server offline, disk nearly full …* (`healthAlerts`): "Hallway AP (access point, U6 Pro) has been offline for 45 min", with a recovery message when it is back. For routing it counts as *Server down*; maintenance on the
|
||||||
|
device holds it back; and what depends on it (or it is connected through, if that is down too) is folded under it and named in its message ("— also takes down …"). The controller is read every five minutes; if it cannot be
|
||||||
|
read for twenty, its devices are held — neither announced nor cleared.
|
||||||
|
|
||||||
|
## Your own alert rules, and snoozing
|
||||||
|
|
||||||
|
An alert rule (Alert settings → Alert rules) that is past its limit is announced like any other health alert (*Health Alert*, with a recovery message), counted as **Monitoring** for routing with the rule's own severity, and held
|
||||||
|
back by maintenance on the server. It is checked on the health pass, every 15 minutes.
|
||||||
|
|
||||||
|
An alert that is **snoozed** (Alerts page → *Snooze…*) is not announced or recovered by the health and automation checks until the snooze ends. When it ends and the alert is still there, one **"Snooze ended"** message
|
||||||
|
(⏰) says so, through the usual routing and quiet hours; if the alert went away, nothing is sent. Event webhooks still receive every raise and recovery.
|
||||||
|
|
||||||
|
## Buttons on notifications
|
||||||
|
|
||||||
|
Switched on under **Settings → Notifications → Who hears what → Buttons on notifications** (off by default), a notification about alerts that have a key — the health and automation problems, a flapping alert, a snooze
|
||||||
|
that ended, and the escalation reminder — can be acted on from the message:
|
||||||
|
|
||||||
|
| Channel | One alert | Several |
|
||||||
|
|---|---|---|
|
||||||
|
| **ntfy** | Three buttons (*Acknowledge*, *Silence 1 h* — only when the alert is about something a window can silence — and *Snooze*) that POST to the alert's own link, and a tap that opens the page | One button, *Choose what to do*, to a page that lists them |
|
||||||
|
| **Phone and browser push** | Two buttons on the notification itself (*Acknowledge*, *Silence 1 h*); a press sends it and shows how it went. A tap opens the Alerts page | A tap opens the Alerts page |
|
||||||
|
| **Telegram** | One button under the message, *Acknowledge, silence or snooze*, that opens the page. (If Telegram will not take the button's address — one on a private network, say — the link is added to the text instead) | The same |
|
||||||
|
| **Pushover** | The link is attached to the message (*Acknowledge, silence or snooze*), and a tap opens the page | The same |
|
||||||
|
| **Gotify** | The link is added to the text, and a tap opens the page | The same |
|
||||||
|
| **Email**, **the chat webhook**, an escalation's extra address | The link, as a last line of the text | The same |
|
||||||
|
|
||||||
|
Each link is a **signed token** for the alerts in that message only, valid for **24 hours**, for three actions: acknowledge, silence the thing the alert is about for an hour, snooze until tomorrow 09:00 (in the
|
||||||
|
notification time zone). Opening it shows a page that says what each button would do and changes nothing; only a button press does anything. What was done is in the audit log, *from a notification link*, with the address
|
||||||
|
it came from. **Anyone who can read the notification can use it** for those 24 hours. A message held for quiet hours is sent without buttons; recovery messages never have them.
|
||||||
|
|
||||||
|
## Away
|
||||||
|
|
||||||
|
For a stretch rather than every night. **Settings → Notifications → When → Away** takes the time you will be back (with shortcuts: tomorrow evening, Sunday evening, a week, two weeks), optionally a start later than now (to plan a
|
||||||
|
holiday ahead), a note to yourself and whether to **still send critical alerts**. While it lasts:
|
||||||
|
|
||||||
|
- notifications are **held** — the same queue quiet hours use — and a banner on every page says that someone is away and how many are held;
|
||||||
|
- **critical** alerts still go out at once, if that box is ticked (it is by default); with it off, nothing does;
|
||||||
|
- at the time you gave, or when you press **I'm back**, what was held is sent as **one digest** — "Homelab Manager — Notifications from while you were away" — to the channels each was meant for, and the away stretch is cleared. It is looked at every
|
||||||
|
minute, and also right after the app restarts, so it ends within a minute of its time;
|
||||||
|
- an **escalation** step that says *send even during quiet hours* is sent while away too — that is what it is for, for example an email to the neighbour who holds the key;
|
||||||
|
- if quiet hours are on at that moment, the digest waits for them to end; if you come back in the middle of an away stretch's quiet hours nothing is lost.
|
||||||
|
|
||||||
|
What it does **not** do: it does not stop the alerts themselves — they are on the Alerts page, in the event webhooks and in `/metrics` as ever, so a Home Assistant or Grafana setup still sees everything; it only holds the *messages to you*. Away mode is an
|
||||||
|
administrator's to set (it decides who is told what), everyone sees the banner, and starting, changing and ending it are in the audit log and the Change log. Times typed there are the clock of the screen they are typed on.
|
||||||
|
|
||||||
|
## Flapping alerts
|
||||||
|
|
||||||
|
A health alert that is raised, clears and is raised again is announced the first times as usual. When the same alert has been raised **three times within an hour** (Settings → Notifications → *Servers and disks*; 0 = off) it is *flapping*:
|
||||||
|
|
||||||
|
- one message, **"Flapping alert"**, says "↻ <the alert> — raised 3 times within an hour";
|
||||||
|
- its raise and recovery messages are then **held back** (the alert stays on the Alerts page with a *flapping* badge and still counts for the menu badge and /metrics);
|
||||||
|
- once it has been calm for an hour, one **"Alert settled"** message says whether it is "⚠ still active" or "✓ cleared", and messages for it are sent as usual again.
|
||||||
|
|
||||||
|
This follows the routing rules and quiet hours like other health messages, and is checked every five minutes. **Event webhooks still receive every raise and recovery** — only the messages to people are held. The *Alerts → Noise* tab lists the alerts that come back most often.
|
||||||
|
|
||||||
|
## Patch plans and new ports
|
||||||
|
|
||||||
|
A **patch plan** run sends one message when it ends — "Patch plan "Lab nights" finished" or "didn't finish cleanly" — with a line for each server, under the *update run results* switch (the same one as a single update run's message). A server
|
||||||
|
that begins **listening on a new TCP port** is announced like the other server problems — under *Server offline, disk nearly full …* (`healthAlerts`), once, with a recovery message when the port is noted or stops listening; for routing it counts as *Monitoring*,
|
||||||
|
carries the tags of the server, and maintenance on the server holds it back.
|
||||||
|
|
||||||
|
## Unusual for the time
|
||||||
|
|
||||||
|
A server whose CPU load or memory use over the last hour is clearly higher than is normal for that hour of that weekday (see [MONITORING.md](docs/MONITORING.md#what-is-normal-for-the-time)) is announced like the other server problems — under *Server offline, disk
|
||||||
|
nearly full …* (`healthAlerts`), once, with a recovery message when it is back toward normal. For routing it counts as *Monitoring*, carries the server's tags, and maintenance on the server holds it back. It has its own switch under
|
||||||
|
Settings → Notifications (*Servers and disks*), on by default; with no four weeks of history for the hour there is nothing to announce.
|
||||||
|
|
||||||
|
## Weekly digest
|
||||||
|
|
||||||
|
| Notification | Fires when |
|
||||||
|
|---|---|
|
||||||
|
| "Homelab weekly digest" | Once a week, on the day and at the time chosen under Settings → Notifications (in the notification time zone), **only if switched on** — it is off until then. One plain-text message through every channel that is on; quiet hours hold it like any other notification. |
|
||||||
|
|
||||||
|
It gathers what the app already knows into a few lines per topic, and leaves out any topic with nothing to say:
|
||||||
|
what is wrong **right now**; **uptime** over the last seven days (what was down and for how long, with the percentage);
|
||||||
|
**what happened** (servers that stopped reporting and came back, services that stopped answering, maintenance);
|
||||||
|
**incidents** (how many began and ended, still open, the longest); **disks heading for full**; secrets and domains that are **expiring** or have expired (and hardware whose warranty ends within 60 days); guests that are **not backed up**, have a failed or stale backup; **updates waiting** (how many
|
||||||
|
servers, how many security fixes, who needs a restart, the longest wait); and **planned maintenance** in the next seven
|
||||||
|
days. A week with nothing in it says "A quiet week: nothing needs attention." On the settings page, *Preview* shows
|
||||||
|
what it would say right now and *Send it now* sends it at once (through the channels that are on, whatever the time).
|
||||||
|
|
||||||
## The Alerts page
|
## The Alerts page
|
||||||
|
|
||||||
**Operations → Alerts** shows what these notifications are about — the problems that exist *right now* — as a list you can look at, filter, and
|
**Alerts** (the *Now* view; *History* lists every incident that has come and gone) shows what these notifications are about — the problems that exist *right now* — as a list you can look at, filter, and
|
||||||
export. It uses the same checks as the notifications above, so a problem appears there for exactly the reason it would be notified, but it differs
|
export. It uses the same checks as the notifications above, so a problem appears there for exactly the reason it would be notified, but it differs
|
||||||
in three ways:
|
in a few ways:
|
||||||
|
|
||||||
- It ignores the "Notify on" toggles. Turning a notification off doesn't hide the problem from the page.
|
- It ignores the "Notify on" toggles. Turning a notification off doesn't hide the problem from the page.
|
||||||
- Problems under a **maintenance window** are kept on the list, marked *silenced* and counted separately, instead of being dropped.
|
- Problems under a **maintenance window** are kept on the list, marked *silenced* and counted separately, instead of being dropped. So are **snoozed** ones, marked with when they come back.
|
||||||
|
- It also lists what is on the Alerts page only: forecasts, operating systems out of support, a restore test that failed or is overdue — and, at the top, what could not be checked.
|
||||||
- It also lists things nothing notifies about: Uptime Kuma monitors that are down, osTicket tickets that are overdue, and any integration that's
|
- It also lists things nothing notifies about: Uptime Kuma monitors that are down, osTicket tickets that are overdue, and any integration that's
|
||||||
failing its last few calls (before the threshold that triggers an "integration down" notification).
|
failing its last few calls (before the threshold that triggers an "integration down" notification).
|
||||||
|
|
||||||
It runs the checks live when opened (a recent result is reused for a minute), and shows what it couldn't read at the top, so a missing section means
|
It runs the checks live when opened (a recent result is reused for a minute), and shows what it couldn't read at the top, so a missing section means
|
||||||
"couldn't check" and not "all clear".
|
"couldn't check" and not "all clear". An alert that has an osTicket ticket (see the next section) shows its ticket number.
|
||||||
|
|
||||||
|
## Tickets in osTicket
|
||||||
|
|
||||||
|
Not a notification, but it acts on the same alerts: with **Alert settings → Alert tickets** switched on (off by default), a problem on the Alerts page opens a
|
||||||
|
ticket in osTicket. It runs every 15 minutes, starting about a minute and a half after the app starts. It never closes a ticket by itself; closing is something you choose, in the Close drawer on the Alerts page, when the integration has a separate database login for it (see INTEGRATIONS.md).
|
||||||
|
|
||||||
|
| Rule | What it means |
|
||||||
|
|---|---|
|
||||||
|
| Which problems | The severities and kinds you choose — **critical and warning, every kind** by default. Info (updates waiting and the like) is off. |
|
||||||
|
| Must last | A problem must be there in **2 checks in a row** (15–30 minutes) before it opens a ticket, so a blip that fixes itself — one failed call to an integration, a monitor that flickers — never does. Can be set from 1 (at once) to 4. |
|
||||||
|
| One ticket per problem | However many checks see it, once. After a ticket is open, the problem isn't ticketed again while it lasts — even if someone closes the ticket in osTicket. |
|
||||||
|
| Comes back | A problem that clears and returns **within 60 minutes** is the same problem (no new ticket). After longer, it is a new one and gets a new ticket. |
|
||||||
|
| Many at once | **More than 5 new problems in one check** become a single summary ticket listing them all, so one outage doesn't fill the queue. Up to 5 get a ticket each. |
|
||||||
|
| Maintenance windows | A problem under one never opens a ticket. If it is still there after the window ends, it does. |
|
||||||
|
| Never about osTicket | Overdue tickets, and osTicket's own calls failing, never open a ticket — it would go to the thing that is down, or add to the pile. |
|
||||||
|
| Already there at switch-on | **Left alone by default**: the first check after switching on records what is wrong as the starting line, and only problems that start after it get a ticket. Can be set to ticket those too. Switching it off and on again starts a new line. |
|
||||||
|
| If osTicket can't be reached | Nothing is recorded as opened; the next check tries again. A refused key, a wrong address and similar are shown on the settings page and in the Diagnostic Log. |
|
||||||
|
| If something can't be read | When a check couldn't read everything (an integration was down), nothing is treated as gone, so a problem isn't called cleared just because its source couldn't be looked at. |
|
||||||
|
|
||||||
|
The ticket says what is wrong, the severity, the kind, where it comes from, when it was first seen, and links back to the problem in this app. osTicket's
|
||||||
|
API can't update or close a ticket, so it isn't touched again: close it in osTicket when the problem is dealt with. Every ticket opened (and every test
|
||||||
|
ticket) is in the Audit Log as `alert_ticket`, and changes to the settings as `settings` / `update_alert_tickets`. "Send a test ticket" and "Check now"
|
||||||
|
on the settings page use the saved settings.
|
||||||
|
|
||||||
## What does *not* send a notification
|
## What does *not* send a notification
|
||||||
|
|
||||||
Worth calling out explicitly, since it's easy to assume everything in
|
Worth calling out explicitly, since it's easy to assume everything in
|
||||||
the app alerts on something:
|
the app alerts on something:
|
||||||
|
|
||||||
- **osTicket** — the integration lists open tickets, but there is no
|
- **Forecasts** — a server's disk, a Proxmox storage or a Synology volume that will be full within the forecast window — are on the Alerts page, in the weekly digest and in /metrics,
|
||||||
scheduled check or alert wired up for it (e.g. nothing pings you about
|
but are not sent as they happen.
|
||||||
a newly-overdue ticket). It's a read-only dashboard/page today.
|
- **Operating-system support** — a server whose OS has passed, or is about to pass, its end of support — is on the Alerts page (a warning once it has ended, a note while it is coming), not sent.
|
||||||
|
- **A snoozed alert**: its raise and recovery messages from the health and automation checks are held until the snooze ends (then one "Snooze ended" message, if it is still there). The summaries
|
||||||
|
other checks send — stale backups, expiring secrets, Docker updates — are not held.
|
||||||
|
|
||||||
|
- **osTicket's own tickets** — the integration lists open tickets, but
|
||||||
|
there is no notification for it (e.g. nothing pings you about a
|
||||||
|
newly-overdue ticket); the page and the Alerts page show them. (The
|
||||||
|
other direction — an alert opening a ticket — is the section above.)
|
||||||
- **phpIPAM sync**, **Uptime Kuma monitor/maintenance import**, and any
|
- **phpIPAM sync**, **Uptime Kuma monitor/maintenance import**, and any
|
||||||
other manual "Sync from X" action — these run when you click the
|
other manual "Sync from X" action — these run when you click the
|
||||||
button and report their result on the page itself, not via a
|
button and report their result on the page itself, not via a
|
||||||
@@ -119,8 +360,14 @@ the app alerts on something:
|
|||||||
test") send a one-off message through that one channel only, to
|
test") send a one-off message through that one channel only, to
|
||||||
confirm it's wired up correctly — not a real event and not affected by
|
confirm it's wired up correctly — not a real event and not affected by
|
||||||
quiet hours.
|
quiet hours.
|
||||||
- **Consistency reports**, **Diagnostic Log**, and **Audit Log** are all
|
- **Consistency reports**, the **Diagnostic Log** (Integrations → Call log), and the **Audit Log** (Change log → Raw audit log) are all
|
||||||
read-only views you check yourself; none of them push a notification
|
read-only views you check yourself; none of them push a notification
|
||||||
on their own (the Diagnostic Log's failures are what feed the
|
on their own (the Diagnostic Log's failures are what feed the
|
||||||
integration-down alert above, but reading the log itself never
|
integration-down alert above, but reading the log itself never
|
||||||
triggers anything).
|
triggers anything).
|
||||||
|
|
||||||
|
## Heartbeats
|
||||||
|
|
||||||
|
A **heartbeat** that doesn't hear from its job in time is announced like any other problem — under the *server offline, disk full …* switch (`healthAlerts`) — once when it is late, with a
|
||||||
|
recovery message when the next call comes in. A job that calls its `/fail` address is announced the same way, as critical, at once. For routing both are *Automation*, and carry the tags of
|
||||||
|
the server the heartbeat is linked to; maintenance on that server holds them back, and they alert after it if they are still there. Quiet hours, escalation and push preferences apply as to the rest.
|
||||||
@@ -16,18 +16,23 @@ Authentik (OIDC), with local admin/operator/viewer roles.
|
|||||||
|
|
||||||
All modules from the original plan are built:
|
All modules from the original plan are built:
|
||||||
|
|
||||||
- Monorepo scaffold, Tabler-themed app shell with a grouped sidebar (Infrastructure, Network, Automation, Operations, Administration; groups open on demand, the one holding the current page is always open, and what you leave open is remembered)
|
- Monorepo scaffold, Tabler-themed app shell with a grouped sidebar (Infrastructure, Apps, Backups, Network, Monitoring, Automation, Operations, Records, Reference, Administration; groups open on demand, the one holding the current page is always open, and what you leave open is remembered)
|
||||||
- Authentik OIDC login, roles (first user to sign in becomes admin), audit log
|
- Authentik OIDC login, roles (first user to sign in becomes admin), audit log
|
||||||
(changes made in the app, sign-ins and sign-outs with the IP they came from, new accounts, and the log's own
|
(changes made in the app, sign-ins and sign-outs with the IP they came from, new accounts, and the log's own
|
||||||
automatic trimming — attributed to "system"; settings changes show what changed, but never credentials)
|
automatic trimming — attributed to "system"; settings changes show what changed, but never credentials)
|
||||||
- **Dashboard** — an overview of every system this app tracks, all sharing
|
- **Dashboard** — opens with a status banner that answers "is it fine?": one sentence ("All clear", "3 things need attention"), the root cause when one
|
||||||
|
problem explains others, which kinds of thing are wrong (each links to those alerts) and the worst few. Below it, a card whose status badge is red or yellow
|
||||||
|
comes first in its row, with a coloured top edge (switchable under Customise, remembered in that browser; your own arrangement is untouched underneath).
|
||||||
|
The rest is an overview of every system this app tracks, all sharing
|
||||||
one widget-card design (label + status badge, a small stat row, then its
|
one widget-card design (label + status badge, a small stat row, then its
|
||||||
own breakdown): DNS (domain/record counts per provider, cached records by
|
own breakdown): DNS (domain/record counts per provider, cached records by
|
||||||
type), Secrets (monitored/expiring/expired, by type), and one widget per
|
type), Secrets (monitored/expiring/expired, by type), and one widget per
|
||||||
integration — Tailscale by OS, Proxmox by node (VM/LXC counts too),
|
integration — Tailscale by OS, Proxmox by node (VM/LXC counts too),
|
||||||
Dockhand by container state (plus host count), Semaphore and Gitea by
|
Dockhand by container state (plus host count), Semaphore and Gitea by
|
||||||
last-run status (plus private-repo count), and Synology's CPU/RAM
|
last-run status (plus private-repo count), and Synology's CPU/RAM
|
||||||
alongside its disk-health breakdown. Breakdowns render as a stacked
|
alongside its disk-health breakdown. The integration widgets are in sections that follow the sidebar's groups (Infrastructure, Apps, Network and so on) and show only
|
||||||
|
the groups you have something connected in; an integration with no widget of its own (Jellyfin, Immich, Nextcloud, Hetzner, Speedtest …) gets a small tile: whether it works,
|
||||||
|
how many alerts are open for it, and a link to its page. Breakdowns render as a stacked
|
||||||
proportion bar with a legend — no charting
|
proportion bar with a legend — no charting
|
||||||
library, matching the rest of the app's plain-Tabler-CSS approach.
|
library, matching the rest of the app's plain-Tabler-CSS approach.
|
||||||
The Servers widget shows how many servers are online, offline, or have never
|
The Servers widget shows how many servers are online, offline, or have never
|
||||||
@@ -44,7 +49,9 @@ All modules from the original plan are built:
|
|||||||
stored snapshots have failed verification or were never verified.
|
stored snapshots have failed verification or were never verified.
|
||||||
The osTicket widget shows how many tickets are open, overdue, and awaiting
|
The osTicket widget shows how many tickets are open, overdue, and awaiting
|
||||||
a staff reply.
|
a staff reply.
|
||||||
- **Diagnostic Log** (admin-only) — every call this app makes to a DNS
|
The Technitium DNS widget (in the Network section) shows whether DNS answers and how fast, the last day's queries and the share blocked, a bar of how they were answered
|
||||||
|
(cache, resolved, its own zones, blocked, dropped) and what is wrong with it; it is refreshed every minute and can be hidden under Customise like the rest.
|
||||||
|
- **Diagnostic Log** (admin-only; Administration → Integrations → *Call log*) — every call this app makes to a DNS
|
||||||
provider or integration (Tailscale, Proxmox, Synology, Semaphore, Gitea,
|
provider or integration (Tailscale, Proxmox, Synology, Semaphore, Gitea,
|
||||||
Dockhand, Uptime Kuma, Proxmox Backup Server, osTicket), success or failure, with latency and the error message if it
|
Dockhand, Uptime Kuma, Proxmox Backup Server, osTicket), success or failure, with latency and the error message if it
|
||||||
failed — the last 500 calls, filterable by source/result, for
|
failed — the last 500 calls, filterable by source/result, for
|
||||||
@@ -56,7 +63,9 @@ All modules from the original plan are built:
|
|||||||
real TLS connection (daily, and on demand via "Check now"), reads the
|
real TLS connection (daily, and on demand via "Check now"), reads the
|
||||||
certificate's actual expiry, and keeps the date current — so a renewed cert
|
certificate's actual expiry, and keeps the date current — so a renewed cert
|
||||||
is picked up automatically and an unreachable host is flagged instead of
|
is picked up automatically and an unreachable host is flagged instead of
|
||||||
silently going stale
|
silently going stale. A secret that doesn't expire (a break-glass password,
|
||||||
|
a key with no end date) can be ticked "Never expires": it is listed with the
|
||||||
|
status "no expiry" and never raises an alert or reminder
|
||||||
- **IP Addresses (IPAM)** — inventory of IPs across vendors/locations,
|
- **IP Addresses (IPAM)** — inventory of IPs across vendors/locations,
|
||||||
with "Sync from Tailscale", "Sync from Proxmox", and "Sync from phpIPAM"
|
with "Sync from Tailscale", "Sync from Proxmox", and "Sync from phpIPAM"
|
||||||
actions to pull in tailnet device IPs, VM/LXC IPs, and phpIPAM's own
|
actions to pull in tailnet device IPs, VM/LXC IPs, and phpIPAM's own
|
||||||
@@ -69,12 +78,20 @@ All modules from the original plan are built:
|
|||||||
a range on the spot; managing ranges from either page updates the other
|
a range on the spot; managing ranges from either page updates the other
|
||||||
- **DNS** — zone/record management across Cloudflare, Loopia, Pi-hole, Azure
|
- **DNS** — zone/record management across Cloudflare, Loopia, Pi-hole, Azure
|
||||||
DNS, cPanel, and Technitium; providers are configured in-app (not via env
|
DNS, cPanel, and Technitium; providers are configured in-app (not via env
|
||||||
vars) and their credentials are encrypted at rest
|
vars) and their credentials are encrypted at rest. On a Technitium provider an
|
||||||
|
administrator can also **make a zone** (Primary, Secondary or Forwarder — a name, or a
|
||||||
|
network for a reverse zone) **edit** one (switch it off and on; who may ask it, copy it, and be told when it changes) and **delete** one (its name typed to confirm; every record in it goes), and
|
||||||
|
Technitium's own refusal (a zone that exists, a token that may not delete) is shown as it was said
|
||||||
- **Servers** — cron/systemd tracking across Debian/Raspbian servers via a
|
- **Servers** — cron/systemd tracking across Debian/Raspbian servers via a
|
||||||
lightweight push agent (`agent/linux/`), Windows scheduled-task tracking
|
lightweight push agent (`agent/linux/`), Windows scheduled-task tracking
|
||||||
through a PowerShell agent (`agent/windows/`, see its README), plus manual
|
through a PowerShell agent (`agent/windows/`, see its README), plus manual
|
||||||
entries for things an agent can't see (Docker jobs, backups). The Servers page itself just lists
|
entries for things an agent can't see (Docker jobs, backups). The Servers page itself just lists
|
||||||
registered servers and (admin-only) adds new ones / issues agent tokens;
|
registered servers and (admin-only) adds new ones / issues agent tokens;
|
||||||
|
a server's **hostname** — which an agent replaces with the machine's own at every report — can be
|
||||||
|
changed by an administrator on its page (*Identity → Change*): it is then **kept** even when the
|
||||||
|
agent reports another (marked *set by hand*, with what the agent reports shown beside it), until
|
||||||
|
*Use what the agent reports* gives it up. It is the name this app has for the server, used to
|
||||||
|
match it to DNS names and monitors and to reach it; it does not rename the machine itself;
|
||||||
clicking a server opens its detail page with CPU/RAM/disk status, IP
|
clicking a server opens its detail page with CPU/RAM/disk status, IP
|
||||||
addresses, matching DNS names (looked up from the DNS module's cache), and
|
addresses, matching DNS names (looked up from the DNS module's cache), and
|
||||||
its scheduled tasks — live hardware from Proxmox for VM/LXC-backed
|
its scheduled tasks — live hardware from Proxmox for VM/LXC-backed
|
||||||
@@ -82,11 +99,15 @@ All modules from the original plan are built:
|
|||||||
both showing the same per-disk usage breakdown (an LXC's root filesystem
|
both showing the same per-disk usage breakdown (an LXC's root filesystem
|
||||||
read straight from the host; a QEMU VM's actual mounts via its guest
|
read straight from the host; a QEMU VM's actual mounts via its guest
|
||||||
agent, alongside the allocated size Proxmox already knew about without
|
agent, alongside the allocated size Proxmox already knew about without
|
||||||
one). Proxmox-linked servers also get start/stop/restart buttons right on the
|
one). Servers that report through the agent also get a **Trends** card: small
|
||||||
detail page. The detail page also has an **Admin Links** section
|
lines for CPU load, memory and each disk over the last 24 hours, 7, 30 or 90 days or a year
|
||||||
|
(a sample is kept about every ten minutes, for 35 days; hours and days beyond that, up to a year), and for a disk that is
|
||||||
|
filling, its rate of growth and roughly how many days until it is full — shown
|
||||||
|
only with at least a day of history. Proxmox-linked servers also get
|
||||||
|
start/stop/restart buttons right on the detail page. The detail page also has an **Admin Links** section
|
||||||
(operator/admin to add/edit/remove) for bookmarking that server's own
|
(operator/admin to add/edit/remove) for bookmarking that server's own
|
||||||
admin UIs — Dockge, Webmin, Cockpit, Portainer, or anything else reachable
|
admin UIs — Dockge, Webmin, Cockpit, Portainer, or anything else reachable
|
||||||
by URL. **Operations → Admin Links** summarizes every server's admin links
|
by URL. **Reference → Admin Links** summarizes every server's admin links
|
||||||
in one sortable, searchable table (server, label, URL, an "Open" link, and
|
in one sortable, searchable table (server, label, URL, an "Open" link, and
|
||||||
the same add/edit/delete as the per-server section — adding one here just
|
the same add/edit/delete as the per-server section — adding one here just
|
||||||
asks which server it belongs to), so finding or managing one doesn't mean
|
asks which server it belongs to), so finding or managing one doesn't mean
|
||||||
@@ -118,8 +139,10 @@ All modules from the original plan are built:
|
|||||||
manages (one credential covers all of them), with
|
manages (one credential covers all of them), with
|
||||||
start/stop/restart actions, a host filter, image-update status per
|
start/stop/restart actions, a host filter, image-update status per
|
||||||
container (from Dockhand's own cached update check, plus a button
|
container (from Dockhand's own cached update check, plus a button
|
||||||
to trigger a fresh one), and a live running/total widget (with an
|
to trigger a fresh one), a live running/total widget (with an
|
||||||
updates-available count).
|
updates-available count), and the **vulnerabilities** Dockhand's own
|
||||||
|
scans found per host (critical ones listed; a warning is raised for the critical ones — and, if you give a number, for hosts with that many high ones — and
|
||||||
|
it can be switched off). A number of days you choose also makes running images built longer ago than that a warning — scanned or not (off by default).
|
||||||
- **Uptime Kuma** — every monitor's status (up/down/pending/maintenance),
|
- **Uptime Kuma** — every monitor's status (up/down/pending/maintenance),
|
||||||
response time, and 24h/30d uptime, with certificate days remaining where
|
response time, and 24h/30d uptime, with certificate days remaining where
|
||||||
applicable. Each monitor is matched to one of your servers when its
|
applicable. Each monitor is matched to one of your servers when its
|
||||||
@@ -144,11 +167,17 @@ All modules from the original plan are built:
|
|||||||
read-only user — the only integration here that isn't a REST API.
|
read-only user — the only integration here that isn't a REST API.
|
||||||
Read-only, no actions.
|
Read-only, no actions.
|
||||||
|
|
||||||
The Integrations page itself is now just a list of configured
|
The Integrations page itself is a list of configured
|
||||||
integrations (name/type/status, visible to every role) with an
|
integrations (name/type/status, visible to every role), grouped the way the sidebar
|
||||||
admin-only "Add integration" button and edit/enable/disable/delete
|
groups their pages (Infrastructure, Apps, Backups, Network, Monitoring, Automation),
|
||||||
actions per row — the nine dedicated pages above are where you
|
with an admin-only "Add integration" button — its kind picker is the same groups,
|
||||||
actually use each one.
|
with a search box — and edit/enable/disable/delete actions per row. The dedicated
|
||||||
|
pages are where you actually use each one. A **Cards / Table** switch beside the search box
|
||||||
|
shows the same integrations as a table instead — name, kind, group, address, status, the
|
||||||
|
last 30 days and the same buttons — sortable on any column (Status puts the failing ones
|
||||||
|
first); the choice is in the address (`?layout=table`, with `?sort=`), so it can be linked or
|
||||||
|
saved as a view, and this browser remembers it for the next visit. Search and the status
|
||||||
|
filter work in both.
|
||||||
- Every table in the app is click-to-sort on any column (numbers, booleans,
|
- Every table in the app is click-to-sort on any column (numbers, booleans,
|
||||||
and dates/text sort correctly regardless of how the column formats them)
|
and dates/text sort correctly regardless of how the column formats them)
|
||||||
and has an "Export CSV" button next to it that exports whatever's
|
and has an "Export CSV" button next to it that exports whatever's
|
||||||
@@ -158,8 +187,14 @@ All modules from the original plan are built:
|
|||||||
20 rows per page by default — adjustable under Settings → Display — and
|
20 rows per page by default — adjustable under Settings → Display — and
|
||||||
CSV export still covers every sorted/filtered row, not just the current
|
CSV export still covers every sorted/filtered row, not just the current
|
||||||
page.
|
page.
|
||||||
- **Settings** (admin-only) — notification channels (Gotify, ntfy, SMTP,
|
The few small tables that have no search or sorting of their own (the
|
||||||
generic webhook) with per-channel test buttons, per-event toggles (DNS
|
maintenance lists, tags, share links, API tokens and webhooks, the
|
||||||
|
Backup page's job and run lists, the comparison table) have an Export CSV
|
||||||
|
too, which writes the table as it is on screen — times as exact timestamps,
|
||||||
|
action buttons left out.
|
||||||
|
- **Alert settings** (Monitoring → Alert settings) — what decides what is an alert, on one page with three tabs: *Alert rules*, *Service levels* and *Alert tickets* (`/alert-settings?view=rules|levels|tickets`). Everyone signed in sees the first two; *Alert tickets* is administrators' (it used to be under Settings, and `/alert-rules`, `/service-levels` and `/settings/alert-tickets` still lead to the right tab). Notifications — channels, what is sent, who hears what, quiet hours — serve more than alerts, so they stay under **Settings → Notifications**, with a link to them at the top of this page for administrators. Maintenance stays on its own page.
|
||||||
|
- **Settings** (admin-only) — notification channels (Gotify, ntfy, Telegram,
|
||||||
|
Pushover, SMTP, generic webhook, and push to phones and browsers) with per-channel test buttons, per-event toggles (DNS
|
||||||
record added/updated/deleted, daily secret-expiry reminder and daily
|
record added/updated/deleted, daily secret-expiry reminder and daily
|
||||||
Tailscale key-expiry reminder — both sharing one configurable
|
Tailscale key-expiry reminder — both sharing one configurable
|
||||||
time/timezone), badge-color customization for both DNS
|
time/timezone), badge-color customization for both DNS
|
||||||
@@ -206,11 +241,13 @@ offered as one-click suggestions when tagging), give any tag a colour of your ch
|
|||||||
that already exists merges the two, and both rename and delete rewrite every server
|
that already exists merges the two, and both rename and delete rewrite every server
|
||||||
that carries the tag.
|
that carries the tag.
|
||||||
|
|
||||||
**Name generator** — Operations → Generator suggests server names (and usernames and
|
**Name generator** — Reference → Generator suggests server names (and usernames and
|
||||||
passwords, which are made in your browser and never stored). Server names are picked from
|
passwords, which are made in your browser and never stored). Server names are picked from
|
||||||
name lists: Swedish girl and boy names, Disney and Pixar characters, Norse mythology and
|
name lists: Swedish girl and boy names, Disney and Pixar characters, Norse mythology and
|
||||||
Astrid Lindgren to start with, or "Mixed" for all of them together. A name already used
|
Astrid Lindgren to start with, or "Mixed" for all of them together. A name already used
|
||||||
by a server isn't suggested. Admins edit the lists under Settings → Names — add and remove
|
by a server isn't suggested. "How many" makes up to 20 different names at once (handy when
|
||||||
|
setting up several machines), each with its own Copy button and a "Copy all" that puts
|
||||||
|
them one per line on the clipboard. Admins edit the lists under Settings → Names — add and remove
|
||||||
names, rename, delete or create lists, put a built-in list back as it was — and can
|
names, rename, delete or create lists, put a built-in list back as it was — and can
|
||||||
import the most common Swedish names from Skatteverket's open name statistics for
|
import the most common Swedish names from Skatteverket's open name statistics for
|
||||||
girls or boys over the latest one to five years. The import is shown to you first and only
|
girls or boys over the latest one to five years. The import is shown to you first and only
|
||||||
@@ -232,6 +269,18 @@ relatives), and connect to the address they checked rather than looking the name
|
|||||||
checks and scans; a WHOIS server must be public. IDs in a request (a device, a node, a DNS record) are checked against what a real ID
|
checks and scans; a WHOIS server must be public. IDs in a request (a device, a node, a DNS record) are checked against what a real ID
|
||||||
looks like before they go into a call to a connected system. `npm test --workspace server` covers all of this.
|
looks like before they go into a call to a connected system. `npm test --workspace server` covers all of this.
|
||||||
|
|
||||||
|
**Signing in** — through Authentik (OpenID Connect) with PKCE and a random `state`. The ID token's signature is checked against Authentik's
|
||||||
|
published keys, not just its contents, so the login doesn't rest on the connection to Authentik alone. A new session id is issued at sign-in. A
|
||||||
|
request that changes something is refused when it comes from another origin: the session cookie is SameSite=Lax, which doesn't stop another app on
|
||||||
|
the same parent domain, so the server also checks where each such request came from. A failed sign-in shows a plain message and is written to the
|
||||||
|
audit log (as `session` / `login_failed`, with the address and a short reason), and sign-in is limited to 20 attempts a minute per address.
|
||||||
|
Every response carries `frame-ancestors 'none'`, `X-Frame-Options`, `nosniff` and a no-referrer-to-other-sites policy. In production the app
|
||||||
|
refuses to start without a `SESSION_SECRET`, and warns if it is still the placeholder from `.env.example`.
|
||||||
|
Things to know: the first account to sign in becomes admin, so sign in yourself first on a new install. A session lasts up to 7 days after last use,
|
||||||
|
and disabling someone in Authentik does not end it — end the session under Users → *Signed in now* and it ends at once (accounts themselves are not deleted in the app).
|
||||||
|
If Authentik uses a certificate from an internal CA, point Node at the CA with `NODE_EXTRA_CA_CERTS=/path/to/ca.pem` rather than turning verification
|
||||||
|
off with `NODE_TLS_REJECT_UNAUTHORIZED=0`.
|
||||||
|
|
||||||
**Privacy** — a page every signed-in user can open that says what this
|
**Privacy** — a page every signed-in user can open that says what this
|
||||||
installation stores (accounts, sign-in sessions with their IP and browser, the audit
|
installation stores (accounts, sign-in sessions with their IP and browser, the audit
|
||||||
and diagnostic logs, server reports, the secrets tracker, credentials), where data
|
and diagnostic logs, server reports, the secrets tracker, credentials), where data
|
||||||
@@ -256,7 +305,7 @@ can be ignored with a reason, and stays ignored. The excluded-ranges list is the
|
|||||||
same one the IP Addresses page manages — edit it from either page and both
|
same one the IP Addresses page manages — edit it from either page and both
|
||||||
reflect the change.
|
reflect the change.
|
||||||
|
|
||||||
**Domains** — when each domain registration expires, read from the registry
|
**Domains** (Network → DNS → *Domain names*) — when each domain registration expires, read from the registry
|
||||||
itself. The domains behind your DNS zones are picked up automatically (a zone
|
itself. The domains behind your DNS zones are picked up automatically (a zone
|
||||||
like `lab.example.se` resolves to the `example.se` registration that actually
|
like `lab.example.se` resolves to the `example.se` registration that actually
|
||||||
expires); others can be added by hand. Each is looked up daily over RDAP where
|
expires); others can be added by hand. Each is looked up daily over RDAP where
|
||||||
@@ -281,7 +330,10 @@ already failing, so old failures aren't announced. Toggle it under Settings →
|
|||||||
Notifications. For Gitea this follows the repo's most recent run on any
|
Notifications. For Gitea this follows the repo's most recent run on any
|
||||||
workflow or branch, the same as the Gitea page shows.
|
workflow or branch, the same as the Gitea page shows.
|
||||||
|
|
||||||
**Alerts** (Operations → Alerts, visible to every role) lists everything that's
|
**Alerts** (near the top of the sidebar, visible to every role) opens with one line on where things stand — how many problems need attention and, when some are
|
||||||
|
held back, which problem explains how many ("nasbox accounts for 3") — and marks each row with a coloured edge by severity. Held-back alerts are nested under
|
||||||
|
the alert that explains them (switch "Group under their cause" off for the flat list; the choice is remembered): the cause says "Show the 3 alerts held back
|
||||||
|
under this" and opens them. It lists everything that's
|
||||||
wrong right now in one place, instead of waiting for a notification or visiting
|
wrong right now in one place, instead of waiting for a notification or visiting
|
||||||
each page: servers that stopped reporting, full or nearly full disks and volumes
|
each page: servers that stopped reporting, full or nearly full disks and volumes
|
||||||
(critical from 95%), Synology volume/disk problems, failed or uncovered Proxmox
|
(critical from 95%), Synology volume/disk problems, failed or uncovered Proxmox
|
||||||
@@ -292,23 +344,749 @@ tickets, and integrations whose calls keep failing. It runs the same checks that
|
|||||||
send the notifications — so the two can't disagree — but ignores the on/off
|
send the notifications — so the two can't disagree — but ignores the on/off
|
||||||
toggles, since it's for looking at rather than being interrupted by. Problems
|
toggles, since it's for looking at rather than being interrupted by. Problems
|
||||||
under a maintenance window stay listed, marked silenced and counted separately.
|
under a maintenance window stay listed, marked silenced and counted separately.
|
||||||
|
It also looks ahead: from an agent server's own history (see its Trends card), a
|
||||||
|
disk that is on course to be full within 14 days (Settings → Notifications → "Warn
|
||||||
|
when a disk will fill within"; 0 turns it off) is flagged with its rate of growth
|
||||||
|
and the days left — critical when that is three days or less. It needs at least
|
||||||
|
three days of steady growth to say anything, skips a disk that jumped once or
|
||||||
|
wobbles, and leaves out one already past the usual disk threshold, which has its
|
||||||
|
own alert. These forecasts appear on the page and the dashboard (and in alert
|
||||||
|
tickets if you've chosen disk problems); they are not sent as notifications. On
|
||||||
|
any alert that a maintenance window can silence, operators and admins get a
|
||||||
|
**Silence** button (1, 4 or 24 hours): it starts a maintenance window on that
|
||||||
|
server or integration — so everything about it goes quiet, which the toast says —
|
||||||
|
and it can be ended early under Maintenance.
|
||||||
It checks live (a recent result is reused for a minute; "Check now" forces a
|
It checks live (a recent result is reused for a minute; "Check now" forces a
|
||||||
fresh one), and anything it couldn't read is called out at the top rather than
|
fresh one), and anything it couldn't read is called out at the top rather than
|
||||||
quietly treated as fine.
|
quietly treated as fine.
|
||||||
|
|
||||||
|
**Alert tickets** (Alert settings → Alert tickets, admin only) opens a ticket in osTicket for an alert, through osTicket's own API. Every 15 minutes the
|
||||||
|
problems on the Alerts page are looked at, and one that is new and has lasted two checks in a row gets one ticket — never a second for the same problem,
|
||||||
|
and never one for something under a maintenance window or about osTicket itself. You choose which severities and kinds count (critical and warning by
|
||||||
|
default), and the Alerts page shows the ticket number on each alert that has one. If many problems appear at once — a network drop that makes thirty
|
||||||
|
things look down — they become one summary ticket instead of thirty. Problems already there when you switch it on are left alone unless you ask for
|
||||||
|
tickets for them too. osTicket's API can't close or update a ticket, so once it's open it's yours to close — in osTicket, or, if you give the integration a **separate database login that may update tickets**, from the Alerts page: the Close drawer then offers *Also close the osTicket ticket* — for one alert, or for several picked together (the tickets are closed one after another; Undo opens the alerts again but not the tickets) (it sets the ticket to Closed and writes a note in its thread saying who closed it and why;
|
||||||
|
this writes to osTicket's database, so it checks the tables first, does it all in one transaction, and **Test closing** tries every write and rolls it back). It needs an API key made in osTicket (tied to
|
||||||
|
this server's IP address, allowed to create tickets), entered on the osTicket integration; "Send a test ticket" checks the whole chain.
|
||||||
|
|
||||||
**Maintenance mode** silences alerts about one server, integration, or DNS
|
**Maintenance mode** silences alerts about one server, integration, or DNS
|
||||||
provider while you work on it (server offline / disk, storage and Synology
|
provider while you work on it (server offline / disk, storage and Synology
|
||||||
health, Proxmox backup alerts, Proxmox Backup Server verification alerts, and
|
health, Proxmox backup alerts, Proxmox Backup Server verification alerts, and
|
||||||
"integration down" for that service type).
|
"integration down" for that service type).
|
||||||
Every window has a fixed end (5 minutes to 7 days) and expires on its own, and
|
For a power cut or a rack reboot, the "Many at once" choices at the top of the
|
||||||
|
list start a window on every server, every (enabled) integration, every
|
||||||
|
(enabled) DNS provider, or all of them in one go — asking first before
|
||||||
|
"Everything" — and "End all" on the Active list closes them together. Each is
|
||||||
|
still its own window, so any can be ended alone. Every window has a fixed end (5 minutes to 7 days) and expires on its own, and
|
||||||
a problem that began during a window and is still present when it ends alerts
|
a problem that began during a window and is still present when it ends alerts
|
||||||
then — a forgotten window can't hide an outage. A banner shows what's currently
|
then — a forgotten window can't hide an outage. A banner shows what's currently
|
||||||
silenced to every signed-in user. If you also run Uptime Kuma, "Import from
|
silenced to every signed-in user. If you also run Uptime Kuma, the Maintenance
|
||||||
Uptime Kuma" on the Maintenance page reads which of its monitors are
|
page can take its maintenance windows over: "Sync schedule" reads each window
|
||||||
currently in maintenance and starts (or extends) a window here for whichever
|
(daily, weekly, one-off, cron or switched on by hand), finds the servers,
|
||||||
server each one's target matches, for a duration you choose — Uptime Kuma's
|
integrations and DNS providers its monitors are the address of, and makes a
|
||||||
metrics only say what's in maintenance right now, not for how long, so this
|
window here for each, for the slot running now and the next
|
||||||
doesn't try to mirror its schedule, only its current state.
|
one — listed under Scheduled until it starts, then silencing alerts like any
|
||||||
|
other. Tick "Keep maintenance windows in step" on the Uptime Kuma integration
|
||||||
|
and this happens by itself every 15 minutes, so a daily window needs no
|
||||||
|
clicking at all; a window changed, switched off or deleted in Uptime Kuma is
|
||||||
|
followed, and one you end or cancel here by hand stays ended. Reading the
|
||||||
|
schedule needs a login on the Uptime Kuma integration (see INTEGRATIONS.md).
|
||||||
|
"Import now" is the older, simpler route: it only looks at what is in
|
||||||
|
maintenance this moment and starts a window for a duration you choose.
|
||||||
|
|
||||||
|
**Favourites and a narrow sidebar.** Hover (or tab to) a page in the sidebar and
|
||||||
|
click its star to pin it: pinned pages are listed under "Favourites" at the top
|
||||||
|
of the menu, are offered first in the search box, and follow you to any
|
||||||
|
browser, because they are saved with your user (the search box can also pin or
|
||||||
|
unpin the page you are on). A table's search words, status, tag and sort live
|
||||||
|
in the address (`/servers?status=offline`), so they survive a reload and can be
|
||||||
|
bookmarked; once something narrows the table a **Save view** button pins that
|
||||||
|
view too, listed under Favourites by its filters ("Servers not reporting"). A
|
||||||
|
button at the foot of the sidebar — or the `[` key
|
||||||
|
— narrows it to a strip of icons for more room on a laptop screen: a group's
|
||||||
|
pages then open in a small menu beside its icon, the alert and maintenance
|
||||||
|
counts stay visible, and your choice is remembered in that browser. On a phone
|
||||||
|
the sidebar is the same slide-out menu as ever, and the pages people reach from a notification or a printed label are made for one hand: on Alerts,
|
||||||
|
Updates and Inventory each row becomes a compact card (worst first, with its buttons underneath), the buttons are finger-sized, the
|
||||||
|
acknowledge panel fills the screen, and the Alerts page leaves out its row of figures — the headline already says them. Scanning an
|
||||||
|
inventory label opens that piece of hardware at the top of the page, with an Edit button for operators.
|
||||||
|
|
||||||
|
**Getting around.** Ctrl/Cmd+K opens a search that also runs actions: go to any
|
||||||
|
page, switch light/dark or the density, start (or end) a maintenance window on
|
||||||
|
a server, integration or DNS provider ("silence elsa"), open the wall display,
|
||||||
|
sign out. It also does the things you do most often: **acknowledge** one of the worst alerts nobody is on (for 4 hours, handed to you) or all of them
|
||||||
|
at once, **update** a server that has updates waiting (it opens the update panel, which still asks before anything runs), **print the label** for a piece of
|
||||||
|
hardware or open it in the inventory, and ask **"What depends on …?"** of a server or service (it opens the dependency map with that box picked). "g" then a letter goes to a page (`g d` dashboard, `g a` alerts,
|
||||||
|
`g s` servers, `g m` maintenance, `g i` integrations, `g k` secrets, `g p` map,
|
||||||
|
`g w` wall display), "/" jumps to the page's table search, and "?" lists them
|
||||||
|
all, and "j" / "k" move down and up the rows of a table that opens a panel (Enter
|
||||||
|
opens the row you are on). Shortcuts for pages your role can't open are ignored,
|
||||||
|
and none fire while you are typing. On a table of Proxmox guests, Docker containers, Tailscale
|
||||||
|
devices, Uptime Kuma monitors, Gitea repositories or Semaphore templates, a
|
||||||
|
click (or Enter) on a row opens a panel with everything known about it, without
|
||||||
|
leaving the table.
|
||||||
|
|
||||||
|
**Map** (Reference → Map) has two views, *Connections* and *Dependencies* (see below). *Connections* draws how things connect: which Proxmox host each
|
||||||
|
linked server runs on, which DNS names point at it (an A or AAAA record holding
|
||||||
|
one of the server's own addresses — nothing is guessed from names), and which
|
||||||
|
Uptime Kuma monitors watch it. Hover or tab to a box to light up its
|
||||||
|
connections; each box links to its page. A monitor Uptime Kuma can't be read for
|
||||||
|
leaves a note rather than a silent gap. Both views have a **Lines** switch above the boxes: *Curved* (the default) or *Right-angled* — out from a box, across, up or down
|
||||||
|
and into the next, with the corners softly rounded, like a wiring diagram. It changes only how the lines are drawn, is remembered in this browser, and is shared by both views.
|
||||||
|
|
||||||
|
The Map's second view, **Dependencies**, draws what needs what with the foundations on the left (the servers and services that take part in
|
||||||
|
a dependency or a "runs on" link, each with its status). Pick a box and what would go down with it turns red and what it relies on blue,
|
||||||
|
with a sentence underneath — "If it goes down, 4 things go with it: …" — so the answer to "what breaks if I reboot this?" is there before you do it.
|
||||||
|
|
||||||
|
**Named wall displays** (Settings → Share links → *Wall displays*, administrators). One board for every screen is rarely right: the hall TV wants the overview and what needs attention, the office monitor the apps and the network. A display is a **name**, an **address of its own** (`/wall/hall-tv`, made from the name unless you give one — it stays put if you later correct the name, so a bookmark on a screen keeps working), **which pages it rotates through and in what order** (the overview, *Needs attention*, and a page for each group of integrations; none ticked is every page) and **how fast**. A page that has nothing to show just now (*Needs attention* with nothing wrong, a group with no integrations) is left out, and a display none of whose pages exist shows the overview. The signed-in wall opens one from its address and has a *Display* choice in its footer; the footer says which display it is; an unknown address says so and shows everything. How fast is decided by the address (`?rotate=45`) first, then the display, then what this browser last chose. A **wall share link** can open a display too (choose it when making the link): the public board then rotates through that display's pages at its speed; remove the display and the link goes on opening the whole board. Up to twenty displays; each change is in the audit log and the change log. **A night schedule** can dim or blank a display: from a time to a time on **the screen's own clock** (every day; an end earlier than the start runs past midnight), the board is either **dimmed** (the same picture at about a quarter of its brightness) or **blank** (black, with a faint note; the screen is no longer held awake, so a tablet or computer may go to sleep). A page cannot switch a TV off, so these are the two things it can do. By default **a critical alert brings the board back** whatever the hour (switch that off per display), and **a touch or a key press brings it back for two minutes**. It works the same for a share link that opens the display. A display only chooses among the board's pages — what the board shows (and, for a share link, how much: numbers only or with names) is unchanged.
|
||||||
|
|
||||||
|
**Wall display** (`/wall`, also on the dashboard and the user menu) is a
|
||||||
|
full-screen status board for a spare monitor or tablet: the overall state in
|
||||||
|
large type, servers online, alerts, maintenance windows and failing integrations — and, for what is connected, how many Hetzner servers and game servers are running, how many critical vulnerabilities and stuck downloads there are, and one line for each group of the menu (Infrastructure, Apps …) saying whether it is well —
|
||||||
|
the problems themselves, and each integration's status, refreshing every 30
|
||||||
|
seconds with the screen kept awake where the browser allows. Esc returns to the
|
||||||
|
app.
|
||||||
|
|
||||||
|
**Rotating the wall display.** A spare screen can show more than one board: choose **Rotate → every 15 / 30 / 60 / 120 s** in the footer (or open `/wall?rotate=45`; `?rotate=on` is 30 s, `?rotate=off` stops it) and the lower half of the board changes page by page while the status line, the clock and the figures at the top stay put. The pages are the **Overview** (as before), **Needs attention** (the open alerts and what is in maintenance, in full — only while there is something), and one page for each group of the menu that has integrations (Infrastructure, Apps …) listing them, the failing ones first. The footer says which page it is on and **Pause** / **Resume** holds one on screen; the left and right arrow keys step by hand. The choice is in the address, so a TV's bookmark carries it, and this browser remembers the last one chosen on the board itself; without either the wall stays on the overview, as it always was. A share link's wall rotates the same way (add `?rotate=30` to its address) when it shows names; one that shows numbers only has a single page and offers nothing to rotate.
|
||||||
|
|
||||||
|
**Alerts folded under their host.** A server can say which server it runs on (a
|
||||||
|
virtual machine on its Proxmox host): on its page, "Runs on" — and left alone it
|
||||||
|
is worked out when the server is linked to a Proxmox guest and exactly one
|
||||||
|
server is named after that guest's node. When the host stops reporting, the
|
||||||
|
guests' own "server offline" alerts are folded under the host's ("also takes down
|
||||||
|
anna, elsa") instead of being one alert each: they are held back like a silenced
|
||||||
|
alert, not counted, left out of notifications and osTicket tickets, and labelled
|
||||||
|
"caused by pve1" on the Alerts page. They are still recorded as down (for the
|
||||||
|
uptime ribbons and "Since you last looked"), and if the host comes back while a
|
||||||
|
guest is still down, that guest alerts then as new — an outage is never swallowed.
|
||||||
|
Links that loop back on themselves cover nothing, and setting one that would is
|
||||||
|
refused. "Nothing — it runs on its own" stops the guess for a server that isn't a
|
||||||
|
guest (a physical machine that happens to share a Proxmox node's name), so its
|
||||||
|
alerts always stand alone.
|
||||||
|
|
||||||
|
**Restore tests** (on the Backup coverage page). A backup nobody has restored is a hope, not a backup: when you try one, record it — which guest (or anything else: a
|
||||||
|
share, a database), the day, how it went (restored fine, partly, failed, or *no need to test this one*) and a note. Each guest's newest test sits beside its backups.
|
||||||
|
A **failed** test is always an alert (critical) until a newer one passes. Under Settings → Notifications → "A restore should have been tested within" you can also
|
||||||
|
ask to be reminded: a guest that has backups but no passing test within that many days (or ever) becomes a warning, and so does anything else you chose to track;
|
||||||
|
0, the default, leaves the reminder off. A guest marked *no need to test* never is. Operators record tests, administrators can remove a mistaken one; the history is
|
||||||
|
kept, `/metrics` has each subject's age and result, and everything is in the change log.
|
||||||
|
|
||||||
|
**The change log** (Records → Change log, operators and administrators) is one list of what has changed, newest first and grouped by day: agents updated,
|
||||||
|
settings changed (with what to what), maintenance started and ended, dependencies edited, hardware added, endpoint checks and restore tests, alerts raised and cleared —
|
||||||
|
each as a sentence. Narrow it by kind of change, by server, by how far back (a day to everything kept), or by a word; "Show older" goes on from where it stopped; CSV
|
||||||
|
export. A server's Activity card links to its slice of it. Sign-ins and people's own data exports aren't changes and are left out. The raw audit log is the other tab of the same page (Change log → *Raw audit log*).
|
||||||
|
|
||||||
|
**Dependencies beyond servers** (Reference → Map → *Dependencies*, which draws the graph with the editor under it). Say that Gitea needs the server it is installed on, or that a server
|
||||||
|
needs Proxmox to answer. **Internet** is on the list too, as something a server, a service or a network device can need (a VPS, Hetzner, the router): it is drawn on the map and answers "what depends on Internet?", but nothing watches it, so it is never down and holds no alert back. It is only ever needed — it can't be made to need anything. When what is needed is down, the alert of what needs it is folded under the first one exactly like a
|
||||||
|
guest under its host — shown, labelled "caused by …", left out of notifications and tickets — and the root cause says what else it
|
||||||
|
takes down. It follows chains and the "runs on" links of servers; something needing two things is explained by whichever is down;
|
||||||
|
a loop explains nothing and one that would be made is refused. An integration is judged per kind of service (as its health is),
|
||||||
|
so it is folded only when every enabled instance of that kind is explained. A service that starts failing while the server it needs
|
||||||
|
is offline isn't announced; if it is still failing once the server is back, the next failed call alerts. (DNS providers raise no
|
||||||
|
alerts of their own, so they aren't part of this.)
|
||||||
|
|
||||||
|
**Who hears what** (Settings → Notifications → Who hears what). *Routing rules* send a kind of notification, of at least a severity, about servers with
|
||||||
|
certain tags, to the channels you choose — the first matching rule decides, none matching means every channel that is on, and a rule with no channels tells
|
||||||
|
nobody. Each phone or browser can also say it only wants warnings or critical alerts, and only some kinds. *Escalation* tells again, through louder channels
|
||||||
|
and perhaps an extra email address, about an alert nobody has acknowledged after a time you set (up to three steps, each once; an acknowledgement stops the
|
||||||
|
rest). See NOTIFICATIONS.md.
|
||||||
|
|
||||||
|
**Seeing the impact before maintenance.** Choosing a server or integration under Maintenance → Start maintenance lists what depends on it and
|
||||||
|
will look down while it is — and offers to put those under maintenance too, for the same time, with the reason "Depends on …" (ticked by default;
|
||||||
|
each is its own window, listed and ended like any other, and audited). The recurring-schedule form shows the same note for what is picked and can add
|
||||||
|
the dependents to the selection. The "Update this server now" drawer shows what depends on the server and may look down while it restarts; it doesn't
|
||||||
|
silence them, because their windows would outlast the run — use Maintenance first if you want that.
|
||||||
|
|
||||||
|
**Recurring maintenance** (Maintenance → Recurring) repeats a window on the days
|
||||||
|
and clock time you choose — "Sundays 02:00–04:00 on these servers" — in the time
|
||||||
|
zone of the browser it was made in, so 02:00 stays 02:00 across summer and winter
|
||||||
|
time. Each time becomes an ordinary maintenance window a week ahead (shown on the
|
||||||
|
timeline with vertical stripes, and in Scheduled), so any single one can be
|
||||||
|
cancelled and stays cancelled. Turning a schedule off or removing it takes back
|
||||||
|
the windows that haven't begun. Edit changes a schedule's name, days, time, length
|
||||||
|
or what it covers: the windows that haven't begun are made again from the new
|
||||||
|
settings (a single one cancelled by hand comes back too), and one already under
|
||||||
|
way keeps its times and ends on its own.
|
||||||
|
|
||||||
|
**Running a playbook** from an alert or a server: a server's page has a "Run a
|
||||||
|
playbook" card, and alerts about a server have a "Run…" button, which pick a
|
||||||
|
Semaphore template and run it limited to that server (ansible's `--limit`, which
|
||||||
|
you can edit to match the host name in your inventory — or untick it to run on
|
||||||
|
everything, which asks more sternly). Semaphore is told who started it, the run
|
||||||
|
shows waiting → running → success or failed, and the audit log records what ran
|
||||||
|
and where. Operators and admins only.
|
||||||
|
|
||||||
|
**Updates** (Operations → Updates) shows what each agent says is waiting to be
|
||||||
|
installed — how many, how many are security fixes, whether a restart is due, and
|
||||||
|
how long updates have been waiting — plus the Docker containers with a newer image.
|
||||||
|
The Linux agent asks apt, dnf or zypper (only what they already know; nothing is
|
||||||
|
installed or refreshed — that one question is asked outside the agent's sandbox, which `dnf` and `zypper` need) and the Windows agent asks Windows Update (at most every six
|
||||||
|
hours, cached under `%ProgramData%\HomelabManager`). A server whose agent hasn't
|
||||||
|
said — an older agent, or another package manager — shows "not reported", never "up
|
||||||
|
to date"; run its install command again to update its agent (how, and how to
|
||||||
|
tell it worked: [AGENT_UPDATE.md](AGENT_UPDATE.md)).
|
||||||
|
|
||||||
|
**Acknowledging and handing over alerts.** On the Alerts page, **Acknowledge…** (operators) says
|
||||||
|
"somebody is on this": an optional note, who has it (an administrator or operator), and how long it should
|
||||||
|
be left alone (an hour to a week, or until the alert is gone). It changes nothing about the problem — the
|
||||||
|
alert is still raised, still counted by `/metrics` and still on the wall display — but the alert shows who
|
||||||
|
has it and why, the sidebar count, the dashboard strip and the Alerts cards count only what nobody is on,
|
||||||
|
and the list can be filtered to "nobody is on it" or "handed to me". An acknowledgement that runs out puts
|
||||||
|
the alert back among the unattended; it goes by itself once the alert has been gone for half an hour, so
|
||||||
|
the same problem returning next month is not "still being dealt with". It is in the audit log and in the
|
||||||
|
server's history.
|
||||||
|
|
||||||
|
**Incidents** (Alerts → **History**; the Alerts page has two views, *Now* and *History*, and an incident's own page is under it). Each time an alert is raised and later goes away is recorded as an
|
||||||
|
incident, with when it began and ended and how bad it got, so "what happened on Tuesday night?" has an answer
|
||||||
|
after the alert is gone. Open one for the whole story in order: the alert, **who was told** (each notification that was about it — a new alert, a recovery notice, a reminder, an escalation — through which channels, which refused and what they said, or why it was held for quiet hours or not sent because a routing rule says nobody), who acknowledged it and why, the
|
||||||
|
maintenance windows, what was changed on the server, update runs, other alerts about the same thing that
|
||||||
|
began around the same time, and the notes people wrote (operators can add them; the author or an
|
||||||
|
administrator can remove one). Incidents are recorded from the alerts as they are collected and from a look
|
||||||
|
every five minutes, so one that comes and goes between two looks may be missed. An alert is only called
|
||||||
|
resolved after it has been missing for a couple of minutes in a row — thirty when a check couldn't run, and
|
||||||
|
never in the first minutes after the app starts, when server alerts are held back on purpose. Resolved
|
||||||
|
incidents are kept for 400 days. The weekly digest has an *Incidents* section.
|
||||||
|
|
||||||
|
**How big the data is** (Settings → Data → Storage). The size of the database file, its backups and the signed-in sessions, and every table by the room it takes (with its indexes), its rows, how many it gains a day, how long it is
|
||||||
|
meant to be kept and how far back it actually reaches. A history that holds far more than its limit is flagged **not trimmed** (the trimming is not running), the logs are flagged **no limit** until purging is switched on
|
||||||
|
(Settings → Data → Logs), and a history that has not yet grown to its limit is *filling up*. Administrators only; it only measures.
|
||||||
|
|
||||||
|
**Duplicati** (Backups → Duplicati). If you run Duplicati, the app now tells you when a backup **failed** or **has not succeeded for longer than it should** — the usual way a backup tool goes wrong is silently. Duplicati reports to the app (its own `send-http-url` option): you add each machine, get a secret
|
||||||
|
address once, and paste three options into Duplicati's default options; no password for Duplicati is kept and the app never calls it. Every job then appears with when it last succeeded, how the last run went, how long it took, how big the source is and what changed, with a history; a failed run is a warning (critical from the
|
||||||
|
second in a row), an overdue one (2 days by default, per job) a warning (critical at three times that), and maintenance on the machine's server silences them. A run that ends with **warnings** still counts as a good backup, and the page shows **what the warnings said** (on the job and on each run in its history); a job can be set to **alert on warnings** too (off until you switch it on). See INTEGRATIONS.md and docs/MONITORING.md.
|
||||||
|
|
||||||
|
**Nextcloud** (Apps → Nextcloud). Add a Nextcloud integration (its address; an administrator's app password adds more) and the app tells you when people **cannot use it**: **critical** when its database needs upgrading, a warning when it is stuck in **maintenance mode**, and a warning (critical under 5 GB) when the **data disk is low**.
|
||||||
|
The page also shows the users, the files and the app updates waiting. Whether its **background jobs (cron) are running** is read from Nextcloud's own setup checks (Nextcloud 28 or later), and a warning is raised when they are not — a stopped cron would otherwise go unnoticed for weeks. See INTEGRATIONS.md and docs/MONITORING.md.
|
||||||
|
|
||||||
|
**History of integration figures.** The numbers an integration reports — AdGuard's and Technitium's queries and blocked share, the queue and missing items of Sonarr and Radarr, download speeds, Immich's photos and library size, Paperless's documents, Nextcloud's users and free space, n8n's failed runs, Grafana's firing rules, Prometheus's targets down, Authentik's failed sign-ins, Hetzner's monthly cost — are kept every quarter of an hour (and as hours and days beyond five weeks, up to about two years), and each integration's card has **Show the history**: a chart for each figure, 24 hours to a year, with how it changed and its lowest and highest. A stretch when the integration could not be read is a gap, not a flat line. See INTEGRATIONS.md ("History of the figures").
|
||||||
|
|
||||||
|
**Hetzner Cloud** (Infrastructure → Hetzner Cloud). Add a project's read-only API token and the app lists its servers with their status, this month's traffic against the quota, backups, volumes and snapshots, and roughly what it costs a month. It warns when a server that was running is **off**, when one has used **80% of its traffic** (critical at 100%), when **rescue mode** is left on (the next reboot would not come back up), and when **backups** are enabled but late. A server that is also one the app knows by its agent gets a Hetzner card on its own page. It also lists **load balancers** (with the health of their targets), **floating IPs** and **certificates**, warns when a target is unhealthy, a certificate is about to end or a floating IP points at nothing, and **forecasts the month's traffic** ("at this rate web will pass its quota on 24 June"). Nothing at Hetzner is ever changed. See INTEGRATIONS.md and docs/MONITORING.md.
|
||||||
|
|
||||||
|
**Hetzner Robot** (Infrastructure → Hetzner Robot). Hetzner's dedicated servers and storage boxes — Robot is the other Hetzner, next to the Cloud. Add a webservice user (Robot → Settings → Webservice and app settings) and its password and the app lists the account's dedicated servers (product, data centre, addresses, traffic plan, status, whether the rescue system is on, and what is paid until or when a cancelled one shuts down) and its storage boxes. It warns when **a server or storage box is cancelled and about to shut down** (within 30 days; critical within 7 — the data goes with it), when **a storage box is locked**, when **a rescue system has been left on** (two hours — the next boot is into it) and when **a server is still "in process"** a day after it was first seen so. Nothing is reset, ordered or cancelled. See INTEGRATIONS.md and docs/MONITORING.md.
|
||||||
|
|
||||||
|
**Game servers** (Infrastructure → Game servers, for Fragnet's game panel). Add an account's API key and the app lists its game servers: the game, the IP addresses and ports (click one to copy it), whether each is running, its memory and disk against the plan, its backups, schedules and databases. It warns when a server's **newest backup is late** (7 days by default, or none at all — a server can be left out by name) and when **disk or memory is nearly full** (90% by default); a server that is merely stopped is not an alert, since game servers are often stopped on purpose. Operators can **start, stop, restart and kill** a server from the page, and **back it up now** (asked first, written to the audit log) — the panel makes the backup in the background and the page shows it as being made, then done; nothing else at Fragnet is changed. Fragnet's panel is a fork of Pterodactyl, so the same integration reads any Pterodactyl panel (give its address). See INTEGRATIONS.md and docs/MONITORING.md.
|
||||||
|
|
||||||
|
**Media managers** (Apps → Media managers). Add Sonarr, Radarr, Prowlarr, Lidarr and Readarr with their address and API key and the app shows, for each, what the application says is wrong with itself, its download queue (what is stuck or failed, and why), the disks it fills, how many wanted items have no file and, for Prowlarr, which indexers have failed. It warns when an application reports a **warning or error** (a download client that cannot be reached, indexers switched off), when **downloads have been stuck for two hours**, and when a **disk is nearly full**. Nothing is searched, grabbed or changed. See INTEGRATIONS.md and docs/MONITORING.md.
|
||||||
|
|
||||||
|
**Download clients** (Apps → Download clients). Add qBittorrent (its Web UI address, and the user name and password if it asks for them) or SABnzbd (its address and API key) and the app shows, for each, how fast it is downloading, what is in its queue and in what state, how much room its download folder has, what failed in the last day and SABnzbd's own warnings. It warns when **downloads are stuck** (in an error state, or stalled with nothing moving for a day), when qBittorrent has **no connection to the network** for 15 minutes (a VPN that dropped), when SABnzbd's queue is **paused and forgotten** (six hours, with downloads waiting) or **downloads nothing** (an hour, with work waiting — a news server that does not answer), and when a **download disk is nearly full**. Nothing is added, paused or removed. See INTEGRATIONS.md and docs/MONITORING.md.
|
||||||
|
|
||||||
|
**AdGuard Home** (Network → AdGuard Home). Add its address (and sign-in, if it has one) and the app reads it every two minutes — and each time asks its DNS server a real question over UDP, because the web interface can be up while DNS is not. It shows whether DNS answers (and how fast), whether protection is on, the queries and how many were blocked, the names blocked most and the clients asking most, and its filter lists with when each was updated. It raises **DNS that has stopped answering** (ten minutes of it, critical), **a DNS server that is not running** (critical), **protection left off for an hour** and **filter lists that have not updated for a week**. Nothing is changed. See INTEGRATIONS.md and docs/MONITORING.md.
|
||||||
|
|
||||||
|
**Technitium DNS** (Network → Technitium DNS). Add the address of its web console and an API token (a non-expiring one, made under Administration → Sessions) and the app reads it every two minutes — and, like AdGuard Home, asks its DNS server a real question over UDP each time, because the console can be up while DNS is not. It shows whether DNS answers (and how fast), whether blocking is on, how long the server has run, the last day's queries and how they ended (from the cache, resolved, from its own zones, blocked, dropped, failed), the clients and names asked about most and blocked most, the cache against its limit, its block lists and when they are next updated, how its zones are (an expired or unsynced secondary zone is named) and its DHCP leases. It raises **DNS that has stopped answering** (ten minutes of it, critical), **a secondary zone that has expired** (critical) or **cannot sync** (warning), **blocking left off for an hour** and **block lists that are overdue for an update**. Nothing is changed. Technitium's API has no figures for the machine's CPU, memory or disk — a server agent on that machine reports those. Zones and records are connected separately, as a DNS provider. See INTEGRATIONS.md and docs/MONITORING.md.
|
||||||
|
|
||||||
|
**Grafana** (Monitoring → Grafana). Add its address and a service account token (the Viewer role is enough) and the app shows which alert rules are firing or pending, which cannot be evaluated, and which data sources fail Grafana's own health check. It raises **a rule that is firing** (critical when it is labelled critical or page), **a rule that cannot be evaluated** and **a data source that fails its check**. Nothing is silenced or changed. See INTEGRATIONS.md and docs/MONITORING.md.
|
||||||
|
|
||||||
|
**App versions** (Operations → App versions). The version each connected application runs — as its integration last read it — against the newest release of it, with how far behind it is (a patch, a minor update, a major version) and how long the newer release has been out. It looks the newest releases up on GitHub once a day, and **that is the only thing this app asks of an address that is not yours, so it is switched off until an administrator switches it on**. A number of days (0 = never) makes a newer release that has been out that long a warning. Sonarr, Radarr, Prowlarr, Lidarr, Readarr, AdGuard Home, Grafana, Jellyfin, Immich, Paperless-ngx, Nextcloud, NetBox, Authentik and Prometheus are compared. See docs/MONITORING.md.
|
||||||
|
|
||||||
|
**Read now** (Integrations → *Read now* on each integration, and on the card of each integration on its own page — Media managers, Download clients, Hetzner Robot, Hetzner Cloud, Game servers, AdGuard Home, Technitium DNS, Grafana and Speedtest Tracker; operators and administrators). Reads one integration at once instead of waiting for its next turn, and says how it went. An integration that is read on a schedule and stored (most of them) is read the way the schedule reads it, and what it says is judged again at once, so an alert that has cleared clears and a new one appears without waiting. One that is read live (Proxmox, Synology, Tailscale, Gitea, Semaphore, Uptime Kuma, Proxmox Backup Server, osTicket, phpIPAM, Nginx Proxy Manager) is asked whether it answers — the question *Test* asks — which also brings its Working/Failing mark up to date. The same integration is not read again within 20 seconds, since some systems limit how often they may be asked (Hetzner Robot allows 200 requests an hour). See INTEGRATIONS.md.
|
||||||
|
|
||||||
|
**Inspect an integration** (Integrations → *Inspect*, administrators). Runs the integration's reader once, now, and shows what each request asked and got back — the **shape** of every answer (the names and kinds of its fields, never the values) — beside what the reader made of it and the fields that came out empty. It is how an integration written from documentation is checked against the real system: an answer with a field the reader did not expect shows up as an empty field. *Copy the report* gives plain text that can be pasted into a message. Each inspection is written to the audit log; nothing is stored or changed.
|
||||||
|
|
||||||
|
**Grouping alerts.** On the Alerts page a *Group by* choice puts the alerts under a header for each source (Hetzner, Sonarr …), kind of problem (Disk, Backups …) or type of alert (a Hetzner certificate about to end …) — worst group first, each header showing how many and how serious, each group folding away, and a tick on the header picking everything in it for a bulk action. Remembered per browser; not grouped is the default.
|
||||||
|
|
||||||
|
**Speedtest** (Monitoring → Speedtest). Add a Speedtest Tracker integration (its address and an API token with the *results:read* ability) and the app charts how fast the internet has been — download, upload and ping over up to 90 days — and, if you tell it what the line is sold at, warns when **the middle of the last three tests is under 60%** of it, and when **the tests keep failing**. One slow test is not an alert. The app never starts a test. On a status page the integration reads as the internet. See INTEGRATIONS.md and docs/MONITORING.md.
|
||||||
|
|
||||||
|
**NetBox** (Monitoring → NetBox). Add a NetBox integration (its address and a read-only token): a warning when **no background worker** is running (they are a separate container in the Docker image, and can be down while the pages load), and a list on the page of **the servers you run that have no address in NetBox**. That list is
|
||||||
|
not an alert; documentation that is behind is not an outage. See INTEGRATIONS.md and docs/MONITORING.md.
|
||||||
|
|
||||||
|
**n8n** (Automation → n8n). Add an n8n integration (its address and an API key): a warning for each **workflow that is switched on and failed on its last run** — an automation that has quietly stopped — which clears when it next succeeds. The page lists the workflows with their last run and the failures of the last day. See
|
||||||
|
INTEGRATIONS.md and docs/MONITORING.md.
|
||||||
|
|
||||||
|
**Immich** (Apps → Immich). Add an Immich integration (its address and a read-only API key) and the app watches the thing you cannot recreate: it warns when the **disk under the photo library** is filling (critical at 95 %) and when **background jobs have failed** — the thumbnails and metadata that Immich
|
||||||
|
quietly never made. The page shows the photo and video counts, what the library takes and each job queue. See INTEGRATIONS.md and docs/MONITORING.md.
|
||||||
|
|
||||||
|
**Paperless-ngx** (Apps → Paperless). Add a Paperless-ngx integration (its address and a superuser's API token) and the app tells you when it has stopped taking documents in: **critical** when the database, Redis or the **worker that consumes documents** is in error, and a warning for **documents that failed to import**
|
||||||
|
(named, with the error), a failing index or sanity check, and a filling disk. The page shows the documents, the inbox and Paperless's own system status. See INTEGRATIONS.md and docs/MONITORING.md.
|
||||||
|
|
||||||
|
**Prometheus** (Monitoring → Prometheus). Add a Prometheus integration (just its address) and what you already measure there joins the one list of what is wrong: a **scrape target that stops answering** (five minutes, grouped by job — a node exporter on each host, say) and **alerts that Prometheus's own rules are firing**
|
||||||
|
(critical when labelled so). The page shows targets by job and what is down or firing. Alerts managed by Grafana itself are not seen. See INTEGRATIONS.md and docs/MONITORING.md.
|
||||||
|
|
||||||
|
**Authentik** (Apps → Authentik). Add an Authentik integration (its address and an API token) and the app watches the sign-in in front of everything else: it raises a **critical** alert when an **outpost** that guards applications stops reporting, and warns when a **certificate** or an **API token** is
|
||||||
|
about to run out (the token it reads Authentik with included), when **no worker** is connected, and when **sign-ins are failing** or flows are misconfigured. The page lists the outposts, what runs out soonest and the failed sign-ins of the last day. An Authentik you are not using for the moment can have its **certificate alerts switched off** (a tick box on the integration; the page still lists them). See INTEGRATIONS.md and docs/MONITORING.md.
|
||||||
|
An administrator also has a second tab there, **Users and groups**, and can **make a user** on it — with a second, optional token for it — and gets a **one-time recovery link** to send them, so they choose their own password; no password is ever set or seen here, and the link is shown once and not kept. The same page lists the users and lets an administrator **edit** one (name, email, groups), **switch one off** (and on again) or **delete** one (the user name has to be typed); service accounts, the account the token belongs to and the last active administrator are refused. **Groups** can be made (empty, and never as a group that makes administrators), renamed, deleted (the name has to be typed; never one that makes administrators) and have users put in and taken out (whether a group makes administrators is not changed here); a group that decides roles in this app is marked and asks first.
|
||||||
|
|
||||||
|
**Jellyfin** (Apps → Jellyfin). Add a Jellyfin integration (its address and an API key) and the app shows whether it is up, **who is streaming what** and whether it is being transcoded, the scheduled tasks that failed, and a chart of how many watched at once; it warns when Jellyfin **wants a restart**, a
|
||||||
|
**task failed** or **sign-ins are failing**. And when the Jellyfin runs on a server the app manages, **an update run asks first if people are watching**, and a patch plan **waits for them (up to two hours)** instead of restarting the machine in the middle of a film. See INTEGRATIONS.md and docs/MONITORING.md.
|
||||||
|
|
||||||
|
**Database backups** (Settings → Data → Backup). A copy of Homelab Manager's own database is taken every day (03:30
|
||||||
|
in the notification time zone, the newest 14 kept; on unless you turn it off), checked as it is made, and
|
||||||
|
kept in `data/backups/`. The page lists them with **Download**, **Check**, **Restore…** and **Delete**,
|
||||||
|
makes one now (**Back up now**), takes one from somewhere else (**Upload a backup…**) and sets the schedule.
|
||||||
|
A restore is checked first, then staged, and takes effect the next time the app starts — restart it
|
||||||
|
(`docker compose restart homelab-manager`); the database that was in use is kept as a "before a restore"
|
||||||
|
backup, so a restore can be undone. A backup made by a *newer* version than the one running is refused.
|
||||||
|
If the newest backup gets more than two days old, or a scheduled one fails, it is an alert (and a message).
|
||||||
|
**A backup does not contain `CREDENTIALS_ENCRYPTION_KEY`**, which decrypts the credentials stored in it,
|
||||||
|
and it is still on the same machine until you copy it away: see [BACKUP.md](BACKUP.md).
|
||||||
|
|
||||||
|
**Patch plans** (Updates → *Patch plans*). "Sundays at 02:00, the servers tagged `lab`, one at a time, stop at the first failure." An administrator makes a plan: which days
|
||||||
|
and what time (on the clock of a time zone, so 02:00 stays 02:00 in summer and winter), which servers (everything with any of the tags chosen, and any servers named), and the Semaphore
|
||||||
|
playbook that applies updates (and, if you like, a second one that restarts a server which says a restart is due once its updates are in — see below). When it is due — or an operator presses **Run now** — it goes down the list of servers by name, **one at a time**: each is updated by an ordinary update run
|
||||||
|
(below), and the plan waits for it to finish before it starts the next. A server with nothing waiting is left alone (unless the plan is told not to mind), and what **depends on** a server
|
||||||
|
(a guest that runs on it, a service that needs it) is silenced for the same time and let go when it is done. If a server's update fails and the plan says to stop — the default — the servers
|
||||||
|
after it are left untouched and the run says where it stopped; otherwise it carries on and says at the end that something failed. One message goes to your channels when a run ends, with a line for
|
||||||
|
each server. A plan starts once a day at most (the day it last started is kept), up to half an hour late if the app was down at its time — later than that, that day is skipped rather than
|
||||||
|
surprising anyone mid-morning — and never overlaps itself. A run that is going is followed through a restart of this app, and can be stopped (the servers not yet started are left alone; a playbook
|
||||||
|
already started keeps running in Semaphore). Plans appear in **Coming up** as maintenance, with how many servers they update, and the runs in the change log and the monthly report. Everyone can see
|
||||||
|
plans and how they went; administrators make, change and remove them; operators run and stop them.
|
||||||
|
|
||||||
|
**Update runs** (Updates → Update…, operators). Apply a server's waiting updates in
|
||||||
|
one go: pick the Semaphore playbook that does it, and Homelab Manager puts the server
|
||||||
|
under maintenance (so nothing alerts while it works and restarts), runs the playbook
|
||||||
|
limited to that one server, follows it, waits for the server's agent to report again,
|
||||||
|
and then says how many updates are still waiting and whether a restart is due before
|
||||||
|
ending the window it made. The runs are listed on the Updates page and in the server's
|
||||||
|
history, survive a restart of this app, and end as *failed* (with the window ended, so
|
||||||
|
the alerts can speak) if the playbook fails, Semaphore stops answering, the server
|
||||||
|
doesn't report again within 30 minutes, or it takes more than three hours. "Stop
|
||||||
|
following" ends the window and stops watching; it can't stop a playbook already running
|
||||||
|
in Semaphore. A message goes to your channels when a run ends (a toggle under *What to
|
||||||
|
send*).
|
||||||
|
|
||||||
|
**Restarting after updates.** The update drawer has a second choice, *Restart it afterwards, if it says a restart is due, with* — a Semaphore playbook that only restarts the
|
||||||
|
server (in the same Semaphore as the update's). When the updates are in and the server reports again saying a restart is due, that playbook is run for the same one server, the maintenance
|
||||||
|
window is kept up for it (at least 45 more minutes), and the server is **waited for again and checked**: it must have started again, which is read from the uptime its agent reports (its start is
|
||||||
|
the report's time minus its uptime; a start a couple of minutes before the restart began still counts) — or, where an agent reports no uptime, from its *restart needed* flag clearing. A playbook that
|
||||||
|
finished while the server did not restart is a *failed* run that says so ("check that the playbook really restarts it"); a restart playbook that fails, or can't be started, ends the run saying the
|
||||||
|
updates are in and the server still needs a restart. Without a second playbook nothing changes: the run says a restart is still needed, and an update playbook may restart the machine itself, as before.
|
||||||
|
A server that only needs a restart (nothing waiting, *restart needed* showing) has a **Restart…** button on the Updates page that does the restart on its own, with the same checks. A **patch plan** can
|
||||||
|
carry the restart playbook too: each server is updated and, if it then says so, restarted before the plan goes on to the next; a server with nothing waiting that still needs a restart is restarted instead
|
||||||
|
of left alone; the plan's maintenance time counts for the update and the restart together. Operators start them; the restart is in the server's history and the audit log, and the message at the end
|
||||||
|
says "the server was restarted and is back".
|
||||||
|
|
||||||
|
**Host health and automatic agent updates.** The Linux agent (version 4) also reports services that have
|
||||||
|
failed, each disk's SMART verdict, temperature and wear, and ZFS pool state. They show on a server's page
|
||||||
|
(*Host health*) and become alerts — a failed service is a warning under its own kind, a failing disk or a pool that isn't
|
||||||
|
ONLINE is critical — with the same maintenance silencing, notifications and runbook links as other alerts. Disk space
|
||||||
|
heading for full was already forecast from each server's history (alert, weekly digest, trend note); `/metrics`
|
||||||
|
now carries it too as `homelab_server_disk_days_to_full`. **Servers → Update agents automatically** (off by default) lets
|
||||||
|
an agent that is behind download the newer script from here, check it against a SHA-256 the server gives, and replace itself.
|
||||||
|
How it works and what it can't do: [AGENT_UPDATE.md](AGENT_UPDATE.md).
|
||||||
|
|
||||||
|
**Runbooks.** Write down what to do when something is wrong — "if elsa is down,
|
||||||
|
check the UPS first, then…" — on a server's page (the Runbooks card) or an
|
||||||
|
integration's card. A note can be for anything about the thing, or for one kind of
|
||||||
|
alert (a disk filling up). On the Alerts page an alert about a server or integration
|
||||||
|
shows a Runbook link when a note fits it, and opens it in a side panel with the notes
|
||||||
|
that fit that alert first. A note can also be written once for a whole **kind** of alert — the *What to do* link beside an alert, "when a Hetzner certificate is about to end: renew it in the console, then…", with an optional page to open — and then shows beside every alert of that kind, whatever it is about. Everyone signed in reads them; operators write them. The
|
||||||
|
text is plain: a blank line is a new paragraph, `- ` a list, ``` around lines a block
|
||||||
|
of commands, backticks inline code; links become clickable, and nothing typed in a
|
||||||
|
note can run as HTML.
|
||||||
|
|
||||||
|
**Endpoint checks** (Monitoring → Endpoints). The app asks a **web address**, a **TCP port** or a **TLS certificate** whether it is fine, every few minutes (each
|
||||||
|
check has its own interval, from 1 minute), and remembers how it went: the answer time, uptime over 24 hours and 7 days, a row of the recent results, and — for https
|
||||||
|
addresses and certificate checks — when the certificate runs out. A web check wants a good status (200–399 unless you say otherwise, like `200-204, 401`) and can
|
||||||
|
also want a word in the page, or **a value in a JSON answer within its limit** — `power.now` below 5000, `state` equal to playing, a Prometheus query's result above 0 — with an optional header (a token) sent with the request;
|
||||||
|
the value read is shown, charted under *History…*, and in `/metrics`; a certificate that isn't trusted (self-signed, a private authority) fails unless you tick *Accept a certificate that isn't trusted*, and
|
||||||
|
its expiry is watched either way. A check that has **failed several times in a row** (2 by default, so a blip isn't) is a *Monitoring* alert and one notification, with
|
||||||
|
another when it answers again; a certificate within its warning time (14 days by default; critical in the last three) is an *Expiry* alert. A check can be linked to a
|
||||||
|
server, so maintenance on it silences the check's alerts and the server's tags can route them. Everyone can look; administrators set checks up (each one makes this
|
||||||
|
server connect somewhere — never to itself or the cloud metadata address, though your own network is fine) and operators can ask *Check now*. They also appear on the
|
||||||
|
dashboard and in `/metrics`. Results are kept for two weeks, then as hours (four months) and days (years); "History…" on a check draws them.
|
||||||
|
|
||||||
|
**Suggested checks** (on the Endpoints page, administrators). From what the app already knows it offers checks you could add with a click: a **web port a server's agent
|
||||||
|
sees open** (80, 443, 8080, 8443 — when a server has both the plain and the secure one, the secure one, since it also reads the certificate), under the name DNS gives that
|
||||||
|
server if there is one (an A or AAAA record with one of its addresses, or an alias to it), otherwise its own hostname or address; and a **secret that already reads a
|
||||||
|
certificate**, since a check also hears when the service stops answering; and, with Dockhand connected, **the apps in your containers**, recognised from their images and checked on each one's own health address with what a good
|
||||||
|
answer looks like — Immich's ping must say `pong`, Nextcloud's `status.php` must not be in maintenance mode, Gitea's health must say `pass`, Grafana's database must be `ok`, Authentik's health endpoint must answer, and so on for about forty
|
||||||
|
apps (the ones with no open health endpoint get a plain "the page answers"). They are asked on the Docker host's address (the server the app knows by that name) at the port the container publishes. Each is linked to its server. Something a check already covers is never offered, **Not for me**
|
||||||
|
stops one being offered (and they can be brought back), and nothing is asked until you add it. Uptime Kuma's monitors are not offered: they are already being watched, and
|
||||||
|
the adapter only keeps a host and port for them, not the address.
|
||||||
|
|
||||||
|
**Heartbeats** (Monitoring → Heartbeats). "Did my 3 a.m. job actually run?" An administrator makes a heartbeat for a job — how often it should report, and how late is
|
||||||
|
still fine — and gets an address with a long random token in it. The job calls it when it finishes (`curl -fsS -m 10 -o /dev/null <address>` after the backup in cron);
|
||||||
|
the app writes that down and shows when the next call is due, the run times and the last runs as a row of ticks. A call that does **not come in time** is a *warning*
|
||||||
|
alert and one notification (the same bookkeeping as everything else: said once, again as cleared when the call comes); a job that calls `<address>/fail` — it ran and went
|
||||||
|
wrong — is a *critical* alert at once, with what it said. `/start` before the job lets the finishing call say how long it took; anything sent with a call
|
||||||
|
(`--data-raw "3.2 GB copied"`, or `?msg=`) is kept as a note, up to 200 characters. A heartbeat can be linked to the server the job runs on, so maintenance there silences
|
||||||
|
it and the server's tags route it. Everyone can look; only administrators see the addresses, make, change and remove heartbeats, or give one a **new address** if it got out.
|
||||||
|
The address needs no sign-in (it is a token, like an agent's), and all it can do is say "I'm alive": it can't read anything. Calls are limited per address, and an
|
||||||
|
unknown token and a wrong path are answered alike. They also appear on the dashboard and in `/metrics`; the last two weeks of calls are kept. The format is in [docs/HEARTBEATS-AND-CALENDAR.md](docs/HEARTBEATS-AND-CALENDAR.md).
|
||||||
|
|
||||||
|
**Coming up** (Operations → Coming up, and a card on the dashboard). Everything with a date ahead in one list, nearest first: secrets, domains and certificates that
|
||||||
|
expire (including the ones endpoint checks see), Tailscale keys that are expiring, hardware warranties and end of support, planned and **recurring** maintenance, restore
|
||||||
|
tests that are due (when a time limit is set), the day a **bill** falls (Costs), and — as one line — updates that have been waiting. What has already passed in the last 30 days is shown as past, under
|
||||||
|
*Needs attention now*. Narrow it by kind, server and how far ahead (up to a year); the filters are in the address, so a view can be linked. A schedule that repeats weekly is one line (its next time) unless
|
||||||
|
you ask for every one. An administrator can make a **calendar feed** (Settings → Share links → *A calendar feed*): an address for a phone's calendar to subscribe to, refreshed
|
||||||
|
every few hours, with each day as an all-day event and maintenance with its start and end. A feed made with *kinds only* says only "A secret expires", with no names, so it is fine on a shared
|
||||||
|
calendar; *with names* also gives the name and a link back. Like any share link the address is shown once and can be removed. Tailscale keys are asked for at most every
|
||||||
|
ten minutes, and only those the integration reports as expiring soon are listed.
|
||||||
|
|
||||||
|
**Proxmox backups** (Backups → Proxmox backups) is one page with two tabs: *Backup coverage* and
|
||||||
|
*Proxmox Backup Server* (the integration's datastores and verification; the old `/pbs` address goes to
|
||||||
|
that tab). **Backup coverage** answers "what isn't backed
|
||||||
|
up?" for every Proxmox guest: is a backup job covering it, when was it last backed up
|
||||||
|
(from the newest snapshot on a Proxmox Backup Server, or the last successful Proxmox
|
||||||
|
backup task that names the guest), and did that go well — Recent backup, Stale (older
|
||||||
|
than 1, 2, 3, 7 or 14 days, your choice), Failed, No backup, or Unknown (a job covers it
|
||||||
|
but nothing says when it last ran, typical when backups go somewhere other than a
|
||||||
|
Backup Server — never guessed at). Worst first, searchable, with CSV, and the weekly
|
||||||
|
digest lists what needs a look. A guest whose newest backup is older than **Settings →
|
||||||
|
Notifications → "A guest's backup is too old after"** (48 hours by default; 0 is off) is
|
||||||
|
also an alert of the Backups kind — a warning, critical at three times the limit — with a
|
||||||
|
runbook link like any other, sent once a day with the other backup messages (under the
|
||||||
|
Proxmox backup toggle) and counted in `/metrics` (`homelab_backup_guests`,
|
||||||
|
`homelab_backup_last_age_seconds`). A server that is a guest can have its own limit
|
||||||
|
(a weekly backup is fine at eight days). Failed backups already raise their own alerts —
|
||||||
|
the latest failing run on a node, and snapshots that fail verification — so those aren't
|
||||||
|
repeated per guest. The page's own "Stale after" selector is only a view.
|
||||||
|
|
||||||
|
**Alert limits per server.** On a server's page an administrator can give it limits of
|
||||||
|
its own — offline after, disk usage percentage, the disk-forecast window, and (for a
|
||||||
|
Proxmox guest) how old its backup may be — and per disk, either a limit or "no alerts
|
||||||
|
for this disk". Whatever is left empty follows Settings, so changing the global limit
|
||||||
|
still reaches every server that hasn't said otherwise. They apply everywhere the global
|
||||||
|
ones did: the alerts, the notifications, the dashboard, the map, the forecast, the
|
||||||
|
digest and `/metrics`. The same place keeps the **services to watch** on that server — only a service you list there raises an alert when it has failed, so the many that don't matter stay quiet; the list can also be set from its
|
||||||
|
Host health card (a failed service that isn't watched shows there with a **Watch** button; `*` watches them all); and a card, **What the agent couldn't collect**, tells you why a figure is missing (no `smartctl`, an update check that didn't answer …) and what
|
||||||
|
to do. See AGENT_UPDATE.md.
|
||||||
|
|
||||||
|
**Inventory** (Reference → Inventory) keeps the hardware behind it all: name and
|
||||||
|
kind, maker, model, serial, where it sits (and rack unit), when it was bought and for
|
||||||
|
how much, when its warranty ends, its typical power draw, and which server it is.
|
||||||
|
Summary cards show how much is under warranty (and what ends within 90 days), the total
|
||||||
|
draw, and — with a price per kWh set at the foot of the page — a rough yearly running
|
||||||
|
cost. Everyone signed in can look; operators keep it up to date; the weekly digest
|
||||||
|
mentions a warranty ending within 60 days, or support ending within 90.
|
||||||
|
|
||||||
|
**Lifecycle planning** (Inventory → Lifecycle). Give a piece of hardware an **end of support** (when the maker stops patching it, or when you plan to retire it) and a
|
||||||
|
**replacement cost**, and the Lifecycle page lists what ages out, year by year for the next 3, 5, 7 or 10 years, with what replacing each year's hardware is expected to
|
||||||
|
cost (the total says how many have no figure, so it is "at least" that). What is already past its date is listed first, and what is further off, or has no date, is kept
|
||||||
|
apart. Until a piece has an end of support, the end of its warranty stands in — and the page says so — and until it has a replacement cost, what it was bought for does.
|
||||||
|
CSV export; the weekly digest also mentions support ending within 90 days.
|
||||||
|
|
||||||
|
**Empty and loading lists.** A list with nothing in it says what is missing, why that is, and — when you can — the one thing to do about it (a button that opens the right form, or
|
||||||
|
a link to the page that fills it). A list that has things but not the ones you searched for says so, with a button that clears the search and filters. While a list is on its way
|
||||||
|
it shows grey placeholder lines rather than looking empty.
|
||||||
|
|
||||||
|
Entries that are a server also pick up what its agent says the machine is: when you link an entry to a server (or the
|
||||||
|
agent reports later), a blank maker, model or serial number is filled in from the firmware — never over what a person wrote.
|
||||||
|
When the agent says something different, the row offers **Refresh from agent**, which asks before replacing. **Print labels**
|
||||||
|
(on the Inventory page) makes a sheet of stickers — a QR code, the name, what it is, its serial number and where it sits, in
|
||||||
|
three sizes — and scanning a code opens that entry (`/inventory?asset=…`) for whoever is signed in. The codes are drawn in the
|
||||||
|
browser; nothing about them leaves the page.
|
||||||
|
|
||||||
|
**Roles from groups** (Settings → Sign-in, off until switched on): name the groups
|
||||||
|
that mean administrator and operator, and at each sign-in a person's role follows the
|
||||||
|
groups the identity provider (Authentik) reports; everyone else becomes a viewer, or
|
||||||
|
keeps what they have. "Check" shows what each person would become from the groups they
|
||||||
|
had last time, before you switch it on. Safeguards: a sign-in with no groups at all (a
|
||||||
|
wrong claim name) leaves roles alone, the only administrator is never demoted by a
|
||||||
|
group rule, the first person ever is still the administrator, and while it is on the
|
||||||
|
Users page shows roles instead of letting you set them. The Users page also shows the
|
||||||
|
groups each person had at their last sign-in.
|
||||||
|
|
||||||
|
**Profile pictures** (Settings → Sign-in, both sources off until switched on). Without one a person is their initials. An administrator can allow the picture the
|
||||||
|
identity provider names in its `picture` claim (kept at each sign-in; only an address on Authentik's own host or on Gravatar is ever used, so a claim can't make the
|
||||||
|
server fetch just anywhere) and/or **Gravatar** (by a SHA-256 hash of the person's email — never the address itself). The picture from the provider is tried first.
|
||||||
|
Either way it is **the server** that asks: it checks what comes back (a small PNG, JPEG, WebP or GIF — never SVG), keeps it for a day, and serves it from `/api/avatars/<id>`
|
||||||
|
to anyone signed in, so people's browsers never contact Gravatar or Authentik for it, and a person with no picture, or a host that can't be reached, simply shows initials.
|
||||||
|
The Privacy page says when it is on. The picture shows in the user menu and on the Users page.
|
||||||
|
|
||||||
|
**Prometheus and webhooks** (Settings → API and webhooks). `/metrics` answers in
|
||||||
|
Prometheus's text format — servers up or not, load, memory, disks, waiting updates,
|
||||||
|
agent versions, alert counts, which integrations work, maintenance, what is about to
|
||||||
|
expire — to a request with a metrics token (a bearer token made on that page, shown
|
||||||
|
once, only its hash kept). Event webhooks send a signed JSON message to a URL when an
|
||||||
|
alert begins or ends or maintenance starts or finishes, for Home Assistant, n8n or a
|
||||||
|
status board; **Sent** on a webhook lists the last 100 messages to it (none older than 14 days) with how each went and the message itself, and **Send again** sends one exactly as it was, same id and signature. Both are described, with the exact series and message formats, in
|
||||||
|
[docs/PROMETHEUS-GRAFANA.md](docs/PROMETHEUS-GRAFANA.md) and [docs/WEBHOOKS.md](docs/WEBHOOKS.md).
|
||||||
|
|
||||||
|
**Service levels** (Alert settings → Service levels). Say how available something should be — **99.9 % for Gitea** — and see what is left of this month's **error budget**: the downtime the target allows (99.9 % of a 31-day
|
||||||
|
month allows about 45 minutes; 99 % about 7 hours). A target is set on an **endpoint check** (measured from its answers, day by day), a **server** or an **integration** (measured from when the audit log recorded it going away and
|
||||||
|
coming back, to the minute), and is measured over calendar months in the notification time zone. Each card shows the figure so far, the downtime used, what is allowed and what is left, how much of the budget is used (and where the
|
||||||
|
pace so far would end the month), and how each of the last six months went. When **80 %** of the budget is used it is a warning, and when it is overspent a critical alert, until the month turns over — switchable per target. Planned maintenance is
|
||||||
|
**not** taken out: down is down. A target with nothing recorded for the period says so rather than guessing. `/metrics` has the target, the figure and the minutes left; administrators set targets, everyone sees them.
|
||||||
|
|
||||||
|
**Monthly, quarterly and annual reports** (Records → Reports; and, if switched on, a message on a day of the month). How a calendar month went, on one page that can be read, printed and copied as text:
|
||||||
|
**availability** (servers, from when each stopped reporting and reported again; endpoints, from every check's answers, kept day by day for years; integrations), **incidents** (how many began and
|
||||||
|
ended, how long they took, the longest, by kind, and which are still open), **updates** (update runs and patch plan runs, how many updates were installed, by server), **restore tests** recorded in the
|
||||||
|
month and the **backups as they are now**, and what was **renewed** (a certificate or a domain whose expiry moved clearly later — noticed when the app re-reads the date). The month is on the clock of the notification time zone;
|
||||||
|
the current month is "so far". It can only say what was recorded: endpoint availability starts from the last two weeks of results when the daily figures were added, renewals from when the app began noticing them,
|
||||||
|
and the backups are a snapshot, not a history. The same page also gives a **quarter** (Q3 2026) or a **year** — the same figures over longer, with a **Service levels** section saying how each target did against the downtime it allowed over the whole stretch — chosen with the *Month · Quarter · Year* buttons. Settings → Notifications → *Monthly report* sends last month's report on a day (1–28) and time of your choice (off until switched on), with *Preview* and *Send now*; two more boxes add the **quarterly** report after the month that ends a quarter (March, June, September, December) and the **annual** one after December.
|
||||||
|
|
||||||
|
**Weekly digest** (Settings → Notifications, off until switched on) sends one message a
|
||||||
|
week through your notification channels: what is wrong now, what was down and for how
|
||||||
|
long, what happened, disks heading for full, things about to expire, updates waiting
|
||||||
|
and maintenance coming up — each topic only if it has something to say, and "a quiet
|
||||||
|
week" when nothing does. "Preview" shows what it would say right now and "Send it now"
|
||||||
|
sends it at once. Details in [NOTIFICATIONS.md](NOTIFICATIONS.md).
|
||||||
|
|
||||||
|
**Agent versions.** Each agent script says its version in every report, and Homelab
|
||||||
|
Manager compares it with the scripts it serves, so a server whose agent is behind is
|
||||||
|
marked on the Servers page, on its own page and on the Updates page (which counts them).
|
||||||
|
How to update one: [AGENT_UPDATE.md](AGENT_UPDATE.md).
|
||||||
|
|
||||||
|
**Customising the dashboard.** Customise (top of the dashboard) lets you move each
|
||||||
|
Overview card earlier or later among its neighbours and hide the ones you don't use —
|
||||||
|
hidden cards stay listed, dimmed, while you arrange so they can be brought back,
|
||||||
|
and Reset returns the default. The cards at the top (*Needs attention*, *My work*,
|
||||||
|
*Since you last looked*, *Getting started* and the *To do list*) can be hidden the same
|
||||||
|
way, each under a bar with its name and a Hide / Show button (they are not moved: the top
|
||||||
|
is two columns). Hide every card on one side of the top and the other takes the full width;
|
||||||
|
hide all of them and the row is gone and the Overview moves up. It is saved with your user,
|
||||||
|
so it follows you to any browser, and it is yours alone.
|
||||||
|
|
||||||
|
**Comparing servers.** On the Servers page, Compare lets you pick two to four
|
||||||
|
servers and open their load, memory or a disk on one chart (24 hours, 7 or 30
|
||||||
|
days), each line with its own colour and dash pattern and a table of now, average
|
||||||
|
and peak beside it. Servers that don't report through an agent have no history
|
||||||
|
to draw. The Servers page also has a search box and a "not reporting /
|
||||||
|
reporting" filter, and the dashboard links to the not-reporting ones.
|
||||||
|
|
||||||
|
**Share links** (Settings → Share links, administrators only) make an address
|
||||||
|
that opens a read-only wall display **without signing in**, for a TV or someone
|
||||||
|
who only wants to know if things are up. Each link has a name, how long it works
|
||||||
|
(until removed, 7 days … a year) and how much it shows: *numbers only* (servers
|
||||||
|
online, alert counts, what is in maintenance, integrations failing — never a
|
||||||
|
name) or *with names* (also the text of the alerts, what is in maintenance and
|
||||||
|
the integrations' names — never addresses, tokens or settings). The picture is
|
||||||
|
built on the server, so nothing else leaves it. The secret in the address is
|
||||||
|
shown once; only a hash is stored, a link that never existed, ended or was
|
||||||
|
removed answers the same way, and making and removing links are in the audit log
|
||||||
|
(without the secret). It is as private as the address: remove it when it isn't
|
||||||
|
needed.
|
||||||
|
|
||||||
|
**Costs** (Records → Costs). What the homelab costs, in one place: the domains, VPN, licences and subscriptions you type in (an amount per month or per year, a currency, a kind, and optionally the day it is billed) and a Hetzner project's estimate, read by itself. Each currency has its own monthly and yearly total and a split by kind — none is converted into another. A day a bill falls on is an anchor: the next time it falls (the same day next month, or next year) is worked out and shows in **Coming up** as *Renewals*. Everyone signed in sees it; an administrator changes it.
|
||||||
|
|
||||||
|
**Electricity** (the *Electricity* card on Records → Costs). The typical power draw you enter for each piece of hardware in the **Inventory**, the **price of a kWh** and the **currency** set under Inventory (Running cost) give an estimate: the draws added up, run all day every day (730 hours a month), at one price — shown as watts, kWh and a cost per month and per year, with **each device’s own cost and share, biggest first**, and how many pieces of hardware have no draw entered. When there is a draw, a price and a currency that is a **three-letter code** (SEK, EUR …) it is also **counted in the totals** above under its own *Electricity* category, listed with the other things that are read, and kept in the cost history; with a currency typed as “kr” it is worked out and shown but not counted (the totals are per ISO currency), and without a price the card says what to set. It is an estimate of what the hardware draws, not a reading from the meter.
|
||||||
|
|
||||||
|
**Cost history** (the *History* card at the bottom of Records → Costs). What the homelab cost **in each past month**, per currency (6 to 36 months back): a bar for each month — the items you typed in, and the darker part for Hetzner's estimate — and beneath it the figure for each month and how much it moved from the month before. A month's figure is the monthly amount of everything that was in force at the month's end (a yearly bill counts a twelfth, as the totals do; the month that is not over is as of now), so an item that was dearer last spring, or has since been removed, is in the months it was there. Under it, **what changed**: items added, changed (*what it was → what it is*, and a rename or a new kind) and removed, and the day a Hetzner estimate moved from one figure to another. It starts when the app began keeping it: an item that was there before is shown at its current amount from the day it was added (what it was earlier is not known), and Hetzner's estimate is written down once a day, so it is kept from the first day it was. Each item's earlier amounts are kept even after the item is removed. Everyone signed in sees it.
|
||||||
|
|
||||||
|
**Status pages** are the other thing a share link can open: a page for family or housemates
|
||||||
|
that answers "is the movie server up?" without a sign-in and without anything that isn't
|
||||||
|
theirs to see. Pick **A status page** when making a link, then choose what is on it and what
|
||||||
|
each thing is called ("Movies and TV", "Family photos"): each is a server (reporting, or not
|
||||||
|
— judged by its own limits), an integration (its latest calls work, or don't) or an Uptime
|
||||||
|
Kuma monitor. It says *Working*, *Having trouble* (it answers, but has an alert open that nobody has silenced, snoozed or closed — a disk filling, a stuck queue, an internet line slower than it is sold at; the alert itself is never shown), *Not working*, *Being worked on* (under a maintenance
|
||||||
|
window) or *Can't tell right now*, with the 30-day figure where one is known, a heading and
|
||||||
|
an optional message ("Movies are being updated tonight"). The page follows the device's
|
||||||
|
light or dark setting and looks again every 30 seconds. Whoever has the address is given only
|
||||||
|
the names you chose and those states — no addresses, no ids, no alert text, nothing about
|
||||||
|
what the thing really is — and a status page's address doesn't open the wall display (or the
|
||||||
|
other way round). Uptime Kuma is asked at most every 30 seconds however many people look. Up
|
||||||
|
to 30 things per page.
|
||||||
|
|
||||||
|
**A Grafana dashboard** for the `/metrics` series: Settings → API and webhooks → *Download
|
||||||
|
the dashboard*, then in Grafana Dashboards → New → Import and pick your Prometheus when
|
||||||
|
asked. It has rows for what needs attention, servers (with a server picker), disks (usage,
|
||||||
|
filling up, SMART health, SSD wear, temperature, ZFS pools), backups (guests by state, age
|
||||||
|
of each newest backup, the app's own backup) and secrets, power and the app itself. It is
|
||||||
|
built from the same code as `/metrics`, and a test checks that it only asks for series that
|
||||||
|
are really served. See [docs/PROMETHEUS-GRAFANA.md](docs/PROMETHEUS-GRAFANA.md).
|
||||||
|
|
||||||
|
**Since you last looked.** The dashboard opens with a card on what happened
|
||||||
|
since you last pressed "Got it" (the first time, the last day; never more than
|
||||||
|
two weeks): servers that stopped reporting or filled a disk, services that
|
||||||
|
stopped answering, and maintenance that began or ended — each problem and its
|
||||||
|
recovery folded into one line ("elsa was offline for 12 min — reporting again";
|
||||||
|
a problem that hasn't ended says "still offline"). Operators and admins also see
|
||||||
|
how many changes other people made, with a link to the audit log. Nothing to say
|
||||||
|
means no card. The mark is stored on your user, so it follows you to any browser.
|
||||||
|
|
||||||
|
**Uptime ribbons.** Servers (list, cards and the server's page) and integration
|
||||||
|
cards show a strip of one bar per day for the last 30 days — green for a day with
|
||||||
|
nothing wrong, amber for some time down, red for half a day or more, grey where
|
||||||
|
nothing is known — and the share of time up. It is worked out from the moments
|
||||||
|
the health checks write in the audit log (a server stopped reporting / reported
|
||||||
|
again; an integration's calls started failing — the same threshold as the failure
|
||||||
|
alert — / worked again), so it can only be as old as those records: servers go
|
||||||
|
back to when they were added, integrations to the first time this version ran.
|
||||||
|
Integrations are tracked per kind of service, as the health checks are.
|
||||||
|
|
||||||
|
**Maintenance timeline.** The Maintenance page draws the running and scheduled
|
||||||
|
windows along a time axis (the next 24 hours, 7 days or 30 days), one row per
|
||||||
|
server, integration or DNS provider and a top row for "anything in
|
||||||
|
maintenance", so overlaps and gaps are visible. Windows mirrored from Uptime
|
||||||
|
Kuma are striped. The tables below still list the same windows.
|
||||||
|
|
||||||
|
**Times as people say them.** "Last seen", sync times, session activity and
|
||||||
|
similar columns read "5 min ago" and show the exact date and time, in the format
|
||||||
|
chosen under Settings → Display, on hover. Exports and the audit log keep exact
|
||||||
|
times.
|
||||||
|
|
||||||
|
**Activity.** Operators and admins see an Activity card on each server's page:
|
||||||
|
what changed on it and who changed it, the moments the health check saw it stop
|
||||||
|
reporting (and report again) or a disk fill (and recover) — now written to the
|
||||||
|
audit log as they happen — and its maintenance windows, including ones set up by
|
||||||
|
the Uptime Kuma sync. **Getting started**: until the basics are in place, an
|
||||||
|
admin's dashboard shows a checklist (an integration, a server and agent, DNS, a
|
||||||
|
secret, notifications, a second user) whose ticks come from what actually
|
||||||
|
exists; it can be hidden, and brought back from the search box.
|
||||||
|
|
||||||
|
**Proxmox snapshots.** Click a guest on Infrastructure → Proxmox and its drawer has a **Snapshots** section: what there is (oldest first, a branch indented, *with memory* marked), **Take a snapshot** for operators (a name or one from the time, a description, and for a running VM whether to keep its memory), and **Roll back…** and **Delete…** for administrators, each behind a confirmation that says what is lost. Proxmox does each as a task, followed for up to 20 seconds; each is in the audit log and the change log. The API token needs VM.Snapshot and VM.Snapshot.Rollback (the PVEVMAdmin role has them). The guest actions (start, shut down, restart, stop) were there already.
|
||||||
|
|
||||||
|
**Small trends in lists.** The rows of three lists carry a small line for which way things are going: each **server's card** (Infrastructure → Servers) has its load and memory over the last 24 hours and its **fullest disk over the last week** (a disk moves too slowly to show in a day; the line says which mount it is, and turns orange from 80% and red from 90%), and the admin table behind *Manage servers* has the same three lines side by side in a *Load and disk* column (load and memory turn orange from 75% and red from 90%), each **endpoint check** has how long it took to answer over the last day, with a red tick under each stretch in which a check failed, and each **integration** that keeps a history (Integrations, as cards or table, with a *Last week* column) has the figure it is known for over the last week — DNS queries for AdGuard, the queue for Sonarr, photos for Immich, the monthly cost for Hetzner. A gap in the data is a gap in the line, the newest reading is a dot, and each line is a picture with a sentence for a screen reader (*CPU load, last 24 hours: 12% now, between 3% and 40%*). They are drawn from the history the app already keeps (nothing new is stored), come from one request for the whole list, refresh every five minutes and are simply left out where there is no history. A line fitted to its own data makes a small change look steep — the sentence and the numbers beside it say how big it really is. The full charts are one click away on each page.
|
||||||
|
|
||||||
|
**On a phone, under the thumb.** Below tablet width a **bar along the bottom** holds the places used most — Dashboard, Alerts (with how many are active, red when one is critical), Servers — and **Menu**, which opens the full menu; it keeps clear of the home bar and a page you have hidden is not offered. On the Alerts page an operator can **acknowledge with one tap** (*✓ I'm on it*: acknowledged until the alert is gone, no note) or **snooze for an hour** (*Snooze 1 h*), and **swipe a row**: to the right acknowledges, to the left snoozes — the row follows the finger, says what letting go will do, and does it only once it has been dragged far enough, and a drag that is mostly up or down (a scroll) never does anything. Each of these has an **Undo** in the toast that follows, nothing can be done only by swiping (the buttons are there too, and *Acknowledge…* and *Snooze…* open the full forms for a note, a person or another time), and an alert that is silenced, snoozed, closed or already taken is not offered the same again.
|
||||||
|
|
||||||
|
**On a phone.** Below desktop width the top bar (logo and menu button) stays at the top of the screen so the menu is always within reach, with a menu button of at least 44 px; an open menu scrolls by itself (it no longer pushes the page down), its rows are finger-sized and the favourite stars are always visible; text is a little larger on a small screen, toasts run across the width and drawers fill it. The page asks for the whole screen (`viewport-fit=cover`) and keeps clear of the notch, the rounded corners and the home bar, which matters when it is installed to the home screen (the status bar then sits over the app). Tables turn into stacked cards, form fields are 16 px so a phone does not zoom in on them, and what is pressed does not wait for a second tap. Checked at 375 px: no page scrolls sideways.
|
||||||
|
|
||||||
|
**Swedish interface.** Press *Svenska* under *Language* in the user menu and the whole screen is drawn again in Swedish at once; the choice is kept on your account and follows you to any browser (without a choice the browser's language decides, and that guess is not stored). Swedish covers the menu, the shared parts (search, tables, dialogs, "5 min ago"), the sign-in page, and the headings, buttons, table columns, filters, status words, empty states and short messages of every page — the integration pages and the settings included. Text with no Swedish yet is shown in English, so nothing looks broken. Not yet translated: what the server writes (alert messages, notifications, digests, reports, e-mail), the longer explanatory paragraphs and sentences with numbers or names built into them. How to add more is in docs/LANGUAGES.md.
|
||||||
|
|
||||||
|
**Accessibility.** There is a skip-to-content link, labelled landmarks, visible
|
||||||
|
keyboard focus, and page changes are announced to screen readers; sortable table
|
||||||
|
headings are real buttons; every form field is tied to its label; status is
|
||||||
|
never colour alone; the search box and dialogs are proper dialogs (keyboard
|
||||||
|
trapped and restored); and the text colours were checked against the contrast
|
||||||
|
minimum in both themes. Motion is turned down for those who ask the system to.
|
||||||
|
|
||||||
|
**New ports.** When an agent first reports what a server is listening on, that is taken as normal (the baseline). After that, a **TCP port that begins listening**, that is reachable from outside the machine, and that
|
||||||
|
nobody has noted is a warning on the Alerts page (kind *Monitoring*) and a notification, with what is listening on it: "nas: a new port is listening — 8080/tcp (node)". UDP (clients and discovery open them all the time), ports from 49152 up
|
||||||
|
(where programs pick a free one) and anything bound to localhost only are left out. The port's row on the server's **Ports** card says *New* and has **Mark as expected**, which notes it and ends the alert; so does a label or comment you write on
|
||||||
|
it yourself. **Network → Ports** shows the same across every server: a *New* badge on each such port, a notice that counts them (with **Show only new**), **Mark as expected** on a row and **Mark all as expected** (which asks first, naming the servers) — so there is no need to open each server. It also ends when the port stops listening, and maintenance on the server holds it back like its other alerts. It can be switched off under Settings → Notifications (*Servers and disks*); servers get their baseline from their next report after an upgrade, so nothing alerts for what was already there.
|
||||||
|
|
||||||
|
**Broken containers.** With Dockhand connected, the app now says when a **container is broken**, not only when an image has an update: one that **keeps restarting** (critical, with its exit code), one whose **own health check fails**, one that **stopped when nobody meant it to**, one **created and
|
||||||
|
never started**, and a **Docker host Dockhand can no longer reach**. A stop is only news when it was seen happen — a stack switched off weeks ago stays quiet — and an alert stays until the container runs again. The Docker page marks each problem and has a **Leave alone** button;
|
||||||
|
the same list is under Settings → Notifications (names, `host/name`, `*`). See docs/MONITORING.md.
|
||||||
|
|
||||||
|
**What is normal for the time.** For each server's CPU load and memory use the app learns what is usual for **the same hour of the same weekday** — "a Tuesday at 03:00" — from that server's own history (the median of that hour's
|
||||||
|
averages over up to ten weeks, with a robust spread; it needs **four weeks** of that hour before it says anything, and the hours are those of the notification time zone). A server's *History → Trends* then draws a grey
|
||||||
|
**usual** line with a band around it behind the real one, and says "usually about 8 % now". It is **quiet on purpose**: only a *high* reading is ever a warning, and only when the last hour averaged at least 50 % CPU (70 % memory), at least
|
||||||
|
30 points (15) above the usual and four spreads above it, and **never dropped back toward normal during that hour** — so a spike, a job that always runs a bit late, or a server that is simply always busy at that hour is not an alert. The
|
||||||
|
warning ("wiki-host: CPU load is unusually high for a Tuesday at 03:00 — it averaged 79 % over the last hour, where it is normally about 8 % (from 6 weeks of this hour)") is an ordinary *Monitoring* alert and notification: held back by
|
||||||
|
maintenance on the server, snoozed, flapping-damped and routed like the others, and left to the server's offline alert when it has stopped reporting. Switch it off under Settings → Notifications (*Servers and disks*).
|
||||||
|
|
||||||
|
**A year of history.** The figures behind the trend lines and the answer-time charts used to reach back two weeks or five. A server's load, memory and disks are still kept as a sample every ten minutes for 35 days, and now also as an hour's and a day's average, lowest and highest (hours for four
|
||||||
|
months, days for over two years); a check's answers are kept as hours for four months and, as before, days for years. A chart asks for 24 hours, 7, 30 or 90 days or a year and is drawn from whichever is fine enough. A disk that two weeks call flat is looked at again over ninety days, so a slow, steady climb is forecast too
|
||||||
|
(on the Alerts page, the weekly digest and in `/metrics`). The first start after the upgrade builds the hours and days from the samples already kept, once.
|
||||||
|
|
||||||
|
**One chart.** Server trends, comparing servers, a check's availability and answer times, and a server's availability all use the same chart and the same ranges: hover (or press the arrow keys once it has focus) to read the values at any moment, a shaded band shows the lowest and highest where a point
|
||||||
|
stands for an hour or a day, a gap in the data is a gap in the line, and the range you pick is remembered for the next chart. **Endpoints → History…** opens a check's charts; a server's **History** section has its availability (30 days to a year) above its trends.
|
||||||
|
|
||||||
|
**API tokens.** The user menu → **API tokens** (for everyone signed in): make a read-only token for a script, Home Assistant or a dashboard — shown once, 30, 90 or 365 days or no end, ten at most, removed at once on request and with the person — and use it with `GET /api/v1/…` for
|
||||||
|
servers, alerts, incidents, inventory, network equipment, endpoint checks and maintenance. The addresses, what each answers and an example are in [docs/API.md](docs/API.md).
|
||||||
|
|
||||||
|
**UniFi network equipment.** Add a UniFi controller under Integrations (an API key, or a read-only local account; see INTEGRATIONS.md) and **Network → Network devices** lists its switches, access points and
|
||||||
|
gateways: whether each is up (and since when it was not), IP and MAC address, uptime, firmware (with *update available*), the device it is connected through, how many things depend on it, and — on the *Networks & Wi-Fi* tab —
|
||||||
|
its networks (VLAN, address range) and Wi-Fi names (security, which network). It is read every five minutes and on **Read now**; nothing is ever written to the controller, and Wi-Fi passphrases are never read. The devices
|
||||||
|
join the **dependency map** (Map → Dependencies): what each is connected through becomes a dependency by itself, you can write others ("the NAS needs the rack switch"), and when a device is down, what needs it is folded under
|
||||||
|
its alert — "Hallway AP has been offline for 45 min — also takes down pi". A device offline for longer than a set time (Settings → Notifications, 10 minutes) is a critical *Server down* alert, can be silenced with maintenance
|
||||||
|
(also in the Maintenance page and the command palette), and can have runbooks. Not yet tried against a real controller — see INTEGRATIONS.md.
|
||||||
|
|
||||||
|
**Which switch a server is plugged into.** Every read also asks the controller what is connected, and a client whose IP address is exactly one of a server's own (and only one server's) is taken to be that server: the server is then
|
||||||
|
*connected through* that switch or access point (and port, where the controller says; Wi-Fi if it is on an access point). That is a dependency made by itself — when the switch is down, the server's offline alert is folded under
|
||||||
|
the switch's. It shows on the server's page (Network section), on the Network devices page (*Plugged in:* under each device), and on **Map → Connections**, where a switch or access point with servers on it is drawn next to them.
|
||||||
|
**IP Addresses → Sync from UniFi** adds the devices' addresses (label from the device name, the model and MAC in the notes); an entry made by hand or by another sync is left alone.
|
||||||
|
|
||||||
|
**My work** (the menu, for operators and administrators; also a line on the Dashboard when something wants a look). Everything with your name on it: the alerts you acknowledged or were handed (what runs out first at the top, with *Change…* to
|
||||||
|
edit the acknowledgement), the maintenance windows you started (*+1 h*, *+4 h*, *End now* with an Undo), and the update runs you started that are still going or finished in the last day. What runs out within a day (acknowledgements), ends within two
|
||||||
|
hours (maintenance), is handed to you or is still going is counted in the badge beside the menu entry.
|
||||||
|
|
||||||
|
**A server's page in sections.** Overview (links, hardware, identity, runbooks — fold a card by clicking its header, and *Arrange cards* puts them in the order you like, for every server), Health, Network (what it is connected through, ports), History (trends, activity), Tasks (scheduled tasks, run a playbook) and Settings (Proxmox link, its own alert limits),
|
||||||
|
switched by `?view=`, so a section can be linked to: an alert about a failing service opens the Health section, a new-port alert the Network section.
|
||||||
|
|
||||||
|
**The status strip.** Under a server's name, on every section of its page, a row of figures says how it is at a glance: **Status** (online, with when it last reported — or *offline*, with when it did, and a note when Proxmox says the guest is still running, so a stopped agent is not read as a dead machine; for a server with no agent report, what Proxmox says, or that nothing has reported), **Maintenance** (only while a window on this server is on, with when it ends and why), **Alerts** (how many are open about this server and how bad — what a maintenance window, a snooze or a closing has put aside is not counted, but said — and a link to the Alerts page), **Updates** (what is waiting, how many of them security fixes, whether a reboot is needed), **Disk** (the fullest, against its own limit: green, yellow within ten points of it, red at it; a mount with its own limit is judged by that, and an ignored mount is left out), **CPU · memory** and **how long it has run** (for a guest Proxmox reads, or from an agent of version 8 on Linux, 7 on Windows, which say it in every report; left out while the server is offline, since it is then no longer how long it has been up). A figure nothing is known for is left out or says so — it is never filled in. It uses what the page already has plus the alerts and the maintenance windows, read again whenever the page is (the *Refresh* button). Everyone signed in sees it.
|
||||||
|
|
||||||
|
**Accessibility.** The main pages were checked against WCAG 2.2 AA with axe-core in both themes and now have no violations: text in the status colours is a shade that reads on the light theme, the tabs, pills, switch buttons and info alerts read on the dark one, a link in a sentence is underlined, a link that is a whole line is at least 24 px tall, the controls that had no name have one, and the map’s nodes are named by what is written on them. What was found, what was done and how to run the check again are in docs/ACCESSIBILITY.md.
|
||||||
|
|
||||||
|
**Brand colours and logos.** Every kind of integration and DNS provider has its brand's colour and, where the open icon set [Simple Icons](https://simpleicons.org) (CC0) has one, its logo — Proxmox, Synology, Tailscale, Gitea, Jellyfin, Immich, Nextcloud, Grafana, Prometheus, Home Assistant, Sonarr, Radarr, qBittorrent, Hetzner, Cloudflare, Pi-hole and more. The kind **tags** on the Integrations page (cards and table), the Servers page and the DNS page are in that colour with the logo, made readable on the light and the dark theme alike (the lightness is held where the text reads, the hue kept; every tag is at least 4.9:1 on dark and 5.2:1 on light); the dashboard's integration cards have the logo beside their title; the kind picker has it on each choice; and **the menu** has a small icon before each page in its groups — the logo for an integration's page, the page's own icon for the rest — which **Settings → Display → Menu → Show an icon before each page** switches off for this browser. A few kinds have no open logo (NetBox, Prowlarr, Lidarr, Readarr, SABnzbd, Technitium, osTicket, phpIPAM, Fragnet, Dockhand, Semaphore, UPS): they keep their generic icon and an approximate colour. **Settings → Display → Badge colours** shows each tag as it looks now, lets you choose another colour for any of them, and **Use the brand's** goes back. Logos and names belong to their owners and are used only to tell the systems apart.
|
||||||
|
|
||||||
|
**Shorter page introductions.** The few lines under a page’s title that say what the page is are folded to two lines when they are longer than that, with **Show more** under them, so the page’s own content starts higher up. What you choose is remembered for that page in this browser; a short one is shown as it is, a link inside the folded part opens it when you tab to it, and a printout shows all of it.
|
||||||
|
|
||||||
|
**Notification history.** Records → Notifications lists every notification that was sent, with how it went through each channel (or why it was not sent: held for quiet hours, a routing rule said nobody, no channel switched on), the error a channel gave when it refused, a filter and a search, and **Send again**. Kept 30 days. See NOTIFICATIONS.md.
|
||||||
|
|
||||||
|
**A shorter menu for a smaller homelab.** The menu has a page for every kind of integration the app can read, whether or not you run it. **Settings → Display → Menu → Hide the pages of integrations I have not added** (off until you switch it on, the same for everyone) leaves out of the menu, the command palette and the `g`-shortcuts the pages whose integrations are not in **Administration → Integrations**: Gitea, n8n, Grafana … as the list under the switch says (a disabled integration still counts as added; any one of Sonarr, Radarr, Lidarr, Readarr or Prowlarr keeps *Media managers*, and a Proxmox or a Backup Server keeps *Proxmox backups*). Pages that are useful without an integration — Servers, Alerts, DNS, Endpoints, Secrets, Inventory and the rest — always stay, a group with nothing left disappears, and a hidden page is still there by its address. Adding an integration brings its page back at once.
|
||||||
|
|
||||||
|
**Away mode, and critical alerts through the night.** For a one-person homelab, "on call" is about *when you can be reached*. **Settings → Notifications → When → Away** is a one-off "I'm away until Sunday evening": notifications are held while it lasts
|
||||||
|
(a banner on every page says so and counts them) and sent as **one digest** when it ends — at the time you gave, or when you press *I'm back* — so a week away is one message to read, not a buzzing phone. **Critical** alerts (a server down, a disk full, mains
|
||||||
|
power lost) can still come through at once, which is the default; untick it for true silence. You can plan one ahead (start later) and add a note. **Quiet hours** got the same option, *Still send critical alerts*, off by default: a full disk at 03:00 is
|
||||||
|
sent, the warnings wait for the morning. Escalation steps marked *send even during quiet hours* are sent while away too. Held messages only: the alerts themselves are still on the Alerts page, in webhooks and in `/metrics`. See NOTIFICATIONS.md.
|
||||||
|
|
||||||
|
**Finding a setting, and undo instead of asking.** Settings has a *Find a setting* box (and the command palette lists every setting too) that goes straight to where it is set — it knows what each is for, so "quiet", "prometheus" or "disk full"
|
||||||
|
all find something. Removing a dependency, an admin link, a runbook, a hand-made IP address (one or several), a port opening or a restore test no longer asks first: it is gone at once, with an **Undo** on the message for fifteen seconds that makes it again from
|
||||||
|
what was on screen. Things that could not come back as they were — a secret, a check with its history, a share link or token, a DNS record, a hardware entry that has labels printed with its number — still ask first.
|
||||||
|
|
||||||
|
**Flapping alerts and the noise report.** An alert that is raised and clears again and again is more tiring than one that stays: each raise and each recovery is a message. When the same
|
||||||
|
alert has been **raised three times within an hour** (Settings → Notifications → *Servers and disks* → *Call an alert flapping after*; 0 turns it off) it is called **flapping**: one message says
|
||||||
|
so — "raised 3 times within an hour" — and its further raise and recovery messages are held back. The alert itself is still on the Alerts page, with a *flapping* badge, and keeps counting for the badge in the menu and for /metrics.
|
||||||
|
Once it has been calm (no raise) for an hour, one message says whether it is **still active** or **cleared**, and it is treated as an ordinary alert again. The webhooks that receive every event (Settings → API → event webhooks)
|
||||||
|
still hear every raise and recovery; only the human-facing messages are held. The state is kept in the database, so a restart does not forget what is flapping, and the start and end of each spell are in the change log.
|
||||||
|
**Alerts → Noise** shows which alerts keep coming back over the last 7, 30, 90 or 180 days, worked out from the incident record: how many times each was raised, how long it lasted in total, on average
|
||||||
|
and at the longest, the most times it came within a single hour, whether it flapped or is flapping now, and a line on what usually helps (a longer grace time, another threshold, a look at the thing itself).
|
||||||
|
Only alerts raised at least twice are listed; the list can be searched, sorted and exported as CSV. Everyone signed in can see it.
|
||||||
|
|
||||||
|
**Bulk actions.** On the lists where it is a chore to do the same to many, tick the rows (or *Pick all N shown*) and a bar appears at the foot of the page. Each action works on the whole selection and then says how many were
|
||||||
|
changed and how many were **left alone, and why** (a server already tagged, a loop that a *runs on* choice would make, a date that is not a date). Every one has an **Undo** button on the message that stays for fifteen seconds and puts back exactly
|
||||||
|
what was there before (the earlier acknowledgement text, the earlier tags, the earlier maintenance end times); an undo that cannot be done in full says so. One entry goes in the change log per action, not one per row, and at most 200 are done at once.
|
||||||
|
- **Alerts** (operators): *Acknowledge…* (a note, who has them, for how long), *Remove acknowledgement*, *Snooze…* (hide them all until a time you choose — an hour, tomorrow at 09:00, next Monday, or your own; each comes back once, if it is still there) and *Wake* (end the snoozes of those picked), *Close…* (one reason for all of them, written on each incident; each comes back only if it gets worse) and *Open again* (for closed ones), and *Silence…* (maintenance on what each alert is about, for 5 minutes to a week). Undoing a snooze puts back each alert's earlier snooze exactly; one that would already have run out is dropped rather than announced at once.
|
||||||
|
- **Servers** (operators; press **Select** beside *Compare*): *Tag…* (add a tag to, or take one off, all of them), *Silence…* (maintenance, optionally with what depends on them too), and *Runs on…* (administrators — which server they run on, or back to working it out from Proxmox).
|
||||||
|
- **Inventory** (operators): *Change where / dates…* (where it is, warranty until, end of support — each left as it is, set, or cleared) and *Print labels* for just those pieces.
|
||||||
|
|
||||||
|
**Operating-system support.** The agents now report which operating system a server runs (Linux agent 7, Windows agent 6; re-run the install one-liner to update). A server's page shows it in a *System* row with a
|
||||||
|
badge — *supported*, *ends in N days* or *support ended* — worked out from a table of end-of-life dates for Ubuntu, Debian, CentOS, RHEL and its rebuilds, Fedora, Alpine, openSUSE Leap, Proxmox VE and Windows
|
||||||
|
(client and server). A server whose system is past its end, or ends within **Settings → Notifications → *Flag an operating system whose support ends within*** (90 days; 0 turns it off) is on the Alerts page in the *Updates* category with a link to the server — a warning once
|
||||||
|
support has ended, a note while it is only coming. The dates are a static table in the app, not looked up: a release it does not know is shown without a badge, never guessed at. The read API gives it as `operatingSystem`.
|
||||||
|
|
||||||
|
**Suggested dependencies.** Under **Map → Dependencies**, a *Suggested* card offers what the app can work out without being told: a service (Gitea, Semaphore, Uptime Kuma, phpIPAM, Home Assistant…) that is reached at a
|
||||||
|
server's own address most likely runs there, so it needs that server. *Add* writes it as an ordinary dependency; *Not that* hides the suggestion (*Show the N you said no to* brings them back). Nothing is written until you add it.
|
||||||
|
|
||||||
|
**Notes and a to-do list.** Every server's page has a *To do* card (the open jobs about it, with a line to add another and a box to tick one off) and a *Notes* card, and every network device a *Notes* button (operators; its drawer shows the device's jobs too), for what is worth remembering: free text with the same small formatting as runbooks
|
||||||
|
(bold, lists, links). **Operations → To do** is a list of the small jobs — "replace the UPS battery", "move the NAS to the new switch" — each with a day it is due, a person it is handed to and a server or device it is
|
||||||
|
about. A job with a day shows in **Coming up**, a job handed to someone in their **My work** (and its badge), and ticking it off, editing and removing it go in the change log. Operators and administrators write,
|
||||||
|
everyone signed in reads. The **dashboard** shows every open job in a **To do** card down its right-hand side (beside the attention, since-you-last-looked and getting-started cards on a wide screen, under them on a narrow one): soonest due first, overdue ones in red, a box to tick one off (with an undo) and, for operators, a line to add another; it scrolls when the list is long and is read again every minute.
|
||||||
|
|
||||||
|
**Photos on a note.** A note can carry up to **12 photos** — the rack as it is, a cable's label, the back of the NAS: *Add photo* on the Notes card (from the camera on a phone, or from the files), a click to see one large (arrow keys move between them), a caption, and *Remove*. The browser makes each one
|
||||||
|
small first (1600 pixels on the long side, as a JPEG — a few hundred kilobytes; a small PNG or a GIF is sent as it is) and draws it again, which also leaves out where the photo was taken. The server takes only a **JPEG, PNG, GIF or WebP, told by the picture's own first bytes** (an SVG, which can carry script, is refused
|
||||||
|
whatever the upload calls itself), at most 4 MB, and serves it with `nosniff` and a policy that lets it run nothing. Photos are kept **in the database** (so a database backup has them and no extra volume is needed) and go with the server or device when it is removed. Operators add, caption and remove; everyone signed in sees them; adding
|
||||||
|
and removing are in the change log.
|
||||||
|
|
||||||
|
**Racks and a network diagram.** **Inventory → Racks** draws each rack from the hardware's *where* (the rack's name) and *rack unit* ("U12", or "U12-13" for a tall piece), from the bottom up, with the units used,
|
||||||
|
the power drawn and a piece that clashes with another in red; hardware that names a rack but no unit is listed beside it. It can be printed. **Network devices → Diagram** draws the UniFi gateway, switches and access
|
||||||
|
points as a tree by what each is connected through, with the servers plugged into each, a red box for a device that is down, and a *Print* button.
|
||||||
|
|
||||||
|
**Nginx Proxy Manager.** Add it under Integrations (a user that may only *view* proxy hosts and certificates is enough; see INTEGRATIONS.md) and **Network → Proxy hosts** lists each name it serves, where it forwards to
|
||||||
|
(and which server that is, when the address is one the app knows), its certificate and whether NPM could set the host up. A certificate in use that ends within 14 days (critical in the last three), or has ended, and a
|
||||||
|
host NPM itself says is not working, are alerts; a host that is switched off is left alone. The hosts it serves are also offered as endpoint checks (Endpoints → Suggested) on the name people really use, linked to the
|
||||||
|
server it forwards to. It only reads — unless you give the integration a second user that may *manage* proxy hosts: then an administrator can make, change, switch off and delete hosts from that page (only what differs is sent, a host's access list and custom configuration are never touched, deleting asks for the host's first name, and every change is in the audit log).
|
||||||
|
|
||||||
|
**Home Assistant.** Add it under Integrations with a long-lived access token (made at the bottom of your profile page in Home Assistant). **Apps → Home Assistant** is where you *choose* the few entities worth an
|
||||||
|
alert — a UPS battery, a freezer's temperature, a pump — not all of them. Each is read every five minutes; a number keeps a history (35 days, with a chart), and you can set a limit under and/or over which it is an alert.
|
||||||
|
An entity that has been *unavailable* (or has gone missing) for 15 minutes is an alert too. If Home Assistant cannot be read for twenty minutes its entities are neither raised nor cleared, and the Alerts page says it
|
||||||
|
could not check. Nothing is ever sent to Home Assistant. Neither integration has been tried against the real thing — see INTEGRATIONS.md.
|
||||||
|
|
||||||
|
**A UPS** (Monitoring → UPS). Add an integration of type *UPS*: the address of the machine running NUT (`upsd`), or of the UPS's network card with *SNMP* ticked (version 2c, the standard UPS-MIB). Every minute the app reads it — battery
|
||||||
|
charge, how long it would last, the load, the mains voltage — and the UPS page draws them over time and lists when mains power was lost. **Mains power lost** is a critical alert and a notification (so is a battery nearly empty;
|
||||||
|
a battery to be replaced and a load of 90 % are warnings). The UPS is also a **node in the dependency map**: say what it powers, and while it runs on battery it counts as *down*, so the servers that go offline with it are
|
||||||
|
folded under its alert ("also takes down …") instead of each being announced; the page lists what is on it in the **order to shut it down**, so nothing is switched off while something that needs it is still running. The app only
|
||||||
|
reads — it never sends a command to the UPS. Not tried against a real UPS; see INTEGRATIONS.md.
|
||||||
|
|
||||||
|
**Alert rules of your own.** **Alert settings → Alert rules** (administrators write them, everyone sees them) is for limits the built-in checks do not have: *CPU above 90% for 15 minutes*, *memory above 85%*,
|
||||||
|
*a disk above 80%* (every disk, or the one you name), *a disk warmer than 55 °C*. A rule says what to watch, above or below what, for how long (a reading in the history every 10–15 minutes; every reading in that time has
|
||||||
|
to be past the limit, so a dip starts the count again), how serious it is and which servers it covers (all, one, or those with a tag). **Try it now** shows which servers it would be raised on right now before it is saved.
|
||||||
|
What a rule finds is an ordinary alert on the Alerts page, in the *Monitoring* category with the rule's own severity — announced, recovered, held back while flapping, silenced by maintenance on the server and
|
||||||
|
routed by the notification rules like the rest. A server that is not reporting is not judged (its offline alert says so). Rules are judged **as soon as the server reports** — right after its report is stored, so one that has been past its limit long enough is raised in seconds rather than at the next 15-minute pass — and again on every pass. A temperature is only known as of
|
||||||
|
the disk's last report, so it is judged right now and has no time.
|
||||||
|
|
||||||
|
**Forecasts for Proxmox storage and Synology volumes.** The disk forecast used to work only for servers with an agent. Each health pass now also writes down the figures it has just read from Proxmox (each node's
|
||||||
|
storages and root filesystem, a shared storage once) and Synology (each volume) — one reading an hour, kept 400 days — and the same straight-line forecast is made from them: *full in about 40 days*. It is shown beside
|
||||||
|
each storage on the **Proxmox** page and each volume on the **Synology** page (click it for the history chart), is an alert in the *Disk* category when it is within the forecast window
|
||||||
|
(Settings → Notifications → *Warn when a disk will fill within*, 14 days — the same setting as for server disks) and not yet over the usual "disk nearly full" limit, is a line in the weekly digest, and is
|
||||||
|
`homelab_storage_days_to_full` in /metrics. It needs a few days of readings and growth that is really steady; until then the page says it is collecting history.
|
||||||
|
|
||||||
|
**Snooze.** An alert on the Alerts page has **Snooze…** (operators): an hour, four hours, tomorrow at 09:00, in three days, next Monday, or a time of your own, up to 90 days. A snoozed alert is hidden like one under
|
||||||
|
maintenance — it is not counted in the menu badge, the dashboard or the digest, and the raise and recovery messages of the health and automation checks for it are not sent — but it is not maintenance: nothing is
|
||||||
|
promised about the thing. When the time comes an alert that is **still there is announced again, once** ("Snooze ended — an alert you snoozed is still there"); one that went away is simply gone. *Wake now* ends it early.
|
||||||
|
A snooze, like an acknowledgement, is dropped once its alert has been gone for half an hour. The summaries that other checks send (stale backups, expiry, updates) are not held, only what is announced per alert.
|
||||||
|
|
||||||
|
**Close.** An alert also has **Close…** (operators), for one that has been **dealt with or is accepted** — fixed outside the app, or a warning you have decided to live with. It is hidden exactly like a snoozed one (not counted, its messages not sent) but **with no end**: it comes back only if it **gets worse** (a more serious
|
||||||
|
severity than it had when you closed it — "Closed alert got worse" is said once), or if the problem goes away for half an hour and returns, which makes it a new one. A number in its message changing (a disk at 91 %, then 92 %) does not open it. The reason you give is **written on its incident**, so the history says why, and
|
||||||
|
it shows beside the alert (*closed by Olga*; hover for the note). *Open again* brings it back at once. Closing is not maintenance: nothing is promised about the thing, use *Silence* for planned work. Closing, and opening again, are in the change log and the audit log.
|
||||||
|
|
||||||
|
**What changed before this?** Every incident page has a card with that name: what the app knows happened in the time before the alert began (30 minutes, 2, 6 or 24 hours), **ranked by how
|
||||||
|
likely each is to be connected**. It looks through the audit log (settings, DNS, dependencies, a server being changed or going down), maintenance windows that began or ended, update runs and patch plans, other
|
||||||
|
alerts raised first, and a step in the server's own figures (a disk that jumped from 50% to 73%). Each is scored from how *related* it is (the same thing; something it needs, such as the host it runs on or the
|
||||||
|
switch it is plugged into; something that needs it; a change that touches everything), what *kind* it is (a maintenance window ending or a host going down is likelier than a runbook being edited) and how
|
||||||
|
*close* it was. They are shown as *Likely*, *Possible* or *Background* (the last behind a button). It is a list of suspects, not a diagnosis: the app can say what happened and when, not why, and it cannot see a
|
||||||
|
change made on the machine itself. Everyone signed in can see it; it is also `GET /api/v1/incidents/:id/before` in the read API.
|
||||||
|
|
||||||
|
**Buttons on notifications.** With **Settings → Notifications → Who hears what → Buttons on notifications** switched on (off by default), a notification can be dealt with from the message itself, without opening the app
|
||||||
|
or signing in: **Acknowledge**, **Silence for an hour** (maintenance on what the alert is about) and **Snooze until tomorrow at 09:00** (in the notification time zone). A message about **one** alert gets them as
|
||||||
|
buttons — three on an ntfy message, two on a phone push (the browser's own notification), one button on a Telegram message and one link on a Pushover message, with the rest on the page a tap opens — and a message about several gets one link to a page that lists each with its buttons;
|
||||||
|
Gotify, email and the chat webhook get that link as text (and Gotify's tap opens it). Escalation messages carry them too. Each link is a **signed token for the alerts in that one message only**, good for **24 hours**, and it
|
||||||
|
can do just those three things — it cannot read anything or change a setting. **Opening a link never does anything** (so a preview or a mail scanner is harmless); only pressing a button acts, and it is written to the audit
|
||||||
|
log as *from a notification link*, with the address it came from. The price is that **anyone who can read the notification can press its buttons** for those hours: fine for a private ntfy topic, your own Gotify or inbox, not for
|
||||||
|
a public ntfy.sh topic. Changing `SESSION_SECRET` or switching the setting off ends every link. A message held for quiet hours goes out without them.
|
||||||
|
|
||||||
**Ports** — each server's detail page has a Ports card for finding free ports
|
**Ports** — each server's detail page has a Ports card for finding free ports
|
||||||
and remembering what each one is for. "Scan…" runs a TCP scan of a port range
|
and remembering what each one is for. "Scan…" runs a TCP scan of a port range
|
||||||
@@ -321,6 +1099,12 @@ host (`ss`), which catches services listening on localhost only — a scan from
|
|||||||
elsewhere can't see those, so they'd otherwise look free. Re-run the agent
|
elsewhere can't see those, so they'd otherwise look free. Re-run the agent
|
||||||
install one-liner on a host to pick that up.
|
install one-liner on a host to pick that up.
|
||||||
|
|
||||||
|
**Ports named by their containers.** When a server's Docker host is in Dockhand (the Docker integration), the ports its running containers publish are named after the container — 8096 → *jellyfin* — on the server's Ports card, in the agent table on the Network → Ports page (hover for the image) and in the new-port alert ("a new port is listening — 8096/tcp jellyfin (container)"). It is automatic: nothing to set up beyond the Docker host in Dockhand having the same name as the server (the short name, the hostname or an address — the match Uptime Kuma targets use), and it is only a name beside the port. A label you wrote wins; a container's name wins over a port preset's; it is never written onto the port notes, so a published port is still a new port to be noted, and a container that starts publishing something is still news — now with its name. It is read on the health pass from the container list Dockhand already gives (every 15 minutes, and at start-up, whether or not the container alerts are on); a Docker host Dockhand can't reach keeps its names, and names not refreshed for a day are dropped. Containers on the host network publish nothing Docker lists, so their ports aren't named.
|
||||||
|
|
||||||
|
**Error pages** (404, 403, 500). An address that leads nowhere shows a *Page not found* page inside the menu — the address, the pages of the menu it most looks like (a misspelling like `/sevrers` offers *Servers*), and buttons for the dashboard, back and the search (Ctrl+K) — instead of an empty page. A page your role doesn't open says *No access* (403). A page that fails while it is drawn is caught, so the menu stays and the rest of the app keeps working (*Something went wrong*, with the details folded away); going to another page clears it. A server or an incident that is gone, and a status-page link that no longer works, say so in the same style. The server's own answers match: an address under `/api`, `/auth`, `/agent` or `/metrics` that doesn't exist is a JSON 404 (it used to be answered with the app's front page and a 200), a browser opening such an address, a missing file, or a failed sign-in is shown a page in the look of the sign-in card — light or dark as you last chose, English or Swedish from the browser's language — and a server error shown to a browser is a page rather than raw JSON.
|
||||||
|
|
||||||
|
**Port presets** (Network → Ports → the *Presets* tab). A preset is a named list of ports written once — *Remote access* (SSH, RDP, VNC, WinRM), *Databases* (MySQL, PostgreSQL, Redis, MongoDB …), *Web*, *File sharing*, *Homelab tools*, and what a machine has open out of the box — *Windows client*, *Windows server* (with the Active Directory ports) and *Debian / Ubuntu* — and for the network, *Network services* (DNS, DHCP, NTP, SNMP, syslog, RADIUS, VPNs, MikroTik, iperf3) and *Tailscale*, come ready-made, and are ordinary presets you can change or delete. The ones added later are added once to an install that already had presets (a preset of the same name is left alone, a deleted one is not put back). Add your own by typing one port to a line (`3389 RDP`, `53/udp DNS`). A preset does three things: it **names ports** — a port in any preset shows its name (3389 → RDP) on every server's Ports card, in the agent table and in the new-port alert ("a new port is listening — 3389/tcp RDP"), without a label typed for each host, and a name you write yourself always wins; it **checks hosts** — pick any number of servers and see in one table which of the preset's TCP ports are open on each, from a scan from the app (private addresses only, at the server's first known address; what it finds is remembered like any scan); and it **names the open ports on all of them** in one go, keeping names somebody already wrote unless you say to replace them. Note that a port with a name counts as noted, so writing names also quiets the new-port alert for those ports. A scan only looks at TCP; a UDP port in a preset is only given its name. Operators change presets; everyone sees them.
|
||||||
|
|
||||||
**Network → Ports** is the cross-server counterpart: one page listing every
|
**Network → Ports** is the cross-server counterpart: one page listing every
|
||||||
port every agent currently reports as listening, across all servers at once
|
port every agent currently reports as listening, across all servers at once
|
||||||
(protocol, address, process, last report time), each linking back to its
|
(protocol, address, process, last report time), each linking back to its
|
||||||
@@ -341,10 +1125,18 @@ needs the site to be served over HTTPS (browsers only offer install on secure
|
|||||||
origins; `localhost` also counts). The bundled service worker deliberately
|
origins; `localhost` also counts). The bundled service worker deliberately
|
||||||
caches nothing, so an installed copy always shows the current build.
|
caches nothing, so an installed copy always shows the current build.
|
||||||
|
|
||||||
**More documentation**: [ROLES.md](ROLES.md) — who can see and do what;
|
**More documentation** — the guides are in [docs/](docs/README.md):
|
||||||
|
[docs/SSO.md](docs/SSO.md) — single sign-on with Authentik, what the setup needs, roles from groups, troubleshooting;
|
||||||
|
[docs/API.md](docs/API.md) — the read API (`/api/v1`) and the other machine-facing addresses;
|
||||||
|
[docs/PROMETHEUS-GRAFANA.md](docs/PROMETHEUS-GRAFANA.md) — `/metrics`, Prometheus, the Grafana dashboard and every series;
|
||||||
|
[docs/WEBHOOKS.md](docs/WEBHOOKS.md) — event webhooks (signed JSON) and the notification webhook;
|
||||||
|
[docs/HEARTBEATS-AND-CALENDAR.md](docs/HEARTBEATS-AND-CALENDAR.md) — cron-job heartbeats and the calendar feed;
|
||||||
|
[docs/MONITORING.md](docs/MONITORING.md) — how monitoring works, from a reading to an alert.
|
||||||
|
The references: [ROLES.md](ROLES.md) — who can see and do what;
|
||||||
[NOTIFICATIONS.md](NOTIFICATIONS.md) — every notification the app sends and when;
|
[NOTIFICATIONS.md](NOTIFICATIONS.md) — every notification the app sends and when;
|
||||||
[INTEGRATIONS.md](INTEGRATIONS.md) — the credentials each integration needs;
|
[INTEGRATIONS.md](INTEGRATIONS.md) — the credentials each integration needs;
|
||||||
[DATABASE.md](DATABASE.md) — the database tables, columns and relationships.
|
[DATABASE.md](DATABASE.md) — the database tables, columns and relationships;
|
||||||
|
[BACKUP.md](BACKUP.md) — what to back up, how (also while it runs), how to restore, and how to check it.
|
||||||
|
|
||||||
## Requirements
|
## Requirements
|
||||||
|
|
||||||
@@ -395,6 +1187,15 @@ docker compose up -d
|
|||||||
The app listens on `HOST_PORT` (default `3000`); SQLite data persists in
|
The app listens on `HOST_PORT` (default `3000`); SQLite data persists in
|
||||||
`./data` on the host.
|
`./data` on the host.
|
||||||
|
|
||||||
|
**Telling a dev copy from production:** set `ENVIRONMENT_NAME` (e.g. `Dev`) and
|
||||||
|
optionally `ENVIRONMENT_COLOR` in that instance's `.env` (or under `environment:`
|
||||||
|
in its compose file). The sidebar then takes the colour, a badge with the name
|
||||||
|
sits next to the title and on the sign-in page, and the browser tab reads
|
||||||
|
`[Dev] Homelab Manager`. The colour is a name (`red`, `orange`, `amber`, `green`,
|
||||||
|
`teal`, `blue`, `purple`, `pink`, `gray`) or a hex value such as `#b45309`; a
|
||||||
|
name alone gets orange. Leave both unset on production and nothing changes.
|
||||||
|
Restart the container after changing them.
|
||||||
|
|
||||||
### arm64 (Raspberry Pi, Apple silicon, ARM servers)
|
### arm64 (Raspberry Pi, Apple silicon, ARM servers)
|
||||||
|
|
||||||
Two compose files, one to build the image and one to run the published one:
|
Two compose files, one to build the image and one to run the published one:
|
||||||
|
|||||||
@@ -19,42 +19,82 @@ page's own top-level link.
|
|||||||
| Menu item | Path | Viewer | Operator | Admin |
|
| Menu item | Path | Viewer | Operator | Admin |
|
||||||
|---|---|:---:|:---:|:---:|
|
|---|---|:---:|:---:|:---:|
|
||||||
| Dashboard | `/` | ✅ | ✅ | ✅ |
|
| Dashboard | `/` | ✅ | ✅ | ✅ |
|
||||||
|
| Alerts (Now, History, Noise) | `/alerts` | ✅ | ✅ | ✅ |
|
||||||
|
| My work | `/my-work` | ❌ | ✅ | ✅ |
|
||||||
| **Infrastructure** | | | | |
|
| **Infrastructure** | | | | |
|
||||||
| Servers | `/servers` | ✅ | ✅ | ✅ |
|
| Servers | `/servers` | ✅ | ✅ | ✅ |
|
||||||
| Proxmox | `/proxmox` | ✅ | ✅ | ✅ |
|
| Proxmox | `/proxmox` | ✅ | ✅ | ✅ |
|
||||||
| Synology | `/synology` | ✅ | ✅ | ✅ |
|
| Synology | `/synology` | ✅ | ✅ | ✅ |
|
||||||
| Proxmox Backup | `/pbs` | ✅ | ✅ | ✅ |
|
|
||||||
| Docker | `/docker` | ✅ | ✅ | ✅ |
|
| Docker | `/docker` | ✅ | ✅ | ✅ |
|
||||||
|
| Hetzner Cloud | `/hetzner` | ✅ | ✅ | ✅ |
|
||||||
|
| Hetzner Robot | `/hetzner-robot` | ✅ | ✅ | ✅ |
|
||||||
|
| Game servers | `/game-servers` | ✅ | ✅ | ✅ |
|
||||||
| Tailscale | `/tailscale` | ✅ | ✅ | ✅ |
|
| Tailscale | `/tailscale` | ✅ | ✅ | ✅ |
|
||||||
|
| **Apps** | | | | |
|
||||||
|
| Home Assistant | `/home-assistant` | ✅ | ✅ | ✅ |
|
||||||
|
| Jellyfin | `/jellyfin` | ✅ | ✅ | ✅ |
|
||||||
|
| Media managers | `/media` | ✅ | ✅ | ✅ |
|
||||||
|
| Download clients | `/downloads` | ✅ | ✅ | ✅ |
|
||||||
|
| AdGuard Home | `/adguard` | ✅ | ✅ | ✅ |
|
||||||
|
| Technitium DNS | `/technitium` | ✅ | ✅ | ✅ |
|
||||||
|
| Grafana | `/grafana` | ✅ | ✅ | ✅ |
|
||||||
|
| Immich | `/immich` | ✅ | ✅ | ✅ |
|
||||||
|
| Paperless-ngx | `/paperless` | ✅ | ✅ | ✅ |
|
||||||
|
| Nextcloud | `/nextcloud` | ✅ | ✅ | ✅ |
|
||||||
|
| Authentik | `/authentik` | ✅ | ✅ | ✅ |
|
||||||
|
| **Backups** | | | | |
|
||||||
|
| Proxmox backups (tabs: Backup coverage, Proxmox Backup Server) | `/backups` | ✅ | ✅ | ✅ |
|
||||||
|
| Duplicati | `/duplicati` | ✅ | ✅ | ✅ |
|
||||||
| **Network** | | | | |
|
| **Network** | | | | |
|
||||||
| DNS | `/dns` | ✅ | ✅ | ✅ |
|
| DNS (and Domains) | `/dns` | ✅ | ✅ | ✅ |
|
||||||
| Domains | `/domains` | ✅ | ✅ | ✅ |
|
| Network devices | `/network/devices` | ✅ | ✅ | ✅ |
|
||||||
|
| Proxy hosts | `/proxy-hosts` | ✅ | ✅ | ✅ |
|
||||||
| IP Addresses | `/ipam` | ✅ | ✅ | ✅ |
|
| IP Addresses | `/ipam` | ✅ | ✅ | ✅ |
|
||||||
|
| NetBox | `/netbox` | ✅ | ✅ | ✅ |
|
||||||
| Ports | `/ports` | ✅ | ✅ | ✅ |
|
| Ports | `/ports` | ✅ | ✅ | ✅ |
|
||||||
| Consistency | `/consistency` | ✅ | ✅ | ✅ |
|
| Consistency | `/consistency` | ✅ | ✅ | ✅ |
|
||||||
|
| **Monitoring** | | | | |
|
||||||
|
| Endpoints | `/endpoints` | ✅ | ✅ | ✅ |
|
||||||
|
| Alert settings (tabs: Alert rules, Service levels; and for administrators only Alert tickets, plus a link to Settings → Notifications) | `/alert-settings` | ✅ | ✅ | ✅ |
|
||||||
|
| Heartbeats | `/heartbeats` | ✅ | ✅ | ✅ |
|
||||||
|
| Uptime Kuma | `/uptime-kuma` | ✅ | ✅ | ✅ |
|
||||||
|
| UPS | `/ups` | ✅ | ✅ | ✅ |
|
||||||
|
| Prometheus | `/prometheus` | ✅ | ✅ | ✅ |
|
||||||
|
| Speedtest | `/speedtest` | ✅ | ✅ | ✅ |
|
||||||
| **Automation** | | | | |
|
| **Automation** | | | | |
|
||||||
| Semaphore | `/semaphore` | ✅ | ✅ | ✅ |
|
| Semaphore | `/semaphore` | ✅ | ✅ | ✅ |
|
||||||
| Gitea | `/gitea` | ✅ | ✅ | ✅ |
|
| Gitea | `/gitea` | ✅ | ✅ | ✅ |
|
||||||
| Secrets | `/secrets` | ✅ | ✅ | ✅ |
|
| n8n | `/n8n` | ✅ | ✅ | ✅ |
|
||||||
|
| Secrets (a top-level entry) | `/secrets` | ✅ | ✅ | ✅ |
|
||||||
| **Operations** | | | | |
|
| **Operations** | | | | |
|
||||||
| Alerts | `/alerts` | ✅ | ✅ | ✅ |
|
|
||||||
| Maintenance | `/maintenance` | ✅ | ✅ | ✅ |
|
| Maintenance | `/maintenance` | ✅ | ✅ | ✅ |
|
||||||
| Uptime Kuma | `/uptime-kuma` | ✅ | ✅ | ✅ |
|
| Updates (and Patch plans) | `/updates` | ✅ | ✅ | ✅ |
|
||||||
|
| App versions | `/versions` | ✅ | ✅ | ✅ |
|
||||||
|
| Coming up | `/coming-up` | ✅ | ✅ | ✅ |
|
||||||
|
| To do | `/todo` | ✅ | ✅ | ✅ |
|
||||||
| osTicket | `/osticket` | ✅ | ✅ | ✅ |
|
| osTicket | `/osticket` | ✅ | ✅ | ✅ |
|
||||||
|
| **Records** | | | | |
|
||||||
|
| Reports (monthly, quarterly, annual) | `/reports` | ✅ | ✅ | ✅ |
|
||||||
|
| Costs | `/costs` | ✅ | ✅ | ✅ |
|
||||||
|
| Change log (and the raw audit log, `?view=audit`) | `/changes` | ❌ | ✅ | ✅ |
|
||||||
|
| **Reference** | | | | |
|
||||||
|
| Map (Connections, and Dependencies) | `/map` | ✅ | ✅ | ✅ |
|
||||||
|
| Inventory (and Racks, Labels, Lifecycle) | `/inventory` | ✅ | ✅ | ✅ |
|
||||||
| Admin Links | `/admin-links` | ✅ | ✅ | ✅ |
|
| Admin Links | `/admin-links` | ✅ | ✅ | ✅ |
|
||||||
| Generator | `/generator` | ✅ | ✅ | ✅ |
|
| Generator | `/generator` | ✅ | ✅ | ✅ |
|
||||||
| **Administration** | | | | |
|
| **Administration** | | | | |
|
||||||
| Integrations | `/integrations` | ✅ | ✅ | ✅ |
|
| Integrations | `/integrations` | ✅ | ✅ | ✅ |
|
||||||
|
| Diagnostic Log (Integrations → Call log) | `/integrations?view=calls` | ❌ | ❌ | ✅ |
|
||||||
| Users | `/users` | ❌ | ❌ | ✅ |
|
| Users | `/users` | ❌ | ❌ | ✅ |
|
||||||
| Sessions | `/sessions` | ❌ | ❌ | ✅ |
|
| Sessions (Users → Signed in now) | `/users?view=sessions` | ❌ | ❌ | ✅ |
|
||||||
| Audit Log | `/audit-log` | ❌ | ✅ | ✅ |
|
|
||||||
| Diagnostic Log | `/diag-log` | ❌ | ❌ | ✅ |
|
|
||||||
| Settings | `/settings` | ❌ | ❌ | ✅ |
|
| Settings | `/settings` | ❌ | ❌ | ✅ |
|
||||||
| Privacy (footer link, not in a group) | `/privacy` | ✅ | ✅ | ✅ |
|
| **Outside the menu** | | | | |
|
||||||
|
| API tokens (user menu) | `/api-tokens` | ✅ | ✅ | ✅ |
|
||||||
|
| Privacy (footer link) | `/privacy` | ✅ | ✅ | ✅ |
|
||||||
|
|
||||||
So in practice: **every page is visible to every role except the five
|
So in practice: **every page is visible to every role except a few** — *My work* and the *Change log* (with the raw audit log) need operator; *Users* (with *Signed in now*) and *Settings* need
|
||||||
under Administration** — Users, Sessions, and Diagnostic Log need admin;
|
admin; and the *Call log* tab of Integrations needs admin (operators and viewers see Integrations without it). The sections that used to be pages of their own — Sessions, the audit log, the
|
||||||
Audit Log needs operator or admin; Settings needs admin.
|
Diagnostic Log, Dependencies, Domains — now live inside those pages, and their old addresses redirect there.
|
||||||
|
|
||||||
## Being able to see a page isn't the same as being able to change things
|
## Being able to see a page isn't the same as being able to change things
|
||||||
|
|
||||||
@@ -68,20 +108,102 @@ even further, to admin only:
|
|||||||
- **Integrations & DNS providers**: any operator can use an integration
|
- **Integrations & DNS providers**: any operator can use an integration
|
||||||
once it's configured (start/stop a guest, run a template, sync DNS
|
once it's configured (start/stop a guest, run a template, sync DNS
|
||||||
records, etc.), but adding, editing, testing, or deleting an
|
records, etc.), but adding, editing, testing, or deleting an
|
||||||
integration or DNS provider is admin-only.
|
integration or DNS provider is admin-only. So is **making, editing or deleting a zone** on a DNS provider that can (Technitium): making one makes the server an authority for a name, deleting one takes every record in it, and editing one switches it off or changes who may ask or copy it, so none is an operator's job like adding a record.
|
||||||
- **Servers**: registering a new server, rotating its agent token, and
|
- **Servers**: registering a new server, rotating its agent token, and
|
||||||
editing/deleting a server are admin-only; tagging a server and
|
editing/deleting a server (including setting its hostname by hand, and giving that up) are admin-only; tagging a server and
|
||||||
linking/unlinking it to a Proxmox guest just need operator.
|
linking/unlinking it to a Proxmox guest just need operator.
|
||||||
- **Tags**: creating, renaming, recoloring, or deleting a tag is
|
- **Tags**: creating, renaming, recoloring, or deleting a tag is
|
||||||
admin-only (applying an existing tag to a server needs operator).
|
admin-only (applying an existing tag to a server needs operator).
|
||||||
- **Ports**: everyone can see both the agent-reported and manual tables;
|
- **Ports**: everyone can see both the agent-reported and manual tables;
|
||||||
adding, editing, or deleting a manual port opening needs operator.
|
adding, editing, or deleting a manual port opening needs operator.
|
||||||
|
- **Container ports**: the names Docker containers give a server's ports (from Dockhand) are shown to everyone; there is nothing to edit — they are kept by the health pass.
|
||||||
|
- **Port presets** (Network → Ports → Presets tab): everyone can see them and the names they give ports; creating, changing and removing one, scanning hosts for one and writing its names onto many hosts' ports need operator.
|
||||||
- **Admin Links**: everyone can see and open every server's admin
|
- **Admin Links**: everyone can see and open every server's admin
|
||||||
bookmarks, from that server's own page or the summary page; adding,
|
bookmarks, from that server's own page or the summary page; adding,
|
||||||
editing, or removing one needs operator, from either place.
|
editing, or removing one needs operator, from either place.
|
||||||
- **Generator**: everyone can use it; the name lists it picks server names from
|
- **Generator**: everyone can use it; the name lists it picks server names from
|
||||||
are edited under Settings → Names, which is admin-only (and so is importing
|
are edited under Settings → Names, which is admin-only (and so is importing
|
||||||
names from Skatteverket).
|
names from Skatteverket).
|
||||||
|
- **Alert tickets**: everyone sees which alerts have an osTicket ticket (a
|
||||||
|
badge on the Alerts page, linking to the ticket in osTicket); switching the
|
||||||
|
feature on, choosing what opens a ticket, and sending a test ticket are
|
||||||
|
under Alert settings → Alert tickets, which is admin-only. **Closing a ticket** along with an alert (the tick box in the Close drawer, for one alert or several picked together) is an operator's, like closing the alert; the login it writes with, and **Test closing**, are administrators'.
|
||||||
|
- **Runbooks**: everyone reads them (on a server's page, an integration's card, the *What to do* link beside an alert for a whole kind of alert,
|
||||||
|
next to an alert); writing, editing and removing one needs operator.
|
||||||
|
- **Inventory**: everyone can see the hardware (and print its labels); adding, editing and removing needs
|
||||||
|
operator, and so does *Refresh from agent*; the price of power used for the running cost is admin-only.
|
||||||
|
- **Bulk actions** (the bar that appears when rows are ticked) need the same role as the single action: operator to acknowledge, snooze, close, silence, tag, change hardware and undo any of those; administrator for *Runs on…* on several servers
|
||||||
|
(and its undo). Viewers do not see the tick boxes. The **Noise** tab of Alerts is for everyone signed in.
|
||||||
|
- **API tokens** (user menu): anyone signed in makes, sees and removes their own, and nobody else's; a token can only read (`GET /api/v1`), sees what its owner sees, and is removed with its owner. The chart ranges and history are for everyone signed in.
|
||||||
|
- **My work** is shown to operators and administrators and only ever lists their own alerts, maintenance and update runs; the buttons on it (change an acknowledgement, extend or end a window) need operator, like the pages they come from.
|
||||||
|
**Settings search** is part of Settings, which is administrators'.
|
||||||
|
- **Network devices** (Network → Network devices): everyone can see the devices, networks and Wi-Fi names read from UniFi controllers; **Read now** and putting a device under maintenance need operator; adding the UniFi
|
||||||
|
integration itself (a key or login) is an administrator's job, like any integration.
|
||||||
|
- **Docker vulnerabilities** (Docker page): everyone signed in sees the counts and the critical findings from Dockhand's scans; nothing on the page starts a scan or changes anything. Switching the alert off is in Settings → Notifications (administrators).
|
||||||
|
- **Docker containers** (Docker page): everyone signed in sees each container and what is wrong with it; start, stop and restart need operator; **Leave alone** (no alerts for a container) and the list in Settings are administrators'.
|
||||||
|
- **Away mode** (Settings → Notifications → When): administrators start, plan, change and end it, since it decides who is told what; everyone signed in sees the banner that says someone is away and how many notifications are held.
|
||||||
|
- **Buttons on notifications** (Settings → Notifications → Who hears what): switching them on is administrator-only. A link in a notification needs **no role at all** — the signed token is the access, for the alerts in that message, for 24 hours, to acknowledge, silence for an hour or snooze, and nothing else.
|
||||||
|
- **Alert rules** (Alert settings → Alert rules): everyone signed in sees the rules and where each is raised now; making, changing, trying, switching and removing one is administrator-only (a rule decides who is
|
||||||
|
woken up). **Snooze** an alert, wake it, **close** it and open it again need operator, like acknowledging. The storage forecast beside Proxmox storages and Synology volumes is for everyone who can see those pages.
|
||||||
|
- **What changed before this?** (on an incident's page, and `GET /api/v1/incidents/:id/before`): everyone signed in, like the incident record it is made from.
|
||||||
|
- **Dependencies**: everyone can see what needs what (including what needs the Internet); adding and removing one needs operator. So does adding or hiding a *suggested* one.
|
||||||
|
- **Customise** (the dashboard): everyone signed in can hide and move the dashboard's cards for themselves; it is kept on their own user record and changes nothing for anyone else.
|
||||||
|
- **To do and Notes** (Operations → To do, the *To do* card on the dashboard, the *To do* and *Notes* cards on a server's page, a device's *Notes* drawer): everyone signed in reads them; adding, editing, ticking off and removing a job, and writing a note, and adding, captioning and removing a photo on one, need operator.
|
||||||
|
- **Racks** (Inventory → Racks) and the **network diagram**: everyone signed in. A rack is made from the hardware's *where* and *rack unit*, which operators edit on the inventory page.
|
||||||
|
- **Proxy hosts** (Network → Proxy hosts): everyone signed in sees what Nginx Proxy Manager serves; **Read now** needs operator; adding the integration (its login) is an administrator's job, and so is **making, changing, switching off and deleting a proxy host** — only where the integration was given a user that may manage.
|
||||||
|
- **Home Assistant** (Apps → Home Assistant): everyone signed in sees what is watched, its state and history; choosing what to watch, its limits, stopping and **Read now** need operator (the list of all
|
||||||
|
entities is read live, so only operators see it); adding the integration (its token) is an administrator's job.
|
||||||
|
- **Service levels** (Alert settings → Service levels): everyone signed in sees the targets and what is left of the budget; setting, changing and removing one is administrator-only, and each is in the audit log.
|
||||||
|
- **UPS** (Monitoring → UPS): everyone signed in sees each UPS, its history and what is on it; adding one (its address and SNMP community) is an administrator's job, like any integration, and what it powers is written in the dependency map.
|
||||||
|
- **Endpoints**: everyone sees the checks and how they are doing; adding, changing and removing one is administrator-only (each makes the server connect somewhere); that includes the value a check judges and the header (a token) it sends, whose value is never shown again to anyone;
|
||||||
|
*Check now* needs operator.
|
||||||
|
- **Hetzner Robot** (Infrastructure → Hetzner Robot, for dedicated servers and storage boxes): everyone signed in sees the servers, their addresses, rescue state and what is paid until; adding one (a webservice user and its password) is an administrator's job, like any integration. Nothing on the page resets, orders or cancels anything.
|
||||||
|
- **Hetzner Cloud** (Infrastructure → Hetzner Cloud, and the card on a server's page): everyone signed in sees the servers, their addresses, traffic and cost estimate, volumes and snapshots; adding a project (its API token) is an administrator's job, like any integration. Nothing on the page changes anything at Hetzner.
|
||||||
|
- **Game servers** (Infrastructure → Game servers, Fragnet): everyone signed in sees the servers, their games, addresses, use, backups and schedules; **start, stop, restart and kill**, and **Back up now**, need operator (each is written to the audit log); adding the account (its API key) is an administrator's job, like any integration.
|
||||||
|
- **Speedtest** (Monitoring → Speedtest): everyone signed in sees the speeds, their history and what is wrong; adding one (its address, an API token, and what the line is sold at) is an administrator's job, like any integration. Nothing here starts a speed test.
|
||||||
|
- **Costs** (Records → Costs): everyone signed in sees what is paid for, the estimates read from Hetzner and the totals; adding, changing and removing an item is an administrator's. The **History** card (what it cost each month, and what changed) is seen by everyone signed in; it is kept by the app itself, so changing an item is what writes to it.
|
||||||
|
- **Nextcloud, NetBox and n8n** (Apps → Nextcloud, Network → NetBox, Automation → n8n): everyone signed in sees the status, the counts and what is wrong — for NetBox, the names and addresses of the servers that are not in it; for n8n, the names of the workflows; adding one (its address and a login, token or key) is an administrator's job, like any integration.
|
||||||
|
- **Immich** (Apps → Immich): everyone signed in sees the counts, the disk and the job queues; adding one (its address and API key) is an administrator's job, like any integration.
|
||||||
|
- **Paperless-ngx** (Apps → Paperless): everyone signed in sees the counts, the system status and the names of the files that failed to import; adding one (its address and a superuser's token) is an administrator's job, like any integration.
|
||||||
|
- **Prometheus** (Monitoring → Prometheus): everyone signed in sees the targets, what is down and the alerts firing; adding one (its address, and a token if it needs one) is an administrator's job, like any integration.
|
||||||
|
- **Authentik** (Apps → Authentik): everyone signed in sees the outposts, what runs out and the failed-sign-in count (the names of tokens and certificates, not their values); adding one (its address and API token) is an administrator's job, like any integration. The **Users and groups** tab is an administrator's alone (others see only the status): **making a user** (and the groups to offer), **listing** the users and **editing, switching them off or on or deleting them**, and **making, renaming and deleting groups and changing who is in them**, are an administrator's alone, and only for an Authentik that was given a token for it; each is written in the audit log (who made, edited (which fields, not the new values), switched off, switched on or deleted which user, who made, renamed or deleted which group and who was put in or taken out of it, and who made a recovery link), never the link.
|
||||||
|
- **Jellyfin** (Apps → Jellyfin): everyone signed in sees who is streaming and what is wrong; adding one (its address and API key) is an administrator's job, like any integration. Starting an update run on a server people are streaming from needs `startAnyway` — the update drawer asks (operators).
|
||||||
|
- **Media managers** (Apps → Media managers, for Sonarr, Radarr and Prowlarr): everyone signed in sees what each says is wrong, its queue, disks and indexers; adding one (its API key) is an administrator's job, like any integration. Nothing on the page changes anything in them.
|
||||||
|
- **Download clients** (Apps → Download clients, for qBittorrent and SABnzbd): everyone signed in sees speed, queue, what is stuck or failed and the room left; adding one (its user name and password, or its API key) is an administrator's job, like any integration. Nothing on the page adds, pauses or removes anything in them.
|
||||||
|
- **App versions** (Operations → App versions): everyone signed in sees what each application runs against the newest release; switching the lookup of releases on and off, and the number of days after which a newer release is a warning, are an administrator's.
|
||||||
|
- **Read now** (Integrations → Read now, and on the Media managers, Download clients and Hetzner Robot pages): operators and administrators. It only reads — nothing at the other end is changed — but it makes the integration's requests at once, so the same integration cannot be read again within 20 seconds. Viewers do not see the button.
|
||||||
|
- **Inspecting an integration** (Integrations → Inspect): administrators only — it makes the integration's requests, here and now (and says what came back, as shapes without values); each one is in the audit log.
|
||||||
|
- **Grouping the alerts** is each person's own view, remembered in their browser; it needs no role.
|
||||||
|
- **AdGuard Home** (Network → AdGuard Home), **Technitium DNS** (Network → Technitium DNS) and **Grafana** (Monitoring → Grafana): everyone signed in sees what each says is wrong, the statistics, lists, zones, rules and data sources; adding one (its sign-in or token) is an administrator's job, like any integration. Nothing on the pages changes anything in them.
|
||||||
|
- **Arranging a server's page**: folding the cards of a server's overview and putting them in order is each person's own (kept on their account, the same for every server) — it needs no role.
|
||||||
|
- **Duplicati** (Backups → Duplicati): everyone signed in sees every backup and how it went; setting how often one is expected, muting it and forgetting it need operator; adding a machine (which makes an address that can write to this app) and removing one are administrators'. Reports arrive on a secret address with no sign-in: the token is the access.
|
||||||
|
- **Restore tests**: everyone sees them on the Backup coverage page; recording one needs operator; removing one is administrator-only.
|
||||||
|
- **Change log**: operators and administrators (it is made from the audit log, which has the same readers).
|
||||||
|
- **Push notifications**: an administrator switches the channel on (Settings → Notifications). Each person, whatever their role, adds
|
||||||
|
and removes their own browsers and can send a test to them; an administrator can remove anyone's.
|
||||||
|
- **Backup coverage** is a read-only page for everyone. On **Updates**, everyone sees what is waiting and how update runs
|
||||||
|
are going; starting an update run (**Update…**) or stopping one needs operator.
|
||||||
|
- **Patch plans** (Updates → Patch plans) can be seen by everyone, with how they went; making, changing, switching off and removing one needs administrator (it will run on its own, at night);
|
||||||
|
**Run now** and **Stop** need operator, like an update run. The **Reports** page is for everyone signed in; sending it now, and its schedule (Settings), are administrators'.
|
||||||
|
**Mark as expected** on a server's Ports card, and on the Ports page under Network (one port, or **Mark all as expected**), which ends a new-port alert, needs operator, like any note on a port; everyone signed in sees which ports are *New*.
|
||||||
|
- **Acknowledging an alert, handing it over, and taking it back** need operator (the person it is handed to must be an operator or
|
||||||
|
administrator); everyone sees who has what. **Incidents** (Alerts → History) are readable by everyone signed in; writing a note needs operator, and a note
|
||||||
|
can be removed by its author or an administrator.
|
||||||
|
- **Database backups** (Settings → Data → Backup: making, downloading, uploading, checking, restoring and deleting them, and the schedule) are
|
||||||
|
administrator-only: a backup holds the audit log, every server's address, the hashed agent tokens and all the stored credentials
|
||||||
|
(encrypted).
|
||||||
|
- **Alert limits** on a server's page: everyone can see the ones a server has of its own; only an administrator changes them. The same goes for the **services to
|
||||||
|
ignore** on its Host health card. Everyone sees *What the agent couldn't collect*.
|
||||||
|
- **Host health** (failed services, disk health, ZFS pools) is shown to everyone on a server's page; **Update agents automatically**
|
||||||
|
on the Servers page is admin-only.
|
||||||
|
- **Profile pictures** are shown to everyone signed in (`/api/avatars/<id>`, the same people whose names everyone already sees); which sources are allowed is under
|
||||||
|
Settings → Sign-in, so admin-only.
|
||||||
|
- **Heartbeats** and the **Coming up** list are visible to everyone signed in; only administrators see a heartbeat's address, and make, change, remove or give a new address to one, and make
|
||||||
|
the calendar feed of Coming up (a share link). **Suggested checks** are administrators' like the checks themselves. A job's call to its address needs no role: the token is the access.
|
||||||
|
- **The delivery log of an event webhook** (what was sent, with the messages, and sending one again) and **Settings → Data → Storage** (how big the app's data is) are administrator-only; sending a message again is in the audit log.
|
||||||
|
- **Sign-in groups, metrics tokens, event webhooks, share links (the wall display and the status pages, and what is on them) and the
|
||||||
|
Grafana dashboard download** are all under Settings, so admin-only; `/metrics` itself needs a metrics token, not a login.
|
||||||
|
- **Alert settings**: the *Alert rules* and *Service levels* tabs are for everyone to read (changing them is administrators'); the *Alert tickets* tab, and the link to Settings → Notifications, are only shown to administrators, as they were under Settings.
|
||||||
- Everything under **Settings** (notification channels, badge colors,
|
- Everything under **Settings** (notification channels, badge colors,
|
||||||
display prefs, log retention, backup/restore) is admin-only, matching
|
display prefs, log retention, backup/restore) is admin-only, matching
|
||||||
the page itself being admin-only.
|
the page itself being admin-only.
|
||||||
|
|||||||
@@ -88,6 +88,15 @@ Description=Report scheduled tasks to Homelab Manager
|
|||||||
Type=oneshot
|
Type=oneshot
|
||||||
EnvironmentFile=$CONFIG_DIR/homelab-manager-agent.env
|
EnvironmentFile=$CONFIG_DIR/homelab-manager-agent.env
|
||||||
ExecStart=$INSTALL_DIR/homelab-manager-agent.sh
|
ExecStart=$INSTALL_DIR/homelab-manager-agent.sh
|
||||||
|
# A few things can't be done inside the sandbox below, so one step runs first and outside it (the + does that; the - lets the report go
|
||||||
|
# ahead whatever happens): asking dnf or zypper what is waiting (they write to /var and /run even when only asked), reading disk health and
|
||||||
|
# ZFS pool state (the sandbox closes the devices), and, only if an administrator has switched it on in Homelab Manager, replacing the agent
|
||||||
|
# script with a newer one (the sandbox keeps /usr read-only). It leaves its answers in the unit's private directory, which goes away when
|
||||||
|
# the run ends; the one thing kept between runs is the last SMART reading, in /var/lib/homelab-manager-agent.
|
||||||
|
ExecStartPre=-+$INSTALL_DIR/homelab-manager-agent.sh --prepare
|
||||||
|
RuntimeDirectory=homelab-manager-agent
|
||||||
|
RuntimeDirectoryMode=0700
|
||||||
|
TimeoutStartSec=600
|
||||||
# The agent only reads (crontabs, timers, /proc, listening sockets) and makes one HTTPS request, but it runs as root. These
|
# The agent only reads (crontabs, timers, /proc, listening sockets) and makes one HTTPS request, but it runs as root. These
|
||||||
# keep a bug in it, or in anything it parses, from being able to change the system. It still sees every process, which is
|
# keep a bug in it, or in anything it parses, from being able to change the system. It still sees every process, which is
|
||||||
# how it names the program behind each listening port. Directives an older systemd doesn't know are ignored.
|
# how it names the program behind each listening port. Directives an older systemd doesn't know are ignored.
|
||||||
|
|||||||
+455
-31
@@ -10,6 +10,15 @@
|
|||||||
#
|
#
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
|
# Which version of this script this is, sent with every report so Homelab Manager can tell which agents are behind. Raise it whenever a
|
||||||
|
# change here is worth installing the agent again for (and add it to the list at the top of server/src/services/agentVersions.ts).
|
||||||
|
# 1 = before the number was reported 2 = reports pending updates, and its own version
|
||||||
|
# 3 = the package manager is asked outside the sandbox (see --prepare), so dnf and zypper can answer there too
|
||||||
|
# 4 = also reports failed services, disk health (SMART) and ZFS pools, reads pacman, and can update itself when told to
|
||||||
|
# 5 = also says what the machine is (maker, model, serial number), from the firmware, for the inventory
|
||||||
|
# 6 = also says what it could not collect and why (no smartctl, no systemd, a package manager that failed ...), so the server can say what to fix
|
||||||
|
AGENT_VERSION=8
|
||||||
|
|
||||||
ENV_FILE="${ENV_FILE:-/etc/homelab-manager-agent.env}"
|
ENV_FILE="${ENV_FILE:-/etc/homelab-manager-agent.env}"
|
||||||
if [[ -f "$ENV_FILE" ]]; then
|
if [[ -f "$ENV_FILE" ]]; then
|
||||||
# shellcheck disable=SC1090
|
# shellcheck disable=SC1090
|
||||||
@@ -22,11 +31,50 @@ API_INSECURE="${API_INSECURE:-false}"
|
|||||||
DRY_RUN=0
|
DRY_RUN=0
|
||||||
[[ "${1:-}" == "--dry-run" ]] && DRY_RUN=1
|
[[ "${1:-}" == "--dry-run" ]] && DRY_RUN=1
|
||||||
|
|
||||||
if [[ -z "$API_URL" || -z "$API_TOKEN" ]]; then
|
# Two extra ways to run it, neither of which reports anything:
|
||||||
|
# --prepare the things that can't be done inside the systemd sandbox: asks the package manager what is waiting (dnf and zypper
|
||||||
|
# write to /var and /run even when only asked), reads disk health with smartctl and pool state with zpool (both need the
|
||||||
|
# devices, which the sandbox closes), and — if an administrator has switched it on — replaces this script with a newer
|
||||||
|
# one. The answers are left in the unit's private directory for the report to pick up. The unit runs this step outside
|
||||||
|
# its sandbox (ExecStartPre=-+); a failure here only means those figures are missing. --prepare-updates is the same.
|
||||||
|
# --check-updates for a person at a terminal: runs the same checks and shows what each said, and whether it would update itself.
|
||||||
|
MODE=report
|
||||||
|
case "${1:-}" in
|
||||||
|
--prepare|--prepare-updates) MODE=prepare ;;
|
||||||
|
--check-updates|--check) MODE=check ;;
|
||||||
|
esac
|
||||||
|
UPDATES_FILE="${UPDATES_FILE:-/run/homelab-manager-agent/updates.json}"
|
||||||
|
HEALTH_FILE="${HEALTH_FILE:-$(dirname "$UPDATES_FILE")/health.json}"
|
||||||
|
# What --prepare could not collect and why, left for the report to carry along.
|
||||||
|
DIAG_FILE="${DIAG_FILE:-$(dirname "$UPDATES_FILE")/diagnostics.json}"
|
||||||
|
# Kept between runs (the unit's private directory is not): a SMART reading is only taken about once an hour, so sleeping disks are left alone.
|
||||||
|
STATE_DIR="${STATE_DIR:-/var/lib/homelab-manager-agent}"
|
||||||
|
# Where the package managers' own error output goes: nowhere, unless a person asked to see it.
|
||||||
|
PM_ERR=/dev/null
|
||||||
|
[[ "$MODE" == "check" ]] && PM_ERR=/dev/stderr
|
||||||
|
|
||||||
|
if [[ "$MODE" == "report" && ( -z "$API_URL" || -z "$API_TOKEN" ) ]]; then
|
||||||
echo "API_URL and API_TOKEN must be set (env vars or $ENV_FILE)" >&2
|
echo "API_URL and API_TOKEN must be set (env vars or $ENV_FILE)" >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Things that could not be collected, each a small JSON object: which check, whether it is "unavailable" (nothing to read it with, or nothing
|
||||||
|
# there to read) or "failed" (it tried and could not), why in a word, and at most a line of what the tool said. Only problems are noted: a
|
||||||
|
# check that is not mentioned went fine. The server turns these into what to do about them.
|
||||||
|
DIAG_ITEMS=()
|
||||||
|
diag() {
|
||||||
|
local check="$1" status="$2" reason="${3:-}" detail="${4:-}"
|
||||||
|
detail=$(printf '%s' "$detail" | tr -d '\000-\037' | head -c 160)
|
||||||
|
DIAG_ITEMS+=("$(jq -n -c --arg c "$check" --arg s "$status" --arg r "$reason" --arg d "$detail" \
|
||||||
|
'{check: $c, status: $s} + (if $r != "" then {reason: $r} else {} end) + (if $d != "" then {detail: $d} else {} end)')")
|
||||||
|
}
|
||||||
|
|
||||||
|
# The notes made so far as one JSON array ([] when there are none).
|
||||||
|
diag_json() {
|
||||||
|
if (( ${#DIAG_ITEMS[@]} == 0 )); then echo "[]"; return 0; fi
|
||||||
|
printf '%s\n' "${DIAG_ITEMS[@]}" | jq -s -c '.'
|
||||||
|
}
|
||||||
|
|
||||||
CURL_INSECURE_FLAG=()
|
CURL_INSECURE_FLAG=()
|
||||||
case "${API_INSECURE,,}" in
|
case "${API_INSECURE,,}" in
|
||||||
1|true|yes) CURL_INSECURE_FLAG=(-k) ;;
|
1|true|yes) CURL_INSECURE_FLAG=(-k) ;;
|
||||||
@@ -182,6 +230,327 @@ collect_systemd_timers() {
|
|||||||
done < <(jq -c '.[]' <<<"$timers_json")
|
done < <(jq -c '.[]' <<<"$timers_json")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Updates waiting to be installed, as JSON in UPDATES_JSON ("null" when this host's package manager isn't one we know, or couldn't answer).
|
||||||
|
# Nothing is installed or refreshed: this reads what the package manager already knows about, so it is quick and changes nothing.
|
||||||
|
collect_updates() {
|
||||||
|
UPDATES_JSON="null"
|
||||||
|
local manager="" count="" security="null" reboot="false" out="" status=0
|
||||||
|
|
||||||
|
if command -v apt-get >/dev/null 2>&1; then
|
||||||
|
manager="apt"
|
||||||
|
# -s simulates: "Inst" lines are what an upgrade would install. Security ones come from a "-security" or "Security" archive.
|
||||||
|
if out=$(LC_ALL=C timeout 60 apt-get -s -o Debug::NoLocking=true upgrade 2>"$PM_ERR"); then
|
||||||
|
count=$(grep -c '^Inst ' <<<"$out" || true)
|
||||||
|
security=$(grep '^Inst ' <<<"$out" | grep -ci 'security' || true)
|
||||||
|
fi
|
||||||
|
[[ -f /var/run/reboot-required ]] && reboot="true"
|
||||||
|
elif command -v dnf >/dev/null 2>&1; then
|
||||||
|
manager="dnf"
|
||||||
|
# check-update exits 100 when there are updates, 0 when none, 1 on an error. --cacheonly: never goes to the network from here.
|
||||||
|
out=$(LC_ALL=C timeout 120 dnf -q check-update --cacheonly 2>"$PM_ERR") || status=$?
|
||||||
|
if [[ "$status" -eq 0 || "$status" -eq 100 ]]; then
|
||||||
|
count=$(awk 'NF >= 3 && $1 ~ /^[A-Za-z0-9_.+-]+\.[A-Za-z0-9_]+$/ && /Obsoleting/ == 0 { n++ } /^Obsoleting/ { exit } END { print n + 0 }' <<<"$out")
|
||||||
|
security=$(LC_ALL=C timeout 120 dnf -q updateinfo list --security --cacheonly 2>"$PM_ERR" | grep -c '/Sec\.' || true)
|
||||||
|
fi
|
||||||
|
if command -v needs-restarting >/dev/null 2>&1; then
|
||||||
|
needs-restarting -r >/dev/null 2>&1 || reboot="true"
|
||||||
|
fi
|
||||||
|
elif command -v pacman >/dev/null 2>&1; then
|
||||||
|
manager="pacman"
|
||||||
|
# -Qu is what the local copy of the sync databases says could be upgraded (what `checkupdates` lists, without changing anything). It
|
||||||
|
# exits 1 when there is nothing to upgrade, so an empty answer with status 1 is "none", not an error. Packages held back are left out.
|
||||||
|
out=$(LC_ALL=C timeout 60 pacman -Qu 2>"$PM_ERR") || status=$?
|
||||||
|
if [[ "$status" -eq 0 || ( "$status" -eq 1 && -z "$out" ) ]]; then
|
||||||
|
count=$(grep -v '\[ignored\]' <<<"$out" | grep -c . || true)
|
||||||
|
fi
|
||||||
|
# No security figure (pacman doesn't say). A restart is due when the kernel running is no longer the one installed.
|
||||||
|
[[ -d "/usr/lib/modules/$(uname -r)" ]] || reboot="true"
|
||||||
|
elif command -v zypper >/dev/null 2>&1; then
|
||||||
|
manager="zypper"
|
||||||
|
if out=$(LC_ALL=C timeout 120 zypper -q --no-refresh list-updates 2>"$PM_ERR"); then
|
||||||
|
count=$(grep -c '^v |' <<<"$out" || true)
|
||||||
|
fi
|
||||||
|
[[ -f /run/reboot-needed ]] && reboot="true"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -z "$manager" ]]; then
|
||||||
|
diag updates unavailable no_package_manager
|
||||||
|
elif [[ ! "$count" =~ ^[0-9]+$ ]]; then
|
||||||
|
# In the journal (systemctl status / journalctl -u homelab-manager-agent) so "not reported" has a reason to look for.
|
||||||
|
echo "Note: couldn't read the pending updates from $manager - the report goes without them." >&2
|
||||||
|
diag updates failed error "$manager did not answer"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -n "$manager" && "$count" =~ ^[0-9]+$ ]]; then
|
||||||
|
[[ "$security" =~ ^[0-9]+$ ]] || security="null"
|
||||||
|
UPDATES_JSON=$(jq -n --arg manager "$manager" --argjson count "$count" --argjson security "$security" --argjson reboot "$reboot" \
|
||||||
|
'{manager: $manager, count: $count, security: $security, reboot_required: $reboot}')
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# >>> post_report
|
||||||
|
# Sends $PAYLOAD to the server and prints the HTTP status code.
|
||||||
|
#
|
||||||
|
# Nothing sensitive goes on curl's command line: any user on this machine can read other processes' command lines, and the
|
||||||
|
# token would be there for the length of every report. So the token is handed to curl on its standard input, as a line of
|
||||||
|
# curl's own config format, and the payload and the server's reply go through files in a private directory (mktemp -d is
|
||||||
|
# readable by root only). A fixed name in /tmp is no good either — another user could plant a symlink there and have this
|
||||||
|
# script, running as root, overwrite whatever it points at.
|
||||||
|
curl_quote() {
|
||||||
|
local s=$1
|
||||||
|
s=${s//\\/\\\\}
|
||||||
|
s=${s//\"/\\\"}
|
||||||
|
printf '%s' "$s"
|
||||||
|
}
|
||||||
|
|
||||||
|
post_report() {
|
||||||
|
case "$API_TOKEN" in
|
||||||
|
*$'\n'*|*$'\r'*) echo "API_TOKEN must not contain a line break." >&2; return 1 ;;
|
||||||
|
esac
|
||||||
|
printf '%s' "$PAYLOAD" > "$HLM_TMPDIR/payload.json"
|
||||||
|
printf 'header = "Authorization: Bearer %s"\n' "$(curl_quote "$API_TOKEN")" | curl -sS "${CURL_INSECURE_FLAG[@]}" -K - \
|
||||||
|
-o "$HLM_TMPDIR/response.json" -w "%{http_code}" \
|
||||||
|
-X POST "$API_URL/api/agent/report" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
--data-binary @"$HLM_TMPDIR/payload.json"
|
||||||
|
}
|
||||||
|
# <<< post_report
|
||||||
|
|
||||||
|
# A GET with the token handed to curl on its standard input (as the upload does), the reply into the file named; prints the HTTP status.
|
||||||
|
api_get() {
|
||||||
|
printf 'header = "Authorization: Bearer %s"\n' "$(curl_quote "$API_TOKEN")" | curl -sS "${CURL_INSECURE_FLAG[@]}" -K - -m 30 -o "$2" -w "%{http_code}" "$API_URL$1"
|
||||||
|
}
|
||||||
|
|
||||||
|
# One value from the firmware's table (the kernel shows it under /sys/class/dmi/id), trying the named files in turn and leaving out what a
|
||||||
|
# maker writes when it has nothing to say ("To Be Filled By O.E.M.", "Default string", all zeros ...). DMI_DIR is for the tests.
|
||||||
|
dmi_value() {
|
||||||
|
local name value
|
||||||
|
for name in "$@"; do
|
||||||
|
# (the file is checked first: a failing redirection prints its complaint before any 2>/dev/null applies)
|
||||||
|
[[ -r "${DMI_DIR:-/sys/class/dmi/id}/$name" ]] || continue
|
||||||
|
value=$(tr -d '\0' < "${DMI_DIR:-/sys/class/dmi/id}/$name" 2>/dev/null | head -c 200 | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') || value=""
|
||||||
|
[[ -n "$value" ]] || continue
|
||||||
|
case "${value,,}" in
|
||||||
|
"to be filled by o.e.m."|"default string"|"system serial number"|"system product name"|"system manufacturer"|"not specified"|"not applicable"|"none"|"n/a"|"unknown"|"o.e.m."|"oem") continue ;;
|
||||||
|
esac
|
||||||
|
[[ "$value" =~ ^0+$ ]] && continue
|
||||||
|
printf '%s' "$value"
|
||||||
|
return 0
|
||||||
|
done
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# What the machine is: maker, model and serial number (a serial is only readable by root). "null" when the firmware has nothing to say, as on
|
||||||
|
# many virtual machines and single-board computers.
|
||||||
|
collect_hardware() {
|
||||||
|
HARDWARE_JSON="null"
|
||||||
|
local maker model serial
|
||||||
|
maker=$(dmi_value sys_vendor board_vendor)
|
||||||
|
model=$(dmi_value product_name board_name)
|
||||||
|
serial=$(dmi_value product_serial board_serial)
|
||||||
|
if [[ -z "$maker$model$serial" ]]; then
|
||||||
|
diag hardware unavailable not_found
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
HARDWARE_JSON=$(jq -n -c --arg maker "$maker" --arg model "$model" --arg serial "$serial" \
|
||||||
|
'{} | (if $maker != "" then . + {manufacturer: $maker} else . end) | (if $model != "" then . + {model: $model} else . end) | (if $serial != "" then . + {serial: $serial} else . end)') || HARDWARE_JSON="null"
|
||||||
|
}
|
||||||
|
|
||||||
|
# What the machine runs, as /etc/os-release says (OS_RELEASE_FILE is for the tests): the distribution's id and version, so the app can say when that release stops
|
||||||
|
# being supported. Proxmox VE is Debian underneath, so it is asked by its own command. "null" when there is nothing to say.
|
||||||
|
collect_os() {
|
||||||
|
OS_JSON="null"
|
||||||
|
local file="${OS_RELEASE_FILE:-/etc/os-release}" id="" version="" pretty=""
|
||||||
|
if [[ -r "$file" ]]; then
|
||||||
|
id=$(sed -n 's/^ID=//p' "$file" | head -n 1 | tr -d '"' | tr 'A-Z' 'a-z' | head -c 40)
|
||||||
|
version=$(sed -n 's/^VERSION_ID=//p' "$file" | head -n 1 | tr -d '"' | head -c 40)
|
||||||
|
pretty=$(sed -n 's/^PRETTY_NAME=//p' "$file" | head -n 1 | tr -d '"' | head -c 80)
|
||||||
|
fi
|
||||||
|
if command -v pveversion >/dev/null 2>&1; then
|
||||||
|
local pve
|
||||||
|
pve=$(pveversion 2>/dev/null | sed -n 's#^pve-manager/\([0-9]*\.[0-9]*\).*#\1#p' | head -n 1)
|
||||||
|
if [[ -n "$pve" ]]; then
|
||||||
|
id="proxmox-ve"
|
||||||
|
version="$pve"
|
||||||
|
pretty="Proxmox VE $pve"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
[[ -n "$id" ]] || return 0
|
||||||
|
OS_JSON=$(jq -n -c --arg id "$id" --arg version "$version" --arg name "$pretty" \
|
||||||
|
'{id: $id} | (if $version != "" then . + {version: $version} else . end) | (if $name != "" then . + {name: $name} else . end)') || OS_JSON="null"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Services (systemd units) that have failed. systemctl is asked the way the timers are, so this works inside the sandbox.
|
||||||
|
collect_failed_units() {
|
||||||
|
FAILED_UNITS_JSON="null"
|
||||||
|
if ! command -v systemctl >/dev/null 2>&1; then
|
||||||
|
diag failed_units unavailable no_systemd
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
local out
|
||||||
|
if ! out=$(timeout 20 systemctl list-units --state=failed --plain --no-legend --no-pager --full 2>"$PM_ERR"); then
|
||||||
|
diag failed_units failed error "systemctl list-units did not answer"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
FAILED_UNITS_JSON=$(awk 'NF > 0 { unit = $1; $1 = $2 = $3 = $4 = ""; sub(/^ +/, ""); print unit "\t" $0 }' <<<"$out" \
|
||||||
|
| jq -R -s -c 'split("\n") | map(select(length > 0) | split("\t")) | map({unit: .[0], description: (.[1] // "")}) | .[0:100]') || FAILED_UNITS_JSON="null"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Each disk's own SMART verdict, temperature and (where it says) how much of its rated life is used. Needs the devices, so it runs from
|
||||||
|
# --prepare. Asked at most about once an hour, and with -n standby so a sleeping disk is not woken: what was last known of it is kept.
|
||||||
|
collect_smart() {
|
||||||
|
SMART_JSON="null"
|
||||||
|
if ! command -v smartctl >/dev/null 2>&1; then
|
||||||
|
diag smart unavailable tool_missing smartctl
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
if [[ "$(id -u)" != "0" ]]; then
|
||||||
|
diag smart unavailable not_root
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
local cache="$STATE_DIR/smart.json" now_s previous="[]" items="[]" dev out item scan checked
|
||||||
|
now_s=$(date +%s)
|
||||||
|
if [[ -r "$cache" ]]; then
|
||||||
|
checked=$(jq -r '.checked_at // 0' "$cache" 2>/dev/null || echo 0)
|
||||||
|
previous=$(jq -c '.devices // []' "$cache" 2>/dev/null || echo "[]")
|
||||||
|
if [[ "$checked" =~ ^[0-9]+$ ]] && (( now_s - checked < 3300 )) && [[ "$MODE" != "check" ]]; then
|
||||||
|
SMART_JSON=$previous
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
scan=$(timeout 30 smartctl --scan -j 2>"$PM_ERR") || true
|
||||||
|
while IFS= read -r dev; do
|
||||||
|
[[ -n "$dev" ]] || continue
|
||||||
|
# smartctl's exit status is a bit mask that says "something was noted" more often than "it failed", so it is read from the JSON instead.
|
||||||
|
out=$(timeout 30 smartctl -j -H -A -n standby "$dev" 2>"$PM_ERR") || true
|
||||||
|
item=$(jq -c --arg dev "$dev" 'select(.smart_status != null) | {
|
||||||
|
device: $dev,
|
||||||
|
model: (.model_name // .scsi_model_name // ""),
|
||||||
|
healthy: .smart_status.passed,
|
||||||
|
temperature_c: (.temperature.current // null),
|
||||||
|
wear_percent: (.nvme_smart_health_information_log.percentage_used // ([.ata_smart_attributes.table[]? | select(.id == 177 or .id == 231 or .id == 233) | (100 - .value)] | first) // null)
|
||||||
|
}' <<<"$out" 2>/dev/null) || item=""
|
||||||
|
if [[ -z "$item" ]]; then
|
||||||
|
# Asleep, or no SMART at all (a virtual disk): keep what was known of it, if anything.
|
||||||
|
item=$(jq -c --arg dev "$dev" '.[] | select(.device == $dev)' <<<"$previous" 2>/dev/null | head -n 1) || item=""
|
||||||
|
fi
|
||||||
|
if [[ -n "$item" ]]; then items=$(jq -c --argjson i "$item" '. + [$i]' <<<"$items"); fi
|
||||||
|
done < <(jq -r '.devices[]?.name' <<<"$scan" 2>/dev/null)
|
||||||
|
SMART_JSON=$items
|
||||||
|
if mkdir -p "$STATE_DIR" 2>/dev/null; then
|
||||||
|
chmod 700 "$STATE_DIR" 2>/dev/null || true
|
||||||
|
jq -c -n --argjson at "$now_s" --argjson devices "$items" '{checked_at: $at, devices: $devices}' > "$cache.tmp" 2>/dev/null && mv -f "$cache.tmp" "$cache" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Each ZFS pool and whether it is ONLINE. Needs /dev/zfs, so it runs from --prepare too.
|
||||||
|
collect_zfs() {
|
||||||
|
ZFS_JSON="null"
|
||||||
|
if ! command -v zpool >/dev/null 2>&1; then
|
||||||
|
diag zfs unavailable tool_missing zpool
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
local out
|
||||||
|
if ! out=$(timeout 20 zpool list -H -o name,health,capacity 2>"$PM_ERR"); then
|
||||||
|
diag zfs failed error "zpool list did not answer"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
ZFS_JSON=$(jq -R -s -c 'split("\n") | map(select(length > 0) | split("\t")) | map({name: .[0], health: (.[1] // "UNKNOWN"), capacity_percent: ((.[2] // "" | rtrimstr("%") | tonumber?) // null)})' <<<"$out") || ZFS_JSON="null"
|
||||||
|
}
|
||||||
|
|
||||||
|
# The disk and pool readings --prepare left for this report, if they are there and recent. Re-shaped by jq, never trusted as they stand.
|
||||||
|
use_prepared_health() {
|
||||||
|
[[ -r "$HEALTH_FILE" ]] || return 1
|
||||||
|
[[ -n "$(find "$HEALTH_FILE" -mmin -10 -print 2>/dev/null)" ]] || return 1
|
||||||
|
SMART_JSON=$(jq -c 'if (.smart | type) == "array" then [.smart[] | {device: (.device | tostring), model: ((.model // "") | tostring),
|
||||||
|
healthy: (if (.healthy | type) == "boolean" then .healthy else null end),
|
||||||
|
temperature_c: (if (.temperature_c | type) == "number" then .temperature_c else null end),
|
||||||
|
wear_percent: (if (.wear_percent | type) == "number" then .wear_percent else null end)}] else null end' "$HEALTH_FILE" 2>/dev/null) || SMART_JSON="null"
|
||||||
|
ZFS_JSON=$(jq -c 'if (.zfs | type) == "array" then [.zfs[] | {name: (.name | tostring), health: ((.health // "UNKNOWN") | tostring),
|
||||||
|
capacity_percent: (if (.capacity_percent | type) == "number" then .capacity_percent else null end)}] else null end' "$HEALTH_FILE" 2>/dev/null) || ZFS_JSON="null"
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# Replaces this script with the one this Homelab Manager serves, when an administrator has switched automatic updates on and this agent is
|
||||||
|
# behind. Nothing is trusted but the checksum: the (authenticated) answer gives the SHA-256 of the script, and a download that doesn't
|
||||||
|
# match it, doesn't look like this script, doesn't pass bash's syntax check or isn't newer is thrown away. The new file is put beside the
|
||||||
|
# old and moved over it in one step, and the new version then carries on from here. Runs only from --prepare (outside the sandbox, which
|
||||||
|
# keeps /usr read-only for the report itself); with --check-updates it says what it would do.
|
||||||
|
self_update() {
|
||||||
|
[[ -n "$API_URL" && -n "$API_TOKEN" ]] || return 0
|
||||||
|
[[ -z "${HLM_JUST_UPDATED:-}" ]] || return 0
|
||||||
|
local self work code reply_update version sha got
|
||||||
|
self=$(readlink -f "${BASH_SOURCE[0]}" 2>/dev/null) || return 0
|
||||||
|
[[ -f "$self" ]] || return 0
|
||||||
|
work=$(mktemp -d) || return 0
|
||||||
|
code=$(api_get "/api/agent/report/update?version=$AGENT_VERSION" "$work/reply.json" 2>/dev/null) || code=""
|
||||||
|
reply_update=$(jq -r '.update // false' "$work/reply.json" 2>/dev/null || echo false)
|
||||||
|
if [[ "$code" != "200" || "$reply_update" != "true" ]]; then
|
||||||
|
[[ "$MODE" == "check" ]] && echo "Automatic update: nothing to do (it is off, or this agent is current)."
|
||||||
|
rm -rf "$work"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
version=$(jq -r '.version // empty' "$work/reply.json" 2>/dev/null || true)
|
||||||
|
sha=$(jq -r '.sha256 // empty' "$work/reply.json" 2>/dev/null || true)
|
||||||
|
if [[ ! "$version" =~ ^[0-9]+$ || ! "$sha" =~ ^[0-9a-f]{64}$ ]] || (( version <= AGENT_VERSION )); then
|
||||||
|
rm -rf "$work"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
if ! curl -fsS "${CURL_INSECURE_FLAG[@]}" -m 60 -o "$work/new.sh" "$API_URL/agent/linux/report-tasks.sh" 2>"$PM_ERR"; then
|
||||||
|
echo "Note: couldn't download the newer agent (version $version), so it was not installed." >&2
|
||||||
|
rm -rf "$work"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
got=$(sha256sum "$work/new.sh" 2>/dev/null | awk '{print $1}')
|
||||||
|
if [[ "$got" != "$sha" ]] || [[ "$(head -n 1 "$work/new.sh")" != "#!/usr/bin/env bash" ]] || ! bash -n "$work/new.sh" 2>/dev/null || ! grep -q "^AGENT_VERSION=$version\$" "$work/new.sh"; then
|
||||||
|
echo "Note: the newer agent (version $version) didn't check out against what the server said, so it was not installed." >&2
|
||||||
|
rm -rf "$work"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
if [[ "$MODE" == "check" ]]; then
|
||||||
|
echo "Automatic update: version $version is available and would be installed on the next run."
|
||||||
|
rm -rf "$work"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
if install -m 755 "$work/new.sh" "$self.new" && mv -f "$self.new" "$self"; then
|
||||||
|
echo "Updated the agent from version $AGENT_VERSION to version $version." >&2
|
||||||
|
rm -rf "$work"
|
||||||
|
HLM_JUST_UPDATED=1 exec "$self" --prepare
|
||||||
|
fi
|
||||||
|
rm -f "$self.new"
|
||||||
|
echo "Note: couldn't replace the agent with version $version." >&2
|
||||||
|
rm -rf "$work"
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# The answer --prepare left for this report, if it is there, recent and looks like what it should. Read back through jq rather than
|
||||||
|
# trusted: the report only ever carries these four fields, whatever the file holds.
|
||||||
|
use_prepared_updates() {
|
||||||
|
[[ -r "$UPDATES_FILE" ]] || return 1
|
||||||
|
[[ -n "$(find "$UPDATES_FILE" -mmin -10 -print 2>/dev/null)" ]] || return 1
|
||||||
|
local json
|
||||||
|
json=$(jq -ce 'select(type == "object" and (.manager | type) == "string" and (.count | type) == "number")
|
||||||
|
| {manager: .manager, count: .count, security: (if (.security | type) == "number" then .security else null end), reboot_required: (.reboot_required == true)}' "$UPDATES_FILE" 2>/dev/null) || return 1
|
||||||
|
[[ -n "$json" ]] || return 1
|
||||||
|
UPDATES_JSON=$json
|
||||||
|
}
|
||||||
|
|
||||||
|
# What this report could not collect: the notes made here, then those --prepare left (it does the outside-the-sandbox part), one for each check.
|
||||||
|
# With nothing left by --prepare — on a real run, not a dry one — that step did not run, which is worth saying: disk health and ZFS are only read there.
|
||||||
|
collect_diagnostics() {
|
||||||
|
local from_prepare="[]" shaped
|
||||||
|
if [[ -r "$DIAG_FILE" && -n "$(find "$DIAG_FILE" -mmin -10 -print 2>/dev/null)" ]]; then
|
||||||
|
from_prepare=$(jq -c 'if type == "array" then [.[] | select(type == "object" and (.check | type) == "string" and (.status | type) == "string")
|
||||||
|
| {check: (.check | tostring), status: (.status | tostring)}
|
||||||
|
+ (if ((.reason // "") | tostring) != "" then {reason: (.reason | tostring)} else {} end)
|
||||||
|
+ (if ((.detail // "") | tostring) != "" then {detail: (.detail | tostring | .[0:160])} else {} end)] | .[0:20] else [] end' "$DIAG_FILE" 2>/dev/null) || from_prepare="[]"
|
||||||
|
elif [[ "$DRY_RUN" != "1" ]]; then
|
||||||
|
diag prepare unavailable step_missing
|
||||||
|
fi
|
||||||
|
shaped=$(jq -c -n --argjson here "$(diag_json)" --argjson there "$from_prepare" '($here + $there) | unique_by(.check) | .[0:20]') || shaped="[]"
|
||||||
|
DIAGNOSTICS_JSON=$shaped
|
||||||
|
}
|
||||||
|
|
||||||
collect_system_info() {
|
collect_system_info() {
|
||||||
local ip_json="[]"
|
local ip_json="[]"
|
||||||
if command -v ip >/dev/null 2>&1; then
|
if command -v ip >/dev/null 2>&1; then
|
||||||
@@ -210,6 +579,13 @@ collect_system_info() {
|
|||||||
cpu_load_percent=$(awk -v l="$load1" -v c="$cpu_cores" 'BEGIN { printf "%.1f", (l/c)*100 }')
|
cpu_load_percent=$(awk -v l="$load1" -v c="$cpu_cores" 'BEGIN { printf "%.1f", (l/c)*100 }')
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# How long the host has been up: the first figure in /proc/uptime, in seconds. null if it cannot be read.
|
||||||
|
local uptime_seconds="null"
|
||||||
|
if [[ -r /proc/uptime ]]; then
|
||||||
|
uptime_seconds=$(awk '{ printf "%d", $1 }' /proc/uptime 2>/dev/null || true)
|
||||||
|
[[ "$uptime_seconds" =~ ^[0-9]+$ ]] || uptime_seconds="null"
|
||||||
|
fi
|
||||||
|
|
||||||
local mem_total=0 mem_used=0
|
local mem_total=0 mem_used=0
|
||||||
if command -v free >/dev/null 2>&1; then
|
if command -v free >/dev/null 2>&1; then
|
||||||
read -r mem_total mem_used < <(free -b | awk '/^Mem:/ {print $2, $3}')
|
read -r mem_total mem_used < <(free -b | awk '/^Mem:/ {print $2, $3}')
|
||||||
@@ -243,7 +619,35 @@ collect_system_info() {
|
|||||||
')
|
')
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Best-effort like the rest: a package manager that misbehaves must not cost the report.
|
||||||
|
UPDATES_JSON="null"
|
||||||
|
use_prepared_updates || collect_updates || UPDATES_JSON="null"
|
||||||
|
|
||||||
|
# Failed services are asked here (systemctl works inside the sandbox); disk and pool health come from --prepare. A person running --dry-run
|
||||||
|
# isn't in the sandbox, so there they are read directly.
|
||||||
|
FAILED_UNITS_JSON="null"
|
||||||
|
collect_failed_units || FAILED_UNITS_JSON="null"
|
||||||
|
SMART_JSON="null"
|
||||||
|
ZFS_JSON="null"
|
||||||
|
if ! use_prepared_health && [[ "$DRY_RUN" == "1" ]]; then
|
||||||
|
collect_smart || SMART_JSON="null"
|
||||||
|
collect_zfs || ZFS_JSON="null"
|
||||||
|
fi
|
||||||
|
HARDWARE_JSON="null"
|
||||||
|
collect_hardware || HARDWARE_JSON="null"
|
||||||
|
OS_JSON="null"
|
||||||
|
collect_os || OS_JSON="null"
|
||||||
|
collect_diagnostics
|
||||||
|
HEALTH_JSON=$(jq -n -c --argjson failed "$FAILED_UNITS_JSON" --argjson smart "$SMART_JSON" --argjson zfs "$ZFS_JSON" \
|
||||||
|
'{} | (if $failed != null then . + {failed_units: $failed} else . end) | (if $smart != null then . + {smart: $smart} else . end) | (if $zfs != null then . + {zfs: $zfs} else . end)') || HEALTH_JSON="{}"
|
||||||
|
[[ "$HEALTH_JSON" == "{}" ]] && HEALTH_JSON="null"
|
||||||
|
|
||||||
SYSTEM_JSON=$(jq -n \
|
SYSTEM_JSON=$(jq -n \
|
||||||
|
--argjson updates "$UPDATES_JSON" \
|
||||||
|
--argjson health "$HEALTH_JSON" \
|
||||||
|
--argjson hardware "$HARDWARE_JSON" \
|
||||||
|
--argjson os "$OS_JSON" \
|
||||||
|
--argjson diagnostics "$DIAGNOSTICS_JSON" \
|
||||||
--argjson ip_addresses "$ip_json" \
|
--argjson ip_addresses "$ip_json" \
|
||||||
--argjson listening_ports "$ports_json" \
|
--argjson listening_ports "$ports_json" \
|
||||||
--arg cpu_model "$cpu_model" \
|
--arg cpu_model "$cpu_model" \
|
||||||
@@ -251,16 +655,63 @@ collect_system_info() {
|
|||||||
--argjson cpu_load_percent "$cpu_load_percent" \
|
--argjson cpu_load_percent "$cpu_load_percent" \
|
||||||
--argjson mem_total_bytes "${mem_total:-0}" \
|
--argjson mem_total_bytes "${mem_total:-0}" \
|
||||||
--argjson mem_used_bytes "${mem_used:-0}" \
|
--argjson mem_used_bytes "${mem_used:-0}" \
|
||||||
|
--argjson uptime_seconds "$uptime_seconds" \
|
||||||
--argjson disks "$disks_json" \
|
--argjson disks "$disks_json" \
|
||||||
'{
|
'{
|
||||||
ip_addresses: $ip_addresses,
|
ip_addresses: $ip_addresses,
|
||||||
cpu: { model: $cpu_model, cores: $cpu_cores, load_percent: $cpu_load_percent },
|
cpu: { model: $cpu_model, cores: $cpu_cores, load_percent: $cpu_load_percent },
|
||||||
memory: { total_bytes: $mem_total_bytes, used_bytes: $mem_used_bytes },
|
memory: { total_bytes: $mem_total_bytes, used_bytes: $mem_used_bytes },
|
||||||
|
uptime_seconds: $uptime_seconds,
|
||||||
disks: $disks,
|
disks: $disks,
|
||||||
listening_ports: $listening_ports
|
listening_ports: $listening_ports,
|
||||||
|
updates: $updates,
|
||||||
|
health: $health,
|
||||||
|
hardware: $hardware,
|
||||||
|
os: $os,
|
||||||
|
diagnostics: $diagnostics
|
||||||
}')
|
}')
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if [[ "$MODE" != "report" ]]; then
|
||||||
|
# prepare: first see whether this script is to be replaced (if so it carries on as the new one, from the top).
|
||||||
|
[[ "$MODE" == "prepare" ]] && self_update
|
||||||
|
collect_updates || UPDATES_JSON="null"
|
||||||
|
collect_smart || SMART_JSON="null"
|
||||||
|
collect_zfs || ZFS_JSON="null"
|
||||||
|
if [[ "$MODE" == "check" ]]; then
|
||||||
|
collect_failed_units || FAILED_UNITS_JSON="null"
|
||||||
|
echo "Pending updates as they would be reported:"
|
||||||
|
jq . <<<"$UPDATES_JSON"
|
||||||
|
echo "Failed services:"
|
||||||
|
jq . <<<"$FAILED_UNITS_JSON"
|
||||||
|
echo "Disk health (SMART; null means smartctl isn't installed or this isn't root):"
|
||||||
|
jq . <<<"$SMART_JSON"
|
||||||
|
echo "ZFS pools (null means there is no zpool here):"
|
||||||
|
jq . <<<"$ZFS_JSON"
|
||||||
|
collect_hardware || true
|
||||||
|
collect_os || true
|
||||||
|
echo "What it runs (null means there is no os-release to read):"
|
||||||
|
jq . <<<"$OS_JSON"
|
||||||
|
echo "What could not be collected (empty means nothing was missed):"
|
||||||
|
diag_json | jq .
|
||||||
|
self_update
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
# Leave the answers (or no file at all) for the report that follows. Each is written whole and moved into place, so a report never reads half of one.
|
||||||
|
rm -f "$UPDATES_FILE" "$HEALTH_FILE" "$DIAG_FILE"
|
||||||
|
mkdir -p "$(dirname "$UPDATES_FILE")" 2>/dev/null || exit 0
|
||||||
|
# The notes always go, even when there are none: a missing file is how the report knows this step didn't run.
|
||||||
|
diag_json > "$DIAG_FILE.tmp" 2>/dev/null && mv -f "$DIAG_FILE.tmp" "$DIAG_FILE" 2>/dev/null || true
|
||||||
|
if [[ "$UPDATES_JSON" != "null" ]]; then
|
||||||
|
printf '%s\n' "$UPDATES_JSON" > "$UPDATES_FILE.tmp" 2>/dev/null && mv -f "$UPDATES_FILE.tmp" "$UPDATES_FILE" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
HEALTH_FILE_JSON=$(jq -n -c --argjson smart "${SMART_JSON:-null}" --argjson zfs "${ZFS_JSON:-null}" '{} | (if $smart != null then . + {smart: $smart} else . end) | (if $zfs != null then . + {zfs: $zfs} else . end)' 2>/dev/null || echo "{}")
|
||||||
|
if [[ "$HEALTH_FILE_JSON" != "{}" ]]; then
|
||||||
|
printf '%s\n' "$HEALTH_FILE_JSON" > "$HEALTH_FILE.tmp" 2>/dev/null && mv -f "$HEALTH_FILE.tmp" "$HEALTH_FILE" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
collect_cron
|
collect_cron
|
||||||
collect_systemd_timers
|
collect_systemd_timers
|
||||||
|
|
||||||
@@ -282,43 +733,16 @@ PAYLOAD=$(jq -n \
|
|||||||
--arg hostname "$HOSTNAME_VALUE" \
|
--arg hostname "$HOSTNAME_VALUE" \
|
||||||
--arg os_type "linux" \
|
--arg os_type "linux" \
|
||||||
--arg reported_at "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
|
--arg reported_at "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
|
||||||
|
--argjson agent_version "$AGENT_VERSION" \
|
||||||
--argjson system "$SYSTEM_JSON" \
|
--argjson system "$SYSTEM_JSON" \
|
||||||
--argjson tasks "$TASKS_JSON" \
|
--argjson tasks "$TASKS_JSON" \
|
||||||
'{hostname: $hostname, os_type: $os_type, reported_at: $reported_at, system: $system, tasks: $tasks}')
|
'{hostname: $hostname, os_type: $os_type, reported_at: $reported_at, agent_version: $agent_version, system: $system, tasks: $tasks}')
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == "1" ]]; then
|
if [[ "$DRY_RUN" == "1" ]]; then
|
||||||
echo "$PAYLOAD" | jq .
|
echo "$PAYLOAD" | jq .
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# >>> post_report
|
|
||||||
# Sends $PAYLOAD to the server and prints the HTTP status code.
|
|
||||||
#
|
|
||||||
# Nothing sensitive goes on curl's command line: any user on this machine can read other processes' command lines, and the
|
|
||||||
# token would be there for the length of every report. So the token is handed to curl on its standard input, as a line of
|
|
||||||
# curl's own config format, and the payload and the server's reply go through files in a private directory (mktemp -d is
|
|
||||||
# readable by root only). A fixed name in /tmp is no good either — another user could plant a symlink there and have this
|
|
||||||
# script, running as root, overwrite whatever it points at.
|
|
||||||
curl_quote() {
|
|
||||||
local s=$1
|
|
||||||
s=${s//\\/\\\\}
|
|
||||||
s=${s//\"/\\\"}
|
|
||||||
printf '%s' "$s"
|
|
||||||
}
|
|
||||||
|
|
||||||
post_report() {
|
|
||||||
case "$API_TOKEN" in
|
|
||||||
*$'\n'*|*$'\r'*) echo "API_TOKEN must not contain a line break." >&2; return 1 ;;
|
|
||||||
esac
|
|
||||||
printf '%s' "$PAYLOAD" > "$HLM_TMPDIR/payload.json"
|
|
||||||
printf 'header = "Authorization: Bearer %s"\n' "$(curl_quote "$API_TOKEN")" | curl -sS "${CURL_INSECURE_FLAG[@]}" -K - \
|
|
||||||
-o "$HLM_TMPDIR/response.json" -w "%{http_code}" \
|
|
||||||
-X POST "$API_URL/api/agent/report" \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
--data-binary @"$HLM_TMPDIR/payload.json"
|
|
||||||
}
|
|
||||||
# <<< post_report
|
|
||||||
|
|
||||||
HLM_TMPDIR=$(mktemp -d)
|
HLM_TMPDIR=$(mktemp -d)
|
||||||
trap 'rm -rf "$HLM_TMPDIR"' EXIT
|
trap 'rm -rf "$HLM_TMPDIR"' EXIT
|
||||||
|
|
||||||
|
|||||||
@@ -33,6 +33,9 @@ rm -f \
|
|||||||
"$INSTALL_DIR/homelab-manager-agent.sh" \
|
"$INSTALL_DIR/homelab-manager-agent.sh" \
|
||||||
/tmp/hlm-agent-response.json
|
/tmp/hlm-agent-response.json
|
||||||
|
|
||||||
|
# What it remembered between runs (the last disk-health reading).
|
||||||
|
rm -rf /var/lib/homelab-manager-agent
|
||||||
|
|
||||||
if command -v systemctl >/dev/null 2>&1; then
|
if command -v systemctl >/dev/null 2>&1; then
|
||||||
systemctl daemon-reload
|
systemctl daemon-reload
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -6,7 +6,10 @@ runs as a scheduled task (as SYSTEM, every 15 minutes) and pushes to `POST /api/
|
|||||||
the machine never needs to be reachable from Homelab Manager.
|
the machine never needs to be reachable from Homelab Manager.
|
||||||
|
|
||||||
|
|
||||||
|
Each report also says how many updates Windows Update has waiting (and how many are security or critical, and whether a restart is due). Searching Windows Update goes over the network and can take a minute or two, so it runs in a separate process with a 150-second limit and is done at most every six hours; the answer is kept in `%ProgramData%\HomelabManager\updates.json` and reused between (a search that fails is not tried again for an hour, and that report simply leaves updates out). Nothing is downloaded or installed. To update an agent that was installed before this existed, see [AGENT_UPDATE.md](../../AGENT_UPDATE.md).
|
||||||
|
|
||||||
The installer limits `%ProgramData%\HomelabManager` to SYSTEM and Administrators. The agent runs as SYSTEM, so anyone who could change its script could run code as SYSTEM; the folder is not left to inherit ProgramData's default permissions. Re-running the install one-liner applies this to an existing install.
|
The installer limits `%ProgramData%\HomelabManager` to SYSTEM and Administrators. The agent runs as SYSTEM, so anyone who could change its script could run code as SYSTEM; the folder is not left to inherit ProgramData's default permissions. Re-running the install one-liner applies this to an existing install.
|
||||||
|
|
||||||
## Install
|
## Install
|
||||||
|
|
||||||
1. In Homelab Manager, **Servers → Manage servers → Add a server**, choose **Windows** as the
|
1. In Homelab Manager, **Servers → Manage servers → Add a server**, choose **Windows** as the
|
||||||
|
|||||||
@@ -31,12 +31,31 @@ param(
|
|||||||
|
|
||||||
$ErrorActionPreference = 'Stop'
|
$ErrorActionPreference = 'Stop'
|
||||||
|
|
||||||
|
# Which version of this script this is, sent with every report so Homelab Manager can tell which agents are behind. Raise it whenever a
|
||||||
|
# change here is worth installing the agent again for. 1 = before the number was reported 2 = reports pending updates, and its own version
|
||||||
|
# 3 = can replace itself with the newer script when an administrator has switched that on in Homelab Manager
|
||||||
|
# 4 = also reports services that have failed, disk health, Storage Spaces pools and what the machine is (maker, model, serial)
|
||||||
|
# 5 = also says what it could not collect and why, so Homelab Manager can say what to fix
|
||||||
|
$script:AgentVersion = 7
|
||||||
|
|
||||||
$script:DefaultConfigPath = Join-Path $env:ProgramData 'HomelabManager\agent.json'
|
$script:DefaultConfigPath = Join-Path $env:ProgramData 'HomelabManager\agent.json'
|
||||||
|
|
||||||
|
# Things that could not be collected: which check, whether it is 'unavailable' (nothing there to read) or 'failed' (it tried and could not), why in a
|
||||||
|
# word, and at most a line of what Windows said. Only problems are noted; a check that is not mentioned went fine.
|
||||||
|
$script:Diag = New-Object System.Collections.ArrayList
|
||||||
|
function Add-Diag([string]$Check, [string]$Status, [string]$Reason = '', [string]$Detail = '') {
|
||||||
|
$item = [ordered]@{ check = $Check; status = $Status }
|
||||||
|
if ($Reason) { $item['reason'] = $Reason }
|
||||||
|
$clean = ($Detail -replace '[\x00-\x1f]', ' ').Trim()
|
||||||
|
if ($clean.Length -gt 160) { $clean = $clean.Substring(0, 160) }
|
||||||
|
if ($clean) { $item['detail'] = $clean }
|
||||||
|
[void]$script:Diag.Add([pscustomobject]$item)
|
||||||
|
}
|
||||||
|
|
||||||
# ---- configuration ------------------------------------------------------------
|
# ---- configuration ------------------------------------------------------------
|
||||||
|
|
||||||
function Get-AgentConfig {
|
function Get-AgentConfig {
|
||||||
$config = @{ ApiUrl = $null; ApiToken = $null; Insecure = $false; IncludeMicrosoftTasks = $false }
|
$config = @{ ApiUrl = $null; ApiToken = $null; Insecure = $false; IncludeMicrosoftTasks = $false; IgnoreServices = @() }
|
||||||
|
|
||||||
$path = if ($env:HLM_CONFIG) { $env:HLM_CONFIG } else { $script:DefaultConfigPath }
|
$path = if ($env:HLM_CONFIG) { $env:HLM_CONFIG } else { $script:DefaultConfigPath }
|
||||||
if (Test-Path -LiteralPath $path) {
|
if (Test-Path -LiteralPath $path) {
|
||||||
@@ -45,6 +64,7 @@ function Get-AgentConfig {
|
|||||||
if ($file.apiToken) { $config.ApiToken = [string]$file.apiToken }
|
if ($file.apiToken) { $config.ApiToken = [string]$file.apiToken }
|
||||||
if ($null -ne $file.insecure) { $config.Insecure = [bool]$file.insecure }
|
if ($null -ne $file.insecure) { $config.Insecure = [bool]$file.insecure }
|
||||||
if ($null -ne $file.includeMicrosoftTasks) { $config.IncludeMicrosoftTasks = [bool]$file.includeMicrosoftTasks }
|
if ($null -ne $file.includeMicrosoftTasks) { $config.IncludeMicrosoftTasks = [bool]$file.includeMicrosoftTasks }
|
||||||
|
if ($file.ignoreServices) { $config.IgnoreServices = @($file.ignoreServices | ForEach-Object { [string]$_ }) }
|
||||||
}
|
}
|
||||||
|
|
||||||
# Environment variables win over the file, like the Linux agent.
|
# Environment variables win over the file, like the Linux agent.
|
||||||
@@ -52,6 +72,7 @@ function Get-AgentConfig {
|
|||||||
if ($env:API_TOKEN) { $config.ApiToken = $env:API_TOKEN }
|
if ($env:API_TOKEN) { $config.ApiToken = $env:API_TOKEN }
|
||||||
if ($env:API_INSECURE) { $config.Insecure = $env:API_INSECURE -match '^(1|true|yes)$' }
|
if ($env:API_INSECURE) { $config.Insecure = $env:API_INSECURE -match '^(1|true|yes)$' }
|
||||||
if ($env:INCLUDE_MICROSOFT_TASKS) { $config.IncludeMicrosoftTasks = $env:INCLUDE_MICROSOFT_TASKS -match '^(1|true|yes)$' }
|
if ($env:INCLUDE_MICROSOFT_TASKS) { $config.IncludeMicrosoftTasks = $env:INCLUDE_MICROSOFT_TASKS -match '^(1|true|yes)$' }
|
||||||
|
if ($env:IGNORE_SERVICES) { $config.IgnoreServices = @($env:IGNORE_SERVICES -split ',' | ForEach-Object { $_.Trim() } | Where-Object { $_ }) }
|
||||||
|
|
||||||
if ($config.ApiUrl) { $config.ApiUrl = $config.ApiUrl.TrimEnd('/') }
|
if ($config.ApiUrl) { $config.ApiUrl = $config.ApiUrl.TrimEnd('/') }
|
||||||
return $config
|
return $config
|
||||||
@@ -220,7 +241,184 @@ function Get-ListeningPorts {
|
|||||||
return @($rows | Where-Object { $_.port -ge 1 -and $_.port -le 65535 } | Select-Object -First 2000)
|
return @($rows | Where-Object { $_.port -ge 1 -and $_.port -le 65535 } | Select-Object -First 2000)
|
||||||
}
|
}
|
||||||
|
|
||||||
function Get-SystemInfo {
|
# ---- updates waiting to be installed ------------------------------------------------------------
|
||||||
|
$script:UpdatesCachePath = Join-Path $env:ProgramData 'HomelabManager\updates.json'
|
||||||
|
|
||||||
|
function Test-RebootRequired {
|
||||||
|
foreach ($key in @(
|
||||||
|
'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\WindowsUpdate\Auto Update\RebootRequired',
|
||||||
|
'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Component Based Servicing\RebootPending')) {
|
||||||
|
if (Test-Path $key) { return $true }
|
||||||
|
}
|
||||||
|
return $false
|
||||||
|
}
|
||||||
|
|
||||||
|
# Asks Windows Update what is waiting. That goes over the network and can take a minute or two, and the scheduled task is stopped
|
||||||
|
# after five minutes, so the search runs in a separate process with a time limit, and is repeated at most every six hours; the last
|
||||||
|
# answer is reused in between. A search that fails or runs out of time is not tried again for an hour, so a broken Windows Update
|
||||||
|
# can't slow down every report. Whether a restart is due is cheap and always read fresh. Nothing is installed or downloaded.
|
||||||
|
function Get-PendingUpdates {
|
||||||
|
$cached = $null
|
||||||
|
try {
|
||||||
|
if (Test-Path $script:UpdatesCachePath) {
|
||||||
|
$cached = Get-Content -Raw -Path $script:UpdatesCachePath | ConvertFrom-Json
|
||||||
|
# PowerShell 7 hands a stored date back as a date, Windows PowerShell 5.1 as the text it was written as.
|
||||||
|
$checked = $cached.checked_at
|
||||||
|
if ($checked -isnot [datetime]) { $checked = [datetime]::Parse([string]$checked, [Globalization.CultureInfo]::InvariantCulture, [Globalization.DateTimeStyles]::RoundtripKind) }
|
||||||
|
$limitHours = 6
|
||||||
|
if ($cached.failed) { $limitHours = 1 }
|
||||||
|
if (([datetime]::UtcNow - $checked.ToUniversalTime()).TotalHours -ge $limitHours) { $cached = $null }
|
||||||
|
}
|
||||||
|
} catch { $cached = $null }
|
||||||
|
|
||||||
|
if (-not $cached) {
|
||||||
|
$job = Start-Job -ScriptBlock {
|
||||||
|
$session = New-Object -ComObject Microsoft.Update.Session
|
||||||
|
$found = $session.CreateUpdateSearcher().Search("IsInstalled=0 and IsHidden=0 and Type='Software'")
|
||||||
|
$security = 0
|
||||||
|
foreach ($u in $found.Updates) {
|
||||||
|
foreach ($c in $u.Categories) {
|
||||||
|
if ($c.Name -eq 'Security Updates' -or $c.Name -eq 'Critical Updates') { $security++; break }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
[pscustomobject]@{ count = [int]$found.Updates.Count; security = [int]$security }
|
||||||
|
}
|
||||||
|
$answer = $null
|
||||||
|
try {
|
||||||
|
if (Wait-Job -Job $job -Timeout 150) { $answer = Receive-Job -Job $job -ErrorAction Stop } else { Stop-Job -Job $job }
|
||||||
|
} catch { $answer = $null } finally { Remove-Job -Job $job -Force -ErrorAction SilentlyContinue }
|
||||||
|
|
||||||
|
$count = 0
|
||||||
|
$security = 0
|
||||||
|
if ($null -ne $answer) { $count = [int]$answer.count; $security = [int]$answer.security }
|
||||||
|
$cached = [pscustomobject][ordered]@{ checked_at = [datetime]::UtcNow.ToString('o'); failed = ($null -eq $answer); count = $count; security = $security }
|
||||||
|
try {
|
||||||
|
$dir = Split-Path $script:UpdatesCachePath
|
||||||
|
if (-not (Test-Path $dir)) { $null = New-Item -ItemType Directory -Path $dir -Force }
|
||||||
|
ConvertTo-Json -InputObject $cached | Set-Content -Path $script:UpdatesCachePath -Encoding UTF8
|
||||||
|
} catch { }
|
||||||
|
}
|
||||||
|
|
||||||
|
# No answer is no answer: the report then leaves updates out, and the server keeps what it was last told.
|
||||||
|
if ($cached.failed) {
|
||||||
|
Add-Diag 'updates' 'failed' 'error' 'Windows Update did not answer; it is asked again in an hour'
|
||||||
|
return $null
|
||||||
|
}
|
||||||
|
return [ordered]@{ manager = 'windows-update'; count = [int]$cached.count; security = [int]$cached.security; reboot_required = (Test-RebootRequired) }
|
||||||
|
}
|
||||||
|
|
||||||
|
# ---- failed services, disk health, storage pools and what the machine is -------------------------
|
||||||
|
|
||||||
|
# A service has "failed" when it is set to start by itself, isn't running, and stopped with an error: an exit code that is neither 0 (it
|
||||||
|
# finished its job, as many do) nor 1077 (it has never been started since boot, which is what a service that starts on a trigger
|
||||||
|
# looks like). Services named in IgnoreServices (names or patterns like 'gupdate*') are left out. $Services are Win32_Service objects.
|
||||||
|
function ConvertTo-FailedUnits($Services, [string[]]$Ignore = @()) {
|
||||||
|
$out = @()
|
||||||
|
foreach ($svc in @($Services)) {
|
||||||
|
if ($null -eq $svc -or [string]$svc.StartMode -ne 'Auto') { continue }
|
||||||
|
if ([string]$svc.State -eq 'Running' -or [string]$svc.State -like '*Pending*') { continue }
|
||||||
|
$code = 0
|
||||||
|
if ($null -ne $svc.ExitCode) { $code = [int]$svc.ExitCode }
|
||||||
|
if ($code -eq 0 -or $code -eq 1077) { continue }
|
||||||
|
$name = [string]$svc.Name
|
||||||
|
$skip = $false
|
||||||
|
foreach ($pattern in @($Ignore)) { if ($pattern -and $name -like $pattern) { $skip = $true; break } }
|
||||||
|
if ($skip) { continue }
|
||||||
|
$out += [pscustomobject][ordered]@{ unit = $name; description = [string]$svc.DisplayName }
|
||||||
|
}
|
||||||
|
return @($out | Select-Object -First 100)
|
||||||
|
}
|
||||||
|
|
||||||
|
# Windows' own verdict on each physical disk (Healthy passes; Warning and Unhealthy don't), with temperature and how much of an SSD's
|
||||||
|
# life is used where the drive says. $Disks are Get-PhysicalDisk objects; $Counters maps a disk's DeviceId to its reliability counter.
|
||||||
|
function ConvertTo-DiskHealth($Disks, $Counters = @{}) {
|
||||||
|
$out = @()
|
||||||
|
foreach ($d in @($Disks)) {
|
||||||
|
if ($null -eq $d -or [string]$d.BusType -eq 'File Backed Virtual') { continue }
|
||||||
|
$health = [string]$d.HealthStatus
|
||||||
|
$healthy = $null
|
||||||
|
if ($health -eq 'Healthy') { $healthy = $true } elseif ($health -eq 'Warning' -or $health -eq 'Unhealthy') { $healthy = $false }
|
||||||
|
$counter = $null
|
||||||
|
if ($Counters -is [hashtable] -and $Counters.ContainsKey([string]$d.DeviceId)) { $counter = $Counters[[string]$d.DeviceId] }
|
||||||
|
$temp = $null
|
||||||
|
$wear = $null
|
||||||
|
if ($null -ne $counter) {
|
||||||
|
if ($null -ne $counter.Temperature -and [int]$counter.Temperature -gt 0 -and [int]$counter.Temperature -lt 150) { $temp = [int]$counter.Temperature }
|
||||||
|
if ($null -ne $counter.Wear -and [int]$counter.Wear -ge 0 -and [int]$counter.Wear -le 255) { $wear = [int]$counter.Wear }
|
||||||
|
}
|
||||||
|
$out += [pscustomobject][ordered]@{ device = ('PhysicalDisk' + [string]$d.DeviceId); model = ([string]$d.FriendlyName).Trim(); healthy = $healthy; temperature_c = $temp; wear_percent = $wear }
|
||||||
|
}
|
||||||
|
return @($out | Select-Object -First 100)
|
||||||
|
}
|
||||||
|
|
||||||
|
# Storage Spaces pools, in the words the server uses for ZFS pools: ONLINE, DEGRADED or FAULTED. A pool Windows can't say anything about is left out.
|
||||||
|
function ConvertTo-PoolHealth($Pools) {
|
||||||
|
$out = @()
|
||||||
|
foreach ($p in @($Pools)) {
|
||||||
|
if ($null -eq $p) { continue }
|
||||||
|
$health = switch ([string]$p.HealthStatus) { 'Healthy' { 'ONLINE' } 'Warning' { 'DEGRADED' } 'Unhealthy' { 'FAULTED' } default { $null } }
|
||||||
|
if (-not $health) { continue }
|
||||||
|
$capacity = $null
|
||||||
|
if ($p.Size -and [double]$p.Size -gt 0 -and $null -ne $p.AllocatedSize) { $capacity = [math]::Round(([double]$p.AllocatedSize / [double]$p.Size) * 100, 0) }
|
||||||
|
$out += [pscustomobject][ordered]@{ name = [string]$p.FriendlyName; health = $health; capacity_percent = $capacity }
|
||||||
|
}
|
||||||
|
return @($out | Select-Object -First 100)
|
||||||
|
}
|
||||||
|
|
||||||
|
# The three parts are asked for separately and each only if it can be: a part that can't be read is left out of the report (the server
|
||||||
|
# then keeps what it last said), never reported as "all well".
|
||||||
|
function Get-HostHealth($Config = $null) {
|
||||||
|
$health = [ordered]@{}
|
||||||
|
$ignore = @()
|
||||||
|
if ($Config -and $Config.IgnoreServices) { $ignore = @($Config.IgnoreServices) }
|
||||||
|
try { $health['failed_units'] = @(ConvertTo-FailedUnits @(Get-CimInstance -ClassName Win32_Service -ErrorAction Stop) $ignore) } catch { Write-Warning "Reading services failed. ($($_.Exception.Message))"; Add-Diag 'failed_units' 'failed' 'error' $_.Exception.Message }
|
||||||
|
try {
|
||||||
|
$disks = @(Get-PhysicalDisk -ErrorAction Stop)
|
||||||
|
$counters = @{}
|
||||||
|
foreach ($d in $disks) {
|
||||||
|
try { $counters[[string]$d.DeviceId] = Get-StorageReliabilityCounter -PhysicalDisk $d -ErrorAction Stop } catch { }
|
||||||
|
}
|
||||||
|
$health['smart'] = @(ConvertTo-DiskHealth $disks $counters)
|
||||||
|
} catch { Write-Warning "Reading disk health failed. ($($_.Exception.Message))"; Add-Diag 'smart' 'failed' 'error' $_.Exception.Message }
|
||||||
|
try {
|
||||||
|
$health['zfs'] = @(ConvertTo-PoolHealth @(Get-StoragePool -ErrorAction Stop | Where-Object { -not $_.IsPrimordial }))
|
||||||
|
} catch {
|
||||||
|
# A machine with no Storage Spaces pools gets an error, not an empty list: that is an answer ("no pools"), not a failure to read.
|
||||||
|
if ($_.FullyQualifiedErrorId -like '*NotFound*') { $health['zfs'] = @() } else { Write-Warning "Reading storage pools failed. ($($_.Exception.Message))"; Add-Diag 'zfs' 'failed' 'error' $_.Exception.Message }
|
||||||
|
}
|
||||||
|
if ($health.Count -eq 0) { return $null }
|
||||||
|
return $health
|
||||||
|
}
|
||||||
|
|
||||||
|
# Words a maker puts in the serial number or model when it has nothing to say there.
|
||||||
|
function Test-PlaceholderText([string]$Text) {
|
||||||
|
$t = $Text.Trim()
|
||||||
|
if ($t.Length -eq 0) { return $true }
|
||||||
|
return $t -match '^(to be filled by o\.?e\.?m\.?|default string|system serial number|system product name|system manufacturer|not specified|not applicable|none|n/a|unknown|0+|o\.?e\.?m\.?)$'
|
||||||
|
}
|
||||||
|
|
||||||
|
# What the machine is: maker, model and serial number, from the firmware. For inventory - a virtual machine says what it pretends to be.
|
||||||
|
# A home-built PC has nothing in the system's own fields, so the motherboard's are used when those say nothing.
|
||||||
|
function Get-HardwareInfo {
|
||||||
|
$cs = Get-CimInstance -ClassName Win32_ComputerSystem -ErrorAction Stop
|
||||||
|
$bios = $null
|
||||||
|
$board = $null
|
||||||
|
try { $bios = Get-CimInstance -ClassName Win32_BIOS -ErrorAction Stop } catch { }
|
||||||
|
try { $board = Get-CimInstance -ClassName Win32_BaseBoard -ErrorAction Stop } catch { }
|
||||||
|
$first = { param($values) foreach ($v in $values) { if (-not (Test-PlaceholderText ([string]$v))) { return ([string]$v).Trim() } } return $null }
|
||||||
|
$hw = [ordered]@{}
|
||||||
|
$maker = & $first @($cs.Manufacturer, $board.Manufacturer)
|
||||||
|
$model = & $first @($cs.Model, $board.Product)
|
||||||
|
$serial = & $first @($bios.SerialNumber, $board.SerialNumber)
|
||||||
|
if ($maker) { $hw['manufacturer'] = $maker }
|
||||||
|
if ($model) { $hw['model'] = $model }
|
||||||
|
if ($serial) { $hw['serial'] = $serial }
|
||||||
|
if ($hw.Count -eq 0) { return $null }
|
||||||
|
return $hw
|
||||||
|
}
|
||||||
|
|
||||||
|
function Get-SystemInfo($Config = $null) {
|
||||||
|
$script:Diag.Clear()
|
||||||
$ips = @(Get-NetIPAddress -AddressFamily IPv4 -ErrorAction SilentlyContinue |
|
$ips = @(Get-NetIPAddress -AddressFamily IPv4 -ErrorAction SilentlyContinue |
|
||||||
Where-Object { $_.IPAddress -notlike '127.*' -and $_.IPAddress -notlike '169.254.*' -and $_.AddressState -eq 'Preferred' } |
|
Where-Object { $_.IPAddress -notlike '127.*' -and $_.IPAddress -notlike '169.254.*' -and $_.AddressState -eq 'Preferred' } |
|
||||||
ForEach-Object { $_.IPAddress } | Select-Object -Unique)
|
ForEach-Object { $_.IPAddress } | Select-Object -Unique)
|
||||||
@@ -234,16 +432,47 @@ function Get-SystemInfo {
|
|||||||
$load = ($cpus | Where-Object { $null -ne $_.LoadPercentage } | Measure-Object -Property LoadPercentage -Average).Average
|
$load = ($cpus | Where-Object { $null -ne $_.LoadPercentage } | Measure-Object -Property LoadPercentage -Average).Average
|
||||||
$cores = ($cpus | Measure-Object -Property NumberOfLogicalProcessors -Sum).Sum
|
$cores = ($cpus | Measure-Object -Property NumberOfLogicalProcessors -Sum).Sum
|
||||||
|
|
||||||
|
# How long it has been up, in seconds (null if Windows does not say when it started).
|
||||||
|
$uptime = $null
|
||||||
|
try { if ($os.LastBootUpTime) { $uptime = [int64][math]::Floor(((Get-Date) - $os.LastBootUpTime).TotalSeconds); if ($uptime -lt 1) { $uptime = $null } } } catch { }
|
||||||
|
|
||||||
$disks = @()
|
$disks = @()
|
||||||
foreach ($d in @(Get-CimInstance -ClassName Win32_LogicalDisk -Filter 'DriveType=3')) {
|
foreach ($d in @(Get-CimInstance -ClassName Win32_LogicalDisk -Filter 'DriveType=3')) {
|
||||||
if (-not $d.Size) { continue } # an unformatted or unavailable volume
|
if (-not $d.Size) { continue } # an unformatted or unavailable volume
|
||||||
$disks += [pscustomobject][ordered]@{ mount = [string]$d.DeviceID; size_bytes = [int64]$d.Size; used_bytes = [int64]($d.Size - $d.FreeSpace) }
|
$disks += [pscustomobject][ordered]@{ mount = [string]$d.DeviceID; size_bytes = [int64]$d.Size; used_bytes = [int64]($d.Size - $d.FreeSpace) }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Best-effort: Windows Update being unreachable must not cost the rest of the report.
|
||||||
|
$updates = $null
|
||||||
|
try { $updates = Get-PendingUpdates } catch { Write-Warning "Checking for updates failed. ($($_.Exception.Message))"; Add-Diag 'updates' 'failed' 'error' $_.Exception.Message }
|
||||||
|
|
||||||
|
# The same for what is wrong with the machine, and what it is: neither may cost the rest of the report.
|
||||||
|
$health = $null
|
||||||
|
try { $health = Get-HostHealth $Config } catch { Write-Warning "Reading host health failed. ($($_.Exception.Message))"; Add-Diag 'host_health' 'failed' 'error' $_.Exception.Message }
|
||||||
|
$hardware = $null
|
||||||
|
try {
|
||||||
|
$hardware = Get-HardwareInfo
|
||||||
|
if (-not $hardware) { Add-Diag 'hardware' 'unavailable' 'not_found' }
|
||||||
|
} catch { Write-Warning "Reading the hardware identity failed. ($($_.Exception.Message))"; Add-Diag 'hardware' 'failed' 'error' $_.Exception.Message }
|
||||||
|
|
||||||
|
# What it runs, so the app can say when that release stops being supported: the product's name and its build number (a build is one release of Windows).
|
||||||
|
# Best-effort and silent: nothing here is worth a note of its own.
|
||||||
|
$osInfo = $null
|
||||||
|
try {
|
||||||
|
$caption = ([string]$os.Caption) -replace '^Microsoft\s+', ''
|
||||||
|
if ($caption.Trim() -and $os.BuildNumber) { $osInfo = [ordered]@{ id = 'windows'; name = $caption.Trim(); version = [string]$os.BuildNumber } }
|
||||||
|
} catch { }
|
||||||
|
|
||||||
return [ordered]@{
|
return [ordered]@{
|
||||||
|
updates = $updates
|
||||||
|
health = $health
|
||||||
|
hardware = $hardware
|
||||||
|
os = $osInfo
|
||||||
|
diagnostics = @($script:Diag)
|
||||||
ip_addresses = @($ips)
|
ip_addresses = @($ips)
|
||||||
cpu = [ordered]@{ model = [string]($cpus[0].Name).Trim(); cores = [int]$cores; load_percent = $(if ($null -ne $load) { [math]::Round([double]$load, 1) } else { $null }) }
|
cpu = [ordered]@{ model = [string]($cpus[0].Name).Trim(); cores = [int]$cores; load_percent = $(if ($null -ne $load) { [math]::Round([double]$load, 1) } else { $null }) }
|
||||||
memory = [ordered]@{ total_bytes = $totalBytes; used_bytes = ($totalBytes - $freeBytes) }
|
memory = [ordered]@{ total_bytes = $totalBytes; used_bytes = ($totalBytes - $freeBytes) }
|
||||||
|
uptime_seconds = $uptime
|
||||||
disks = @($disks)
|
disks = @($disks)
|
||||||
listening_ports = @(Get-ListeningPorts)
|
listening_ports = @(Get-ListeningPorts)
|
||||||
}
|
}
|
||||||
@@ -272,6 +501,70 @@ function Send-Report($Config, [string]$Json) {
|
|||||||
return Invoke-RestMethod @params
|
return Invoke-RestMethod @params
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# ---- updating itself -------------------------------------------------------------
|
||||||
|
|
||||||
|
# A GET to the server with the token, set up the same way as the report (modern TLS on 5.1, and the self-signed switch).
|
||||||
|
function New-GetParams($Config, [string]$Path) {
|
||||||
|
$params = @{
|
||||||
|
Uri = "$($Config.ApiUrl)$Path"
|
||||||
|
Method = 'Get'
|
||||||
|
Headers = @{ Authorization = "Bearer $($Config.ApiToken)" }
|
||||||
|
TimeoutSec = 60
|
||||||
|
}
|
||||||
|
if ($PSVersionTable.PSVersion.Major -ge 6) {
|
||||||
|
if ($Config.Insecure) { $params['SkipCertificateCheck'] = $true }
|
||||||
|
} else {
|
||||||
|
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
|
||||||
|
if ($Config.Insecure) { [Net.ServicePointManager]::ServerCertificateValidationCallback = { $true } }
|
||||||
|
$params['UseBasicParsing'] = $true
|
||||||
|
}
|
||||||
|
return $params
|
||||||
|
}
|
||||||
|
|
||||||
|
# Whether a downloaded copy may replace the running script: it is newer, it is exactly the file the server vouched for (SHA-256), it says the
|
||||||
|
# version that was promised, and it parses. Anything else is thrown away.
|
||||||
|
function Test-AgentDownload([string]$Path, [string]$ExpectedSha256, [int]$NewVersion, [int]$CurrentVersion) {
|
||||||
|
if ($NewVersion -le $CurrentVersion) { return $false }
|
||||||
|
if ($ExpectedSha256 -notmatch '^[0-9a-f]{64}$') { return $false }
|
||||||
|
$got = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||||
|
if ($got -ne $ExpectedSha256) { return $false }
|
||||||
|
$text = [System.IO.File]::ReadAllText($Path)
|
||||||
|
if ($text -notmatch ('(?m)^\$script:AgentVersion\s*=\s*' + $NewVersion + '\s*$')) { return $false }
|
||||||
|
$tokens = $null
|
||||||
|
$errors = $null
|
||||||
|
[void][System.Management.Automation.Language.Parser]::ParseInput($text, [ref]$tokens, [ref]$errors)
|
||||||
|
return ($errors.Count -eq 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
# Replaces this script with the one this Homelab Manager serves, when an administrator has switched automatic updates on and this agent is
|
||||||
|
# behind. The new one runs from the next report. Nothing goes wrong if it can't: the report carries on as it was. Only a script that is
|
||||||
|
# installed under its own name is ever replaced, so running a copy from somewhere else (a test, a checkout) never changes it.
|
||||||
|
function Update-AgentSelf($Config, [string]$Self = $PSCommandPath) {
|
||||||
|
$self = $Self
|
||||||
|
if (-not $self -or (Split-Path -Leaf $self) -ne 'homelab-manager-agent.ps1') { return }
|
||||||
|
$download = "$self.download"
|
||||||
|
try {
|
||||||
|
$ask = New-GetParams $Config "/api/agent/report/update?version=$($script:AgentVersion)"
|
||||||
|
$reply = Invoke-RestMethod @ask
|
||||||
|
if (-not $reply.update) { return }
|
||||||
|
$version = [int]$reply.version
|
||||||
|
$fetch = New-GetParams $Config '/agent/windows/report-tasks.ps1'
|
||||||
|
$fetch.Remove('Headers')
|
||||||
|
$fetch['OutFile'] = $download
|
||||||
|
$null = Invoke-WebRequest @fetch
|
||||||
|
if (-not (Test-AgentDownload $download ([string]$reply.sha256) $version $script:AgentVersion)) {
|
||||||
|
[Console]::Error.WriteLine("Note: the newer agent (version $version) didn't check out against what the server said, so it was not installed.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
Move-Item -LiteralPath $download -Destination $self -Force
|
||||||
|
[Console]::Error.WriteLine("Updated the agent from version $($script:AgentVersion) to version $version; the new one runs at the next report.")
|
||||||
|
} catch {
|
||||||
|
[Console]::Error.WriteLine("Note: couldn't check for, or install, a newer agent: $($_.Exception.Message)")
|
||||||
|
} finally {
|
||||||
|
if (Test-Path -LiteralPath $download) { Remove-Item -LiteralPath $download -Force -ErrorAction SilentlyContinue }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
# ---- main -----------------------------------------------------------------------
|
# ---- main -----------------------------------------------------------------------
|
||||||
|
|
||||||
# A plain one-line message on stderr: Write-Error would bury it in a stack trace on every manual run.
|
# A plain one-line message on stderr: Write-Error would bury it in a stack trace on every manual run.
|
||||||
@@ -286,6 +579,9 @@ function Invoke-Agent {
|
|||||||
Stop-Agent "API_URL and API_TOKEN must be set (environment variables, or $script:DefaultConfigPath)."
|
Stop-Agent "API_URL and API_TOKEN must be set (environment variables, or $script:DefaultConfigPath)."
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# A dry run reports nothing and changes nothing, so it never updates the agent either.
|
||||||
|
if (-not $DryRun) { Update-AgentSelf $config }
|
||||||
|
|
||||||
$tasks = @()
|
$tasks = @()
|
||||||
try {
|
try {
|
||||||
$tasks = @(Get-ReportedTasks $config.IncludeMicrosoftTasks)
|
$tasks = @(Get-ReportedTasks $config.IncludeMicrosoftTasks)
|
||||||
@@ -297,7 +593,7 @@ function Invoke-Agent {
|
|||||||
# Hardware and network facts are best-effort, like the Linux agent: a quirk on one host must not cost the whole report.
|
# Hardware and network facts are best-effort, like the Linux agent: a quirk on one host must not cost the whole report.
|
||||||
$system = $null
|
$system = $null
|
||||||
try {
|
try {
|
||||||
$system = Get-SystemInfo
|
$system = Get-SystemInfo $config
|
||||||
} catch {
|
} catch {
|
||||||
Write-Warning "Collecting hardware/network info failed - reporting tasks without it. ($($_.Exception.Message))"
|
Write-Warning "Collecting hardware/network info failed - reporting tasks without it. ($($_.Exception.Message))"
|
||||||
}
|
}
|
||||||
@@ -306,6 +602,7 @@ function Invoke-Agent {
|
|||||||
hostname = Get-Fqdn
|
hostname = Get-Fqdn
|
||||||
os_type = 'windows'
|
os_type = 'windows'
|
||||||
reported_at = (Get-Date).ToUniversalTime().ToString("yyyy-MM-dd'T'HH:mm:ss'Z'")
|
reported_at = (Get-Date).ToUniversalTime().ToString("yyyy-MM-dd'T'HH:mm:ss'Z'")
|
||||||
|
agent_version = $script:AgentVersion
|
||||||
system = $system
|
system = $system
|
||||||
tasks = @($tasks)
|
tasks = @($tasks)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,49 @@
|
|||||||
|
# Accessibility
|
||||||
|
|
||||||
|
The app is checked against **WCAG 2.2 level AA** with [axe-core](https://github.com/dequelabs/axe-core), the standard automated checker, in both themes. This page says what was found, what was done and how to run the same check again.
|
||||||
|
|
||||||
|
## Where it stands
|
||||||
|
|
||||||
|
A sweep of 37 pages (the dashboard, alerts and their history, servers and a server's page, every menu page that has data, the settings pages and the new pages) in the **light and the dark theme**, with the rules for WCAG 2 A and AA, 2.1 A and AA, 2.2 AA and axe's best practices, ends with **no violations**. Keyboard focus shows a 2 px ring on every control (one rule for them all), the skip link and the main landmark are there, and no page uses a positive tab order.
|
||||||
|
|
||||||
|
What an automated check cannot judge is not covered by it: whether a label says the right thing, whether the reading order makes sense to a person, or how a page works with a particular screen reader. Those were looked at by hand on the pages that changed, not on every page.
|
||||||
|
|
||||||
|
## What the sweep found, and what was done
|
||||||
|
|
||||||
|
| Found | Where | Done |
|
||||||
|
|---|---|---|
|
||||||
|
| Colour contrast too low (1.9–4.3:1) on **green, yellow, red and blue text and badges** on the light theme | the status badges and figures on most pages | The text takes a darker shade of the same colour on the light theme only (green `#17692a`, yellow `#8a5600`, orange `#a14d00`, red `#b32d2d`, blue `#0558a8`, grey on a grey badge `#5b6372`): at least 4.5:1 on white and on the badge's tint. The dark theme is unchanged |
|
||||||
|
| Contrast 2.7–3.7:1 for **tabs and pills that are not chosen**, the **chosen pill**, the **switch button that is on**, **outlined primary buttons** and the **info alert** on the dark theme | the view tabs, Cards/Table, the names lists, Settings | Lighter shades on the dark theme (`#9ca3af`, `#5aa9ff`, `#7cc0f7`), at least 4.5:1 on their backgrounds; the count in a pill is no longer faded |
|
||||||
|
| **Links told apart by colour alone** in a sentence (1.4.1) | notes and intros all over | A link inside a sentence is underlined |
|
||||||
|
| **Link targets** under 24 px tall (2.5.8) | a link that is a whole line (the dashboard's list, a table's *Timeline*) | Such a link has padding to at least 24 px |
|
||||||
|
| **Selects with no name** | the severity and kind filters on Alerts, the status filter on Secrets, each user's role on Users | Each has an `aria-label` (a role's names the user) |
|
||||||
|
| **Label in name** (2.5.3) | the nodes on the Map | A node is named by what is written on it first, then what it is and how it is |
|
||||||
|
|
||||||
|
All of this is in the *Accessibility* part at the end of `web/src/styles.css` and in those pages; `server/test/accessibility.test.ts` checks that the shades are the ones used, that each reads (the contrast is computed), and that the labels are there.
|
||||||
|
|
||||||
|
## Running the check again
|
||||||
|
|
||||||
|
The sweep is run in the browser: it needs nothing installed in the project.
|
||||||
|
|
||||||
|
1. Start the app (or a scratch copy with some data) and sign in.
|
||||||
|
2. Get `axe.min.js` (`npm install axe-core` in any folder, then `node_modules/axe-core/axe.min.js`) and serve it from the app's built folder (`web/dist/`, which is not committed), or paste it into the browser's console.
|
||||||
|
3. In the console, on a page of the app:
|
||||||
|
|
||||||
|
```js
|
||||||
|
await new Promise((ok) => { const s = document.createElement("script"); s.src = "/axe.min.js"; s.onload = ok; document.head.appendChild(s); });
|
||||||
|
async function scan(routes, theme) {
|
||||||
|
document.documentElement.setAttribute("data-bs-theme", theme);
|
||||||
|
const out = [];
|
||||||
|
for (const url of routes) {
|
||||||
|
history.pushState({}, "", url); window.dispatchEvent(new PopStateEvent("popstate"));
|
||||||
|
await new Promise((ok) => setTimeout(ok, 1500));
|
||||||
|
const r = await axe.run(document, { runOnly: { type: "tag", values: ["wcag2a", "wcag2aa", "wcag21a", "wcag21aa", "wcag22aa", "best-practice"] }, resultTypes: ["violations"] });
|
||||||
|
out.push([url, r.violations.map((v) => `${v.id} (${v.nodes.length}) ${v.nodes[0].target.join(" ")}`)]);
|
||||||
|
}
|
||||||
|
return out.filter(([, v]) => v.length);
|
||||||
|
}
|
||||||
|
await scan(["/", "/alerts", "/servers", "/integrations"], "dark"); // empty when there is nothing to fix
|
||||||
|
await scan(["/", "/alerts", "/servers", "/integrations"], "light");
|
||||||
|
```
|
||||||
|
|
||||||
|
Do a handful of pages at a time (each takes a second or two), and both themes: the light and the dark theme fail in different places. Check a page again after changing a colour, since a new shade can fix one background and break another.
|
||||||
+408
@@ -0,0 +1,408 @@
|
|||||||
|
# API
|
||||||
|
|
||||||
|
Homelab Manager has two kinds of programmatic interface:
|
||||||
|
|
||||||
|
| Interface | For | Who can use it | Section |
|
||||||
|
|---|---|---|---|
|
||||||
|
| **The read API** — `/api/v1` | Scripts, Home Assistant, dashboards that want JSON about servers, alerts, incidents, inventory, network equipment, endpoint checks and maintenance | Anyone signed in can make a personal, read-only token | [The read API](#the-read-api) |
|
||||||
|
| **Machine addresses** — agent reports, heartbeats, `/metrics`, share links | The agents, cron jobs, Prometheus, status pages and calendars | Each has its own token | [Other addresses for machines](#other-addresses-for-machines) |
|
||||||
|
|
||||||
|
**There is no API that changes anything.** The read API refuses every verb but `GET` and `HEAD`. Everything else the app does (acknowledge an alert, start maintenance, add a check, change a setting)
|
||||||
|
is done in the web interface, which is not a supported API: its addresses and shapes change with the pages. The read API's shapes are a deliberate, stable contract.
|
||||||
|
|
||||||
|
Examples below use `https://hm.example.lan` for your `APP_BASE_URL`. Related pages: [PROMETHEUS-GRAFANA.md](PROMETHEUS-GRAFANA.md) for `/metrics`, [WEBHOOKS.md](WEBHOOKS.md) for being *told* when something
|
||||||
|
happens instead of asking, [HEARTBEATS-AND-CALENDAR.md](HEARTBEATS-AND-CALENDAR.md), [MONITORING.md](MONITORING.md) for what the alert fields mean.
|
||||||
|
|
||||||
|
## The read API
|
||||||
|
|
||||||
|
### Getting a token
|
||||||
|
|
||||||
|
1. Sign in, open the **user menu → API tokens**. This is open to everyone signed in, whatever their role.
|
||||||
|
2. Give the token a name (e.g. `Home Assistant`) and choose how long it lasts: **30, 90 or 365 days, or until removed**.
|
||||||
|
3. Copy the token (it starts with `hlm_p_`). **It is shown once**; only its SHA-256 is kept.
|
||||||
|
|
||||||
|
A token belongs to the person who made it. They see and remove their own, nobody else's. A person can have **ten**. A token **stops working at once** when it is removed, when it expires and when
|
||||||
|
its owner is removed. It sees what its owner sees on the pages: no secrets, no passwords, no server tokens, no settings.
|
||||||
|
|
||||||
|
### Making a request
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -H "Authorization: Bearer hlm_p_xxxxxxxxxxxxxxxxxxxxxxxx" https://hm.example.lan/api/v1/alerts
|
||||||
|
```
|
||||||
|
|
||||||
|
- The token goes in the **`Authorization: Bearer …` header only**. A token in the address (`?token=…`) is **not accepted**, because addresses end up in logs.
|
||||||
|
- **No token, a wrong one, an expired one and a removed one all get the same answer**: `401` with a JSON body and `WWW-Authenticate: Bearer realm="homelab-manager"`.
|
||||||
|
- **Only `GET` and `HEAD`.** Any other verb gets `405` (`{"error":"read_only"}`) with `Allow: GET, HEAD` — with a token or without.
|
||||||
|
- **Limits:** 300 requests a minute from one address (`429` beyond that). Answers carry `Cache-Control: no-store`.
|
||||||
|
- **Be gentle:** the data changes at the pace of the agents (every 15 minutes or so) and the checks (minutes). Polling more often than once a minute gains nothing.
|
||||||
|
|
||||||
|
### Conventions
|
||||||
|
|
||||||
|
- Times are **ISO 8601 in UTC** (`2026-10-20T21:44:30.244Z`). A few older fields written by the database itself (`createdAt` on a server) are `YYYY-MM-DD HH:MM:SS` in UTC.
|
||||||
|
- Sizes are **bytes**. Percentages are **0–100** (not 0–1).
|
||||||
|
- `url` is a **full address** into the app, built from `APP_BASE_URL`, so a script does not need to know where the app lives.
|
||||||
|
- Missing information is `null`, never an invented value. A server whose agent has never said which OS it runs has `operatingSystem: null`.
|
||||||
|
- Errors are JSON: `{ "error": "<code>", "message": "…" }` with `error` one of `unauthorized`, `read_only`, `not_found`, `invalid_id`.
|
||||||
|
- Unknown addresses under `/api/v1` get `404` and point at `GET /api/v1`.
|
||||||
|
|
||||||
|
### Addresses
|
||||||
|
|
||||||
|
| Address | What it answers |
|
||||||
|
|---|---|
|
||||||
|
| `GET /api/v1` | The list of these addresses. |
|
||||||
|
| `GET /api/v1/me` | Whose token this is. |
|
||||||
|
| `GET /api/v1/servers` | Every server. Filter with `?tag=` and `?status=online\|offline\|unknown`. |
|
||||||
|
| `GET /api/v1/servers/:id` | One server, the same shape. |
|
||||||
|
| `GET /api/v1/servers/:id/metrics?days=` | Its load, memory and disks over time. `days` 1–365 (7 by default). |
|
||||||
|
| `GET /api/v1/alerts` | What is wrong now. Filter with `?severity=`, `?category=`, `?acknowledged=true\|false`, `?silenced=true\|false`. |
|
||||||
|
| `GET /api/v1/incidents` | The record of alerts that began and ended. `?status=open\|resolved`, `?limit=` (1–500, default 100). |
|
||||||
|
| `GET /api/v1/incidents/:id` | One incident with its timeline. |
|
||||||
|
| `GET /api/v1/incidents/:id/before?minutes=` | What changed before it began, ranked by how likely each is to be connected. `minutes` 15–1440 (120 by default). |
|
||||||
|
| `GET /api/v1/inventory` | The hardware inventory. |
|
||||||
|
| `GET /api/v1/network` | Network equipment read from UniFi: devices, networks, Wi-Fi names, and which server is plugged into which device. |
|
||||||
|
| `GET /api/v1/endpoints` | The endpoint checks and how they are doing. |
|
||||||
|
| `GET /api/v1/endpoints/:id/history?days=` | One check's availability and answer times over time. |
|
||||||
|
| `GET /api/v1/maintenance` | Maintenance windows running now and coming up. |
|
||||||
|
|
||||||
|
### `GET /api/v1/me`
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"user": { "id": 1, "name": "Demo Admin", "email": "demo@example.lan", "role": "admin" },
|
||||||
|
"token": { "id": 1, "label": "Home Assistant", "expiresAt": "2027-01-18T21:45:43.390Z" }
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`expiresAt` is `null` for a token with no end. A quick way to check a token works.
|
||||||
|
|
||||||
|
### `GET /api/v1/servers`
|
||||||
|
|
||||||
|
```json
|
||||||
|
{ "servers": [
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"name": "wiki-host",
|
||||||
|
"hostname": "wiki-host.lan",
|
||||||
|
"description": null,
|
||||||
|
"os": "linux",
|
||||||
|
"tags": ["prod"],
|
||||||
|
"ipAddresses": ["192.168.1.20"],
|
||||||
|
"status": "online",
|
||||||
|
"lastSeenAt": "2026-10-20T21:44:30.244Z",
|
||||||
|
"createdAt": "2026-01-01 00:00:00",
|
||||||
|
"agentVersion": 7,
|
||||||
|
"runsOnServerId": null,
|
||||||
|
"hardware": { "manufacturer": null, "model": null, "serial": null },
|
||||||
|
"operatingSystem": {
|
||||||
|
"id": "ubuntu", "name": "Ubuntu 24.04 LTS", "version": "24.04", "product": "Ubuntu 24.04 LTS",
|
||||||
|
"endsOn": "2029-05-31", "state": "ok", "daysLeft": 951
|
||||||
|
},
|
||||||
|
"cpu": { "model": "Intel(R) N100", "cores": 4, "loadPercent": 12.4 },
|
||||||
|
"memory": { "totalBytes": 16623411200, "usedBytes": 4210356224 },
|
||||||
|
"disks": [ { "mount": "/", "sizeBytes": 245107195904, "usedBytes": 61276798976, "usedPercent": 25 } ],
|
||||||
|
"updates": { "manager": "apt", "count": 5, "security": 2, "rebootRequired": false, "reportedAt": "2026-10-20T21:44:30.000Z" },
|
||||||
|
"health": { "failedUnits": 0, "reportedAt": "2026-10-20T21:44:30.000Z" },
|
||||||
|
"thresholds": null,
|
||||||
|
"url": "https://hm.example.lan/servers/1"
|
||||||
|
}
|
||||||
|
] }
|
||||||
|
```
|
||||||
|
|
||||||
|
| Field | Meaning |
|
||||||
|
|---|---|
|
||||||
|
| `status` | `online` while the agent has reported within the offline limit (60 minutes unless the server has its own), `offline` after, `unknown` if it never has. |
|
||||||
|
| `lastSeenAt` | When the agent last reported. |
|
||||||
|
| `agentVersion` | The version of the agent script, as it says; `null` until it has said. |
|
||||||
|
| `runsOnServerId` | The server this one runs on (a VM's Proxmox host, say), when one is set. |
|
||||||
|
| `operatingSystem` | What the agent says it runs, and when that release stops being supported. `state` is `ok`, `soon` (inside the warning window), `ended` or `unknown` (a release the app has no date for); `daysLeft` is negative once it has ended. `null` until the agent has said. |
|
||||||
|
| `cpu.loadPercent` | Load average as a share of the cores — an approximation, not instantaneous usage. |
|
||||||
|
| `disks[].usedPercent` | One decimal. |
|
||||||
|
| `updates` | What is waiting to be installed; `null` when the agent does not report it. `security` is `null` when the package manager cannot say. |
|
||||||
|
| `health.failedUnits` | The number of failed services (Linux), when the agent reports them. |
|
||||||
|
| `thresholds` | The server's own alert limits, if it has any that replace the global ones; otherwise `null`. |
|
||||||
|
|
||||||
|
### `GET /api/v1/servers/:id/metrics?days=`
|
||||||
|
|
||||||
|
```json
|
||||||
|
{ "days": 7, "series": [
|
||||||
|
{ "metric": "cpu", "kind": "cpu", "label": "CPU load", "latest": 12.4, "trend": null,
|
||||||
|
"points": [ { "t": "2026-10-20T20:59:30.572Z", "v": 12 }, { "t": "2026-10-20T21:14:30.572Z", "v": 13.5 } ] },
|
||||||
|
{ "metric": "disk:/", "kind": "disk", "label": "Disk /", "latest": 25,
|
||||||
|
"trend": { "perDay": 0.4, "fit": 0.97, "spanDays": 13.9, "daysToFull": 188 },
|
||||||
|
"points": [ … ] }
|
||||||
|
] }
|
||||||
|
```
|
||||||
|
|
||||||
|
- `kind` is `cpu`, `mem` or `disk`; a disk's `metric` is `disk:<mount>`. Values are percentages.
|
||||||
|
- The range decides how fine the points are: up to **35 days** they are the samples themselves (one every 10–15 minutes, thinned to at most about 100 points); beyond that they are **hourly (to 120 days) or daily
|
||||||
|
averages** and carry `lo` and `hi` (the lowest and highest in that hour or day).
|
||||||
|
- A disk's `trend` is a straight-line fit: `perDay` is points per day, `fit` is how straight (0–1), `daysToFull` is set only when the disk is rising steadily. `null` for CPU and memory, and for a disk
|
||||||
|
with too little history.
|
||||||
|
|
||||||
|
### `GET /api/v1/alerts`
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"generatedAt": "2026-10-20T21:45:53.522Z",
|
||||||
|
"needingAttention": { "critical": 1, "warning": 3, "acknowledged": 0 },
|
||||||
|
"counts": { "critical": 1, "warning": 3, "info": 0, "silenced": 0, "grouped": 0 },
|
||||||
|
"couldntCheck": [
|
||||||
|
{ "name": "Proxmox “Lab PVE”", "error": "connect ECONNREFUSED 127.0.0.1:3071", "affects": ["Proxmox backups", "Server and storage health"] }
|
||||||
|
],
|
||||||
|
"alerts": [
|
||||||
|
{
|
||||||
|
"id": "monitoring:rule:1:server:1",
|
||||||
|
"severity": "critical",
|
||||||
|
"category": "monitoring",
|
||||||
|
"source": "Server",
|
||||||
|
"message": "wiki-host: cpu load is 95%, above 90% for 30 min [CPU above 90% for 30 minutes]",
|
||||||
|
"url": "https://hm.example.lan/servers/1?view=history",
|
||||||
|
"silenced": false,
|
||||||
|
"snoozedUntil": null,
|
||||||
|
"closed": null,
|
||||||
|
"flapping": false,
|
||||||
|
"target": { "type": "server", "id": 1, "name": "wiki-host" },
|
||||||
|
"causedBy": null,
|
||||||
|
"affects": [],
|
||||||
|
"acknowledgement": null
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
| Field | Meaning |
|
||||||
|
|---|---|
|
||||||
|
| `needingAttention` | What wants a look: critical and warning alerts that are not silenced and that nobody has acknowledged, and how many somebody has. This is the number on the menu badge. |
|
||||||
|
| `counts` | Everything active by severity, whether acknowledged or not. `silenced` are held back by maintenance (or snoozed); `grouped` are folded under the host that explains them. Neither is in `critical`/`warning`/`info`. |
|
||||||
|
| `couldntCheck` | Sources the app could not read this time, and which checks that leaves blind. **An empty `alerts` list with something here is not "all clear".** |
|
||||||
|
| `id` | Stable for the same problem: `<category>:<key>`. |
|
||||||
|
| `severity` | `critical`, `warning` or `info`. |
|
||||||
|
| `category` | `offline`, `disk`, `services`, `backup`, `updates`, `expiry`, `automation`, `integration`, `monitoring` or `tickets`. |
|
||||||
|
| `source` | Where it comes from: `Server`, `Proxmox`, `Synology`, `UniFi`, `Endpoint`, `Heartbeat` … |
|
||||||
|
| `silenced` | Under a maintenance window, snoozed, or folded under a host that is down. Notifications for it are held back. |
|
||||||
|
| `snoozedUntil` | When a snooze on it ends; `null` otherwise. |
|
||||||
|
| `closed` | `{ by, at, note }` when somebody has closed it (it is also `silenced`; a closed alert has no end — it opens again if it gets worse or has been gone for half an hour); `null` otherwise. |
|
||||||
|
| `flapping` | It keeps being raised and cleared; its messages are being held back. |
|
||||||
|
| `target` | What a maintenance window would silence it with (`server`, `integration`, `dns_provider` or `device`), or `null`. |
|
||||||
|
| `causedBy` | The alert it is folded under, as `{ name, alertId? }`, or `null`. |
|
||||||
|
| `affects` | On a host's alert: the servers on it whose own alerts are folded under this one. |
|
||||||
|
| `acknowledgement` | `{ by, at, note, until, assignee }` while someone has said they are on it; otherwise `null`. |
|
||||||
|
|
||||||
|
### `GET /api/v1/incidents` and `/:id`
|
||||||
|
|
||||||
|
An *incident* is one time an alert was raised and later went away; the same alert coming back is a new incident.
|
||||||
|
|
||||||
|
```json
|
||||||
|
{ "incidents": [
|
||||||
|
{ "id": 4, "alertKey": "updates:os-support:1", "severity": "warning", "peakSeverity": "warning", "category": "updates",
|
||||||
|
"source": "Server", "message": "wiki-host runs Ubuntu 18.04 LTS, whose support ended on 2023-05-31 …",
|
||||||
|
"link": "/servers/1", "target": { "type": "server", "id": 1, "name": "wiki-host" }, "causedBy": null,
|
||||||
|
"openedAt": "2026-10-20T21:45:53.522Z", "resolvedAt": null, "durationMs": 127, "open": true, "notes": 0 }
|
||||||
|
] }
|
||||||
|
```
|
||||||
|
|
||||||
|
`GET /api/v1/incidents/:id` adds the **timeline** (when it was raised, acknowledged, handed over, what was done to the thing it is about in the hour around it, when it resolved) and the notes
|
||||||
|
people wrote. Open incidents are always listed; resolved ones newest first, up to `limit`.
|
||||||
|
|
||||||
|
### `GET /api/v1/incidents/:id/before`
|
||||||
|
|
||||||
|
The "what changed before this?" list for one incident: what the app knows happened in the window before it began — settings and other changes in the audit log, maintenance windows that began or ended, update runs
|
||||||
|
and patch plans, other alerts raised first, and a step in the server's own figures — each scored and the best first.
|
||||||
|
|
||||||
|
```json
|
||||||
|
{ "incidentId": 1, "openedAt": "2026-10-21T20:08:08.000Z", "windowMinutes": 120,
|
||||||
|
"from": "2026-10-21T18:08:08.000Z", "to": "2026-10-21T20:09:08.000Z",
|
||||||
|
"suspects": [
|
||||||
|
{ "at": "2026-10-21T20:02:08.000Z", "minutesBefore": 6, "kind": "maintenance", "title": "Maintenance on pve1 ended", "detail": "Kernel update", "actor": "Olga",
|
||||||
|
"relation": "needs", "about": "pve1 — which it needs", "why": "About something it needs directly, 6 min before",
|
||||||
|
"score": 86, "confidence": "likely", "link": "/maintenance" }
|
||||||
|
],
|
||||||
|
"looked": { "auditEntries": 5, "maintenanceWindows": 1, "updateRuns": 1, "otherAlerts": 2, "series": 1 } }
|
||||||
|
```
|
||||||
|
|
||||||
|
| Field | Meaning |
|
||||||
|
|---|---|
|
||||||
|
| `kind` | `maintenance`, `update`, `alert` (another alert, or a host going down), `change`, `settings`, `metric` (a step in the server's own load, memory or disk) or `rule` (one of your alert rules was changed). |
|
||||||
|
| `relation` | How it is related to what the incident is about: `same`, `needs` (the host it runs on, the switch it is plugged into, something it depends on), `needed_by`, `everything` (a setting, DNS) or `other`. |
|
||||||
|
| `score`, `confidence` | 0–100, from how related it is, what kind of thing it is and how close in time. `likely` is 55 and over, `possible` 25 and over, otherwise `background`. Anything under 8 is left out. |
|
||||||
|
| `minutesBefore` | How long before the incident began. |
|
||||||
|
| `link` | A page in the app, as a path (not a full address). |
|
||||||
|
| `looked` | How much was looked through, so an empty `suspects` can say what it covered. An empty list means *nothing the app knows about*, not *nothing happened* — a change made on the machine itself is invisible to it. |
|
||||||
|
|
||||||
|
At most ten suspects are returned. It is a list of suspects, not a diagnosis.
|
||||||
|
|
||||||
|
### `GET /api/v1/inventory`
|
||||||
|
|
||||||
|
```json
|
||||||
|
{ "assets": [
|
||||||
|
{ "id": 3, "name": "NAS", "kind": "server", "manufacturer": "Synology", "model": "DS923+", "serial": "…", "location": "Rack A", "rackUnit": "U6-9",
|
||||||
|
"purchaseDate": "2024-03-02", "purchasePrice": 6500, "warrantyUntil": "2027-03-02", "endOfSupport": null, "replacementCost": 7200,
|
||||||
|
"powerWatts": 120, "notes": null, "server": { "id": 6, "name": "nas" }, "updatedAt": "2026-10-20T21:45:30.244Z" }
|
||||||
|
] }
|
||||||
|
```
|
||||||
|
|
||||||
|
`server` links a piece of hardware to the server it is, when it is one. `kind` is `server`, `storage`, `network`, `power`, `client` or `other`.
|
||||||
|
|
||||||
|
### `GET /api/v1/network`
|
||||||
|
|
||||||
|
Present once a UniFi integration has been read: `{ devices, networks, ssids, serverLinks }` — each device with its `state` (`online`, `offline`, `updating`, `pending`, `other`), `ip`, `mac`, `uptimeSeconds`,
|
||||||
|
firmware, what it is connected through and what depends on it; the networks (VLAN, address range); the Wi-Fi names (security, which network); and `serverLinks` saying which server is plugged into which
|
||||||
|
device (and port). Wi-Fi passphrases are never read, so they are never here.
|
||||||
|
|
||||||
|
### `GET /api/v1/endpoints` and `/:id/history`
|
||||||
|
|
||||||
|
```json
|
||||||
|
{ "endpoints": [
|
||||||
|
{ "id": 1, "name": "Wiki", "kind": "http", "target": "https://192.168.1.20/", "port": null, "enabled": true,
|
||||||
|
"status": "up", "lastCheckedAt": "2026-10-20T21:50:00.000Z", "lastResponseMs": 41, "lastMessage": null,
|
||||||
|
"value": null, "certificateExpiresAt": "2027-01-04T12:00:00.000Z", "uptime24hPercent": 100, "uptime7dPercent": 99.9,
|
||||||
|
"averageResponseMs24h": 44, "serverId": 1 }
|
||||||
|
] }
|
||||||
|
```
|
||||||
|
|
||||||
|
`status` is `up`, `down`, `unknown` (never asked yet) or `paused` (switched off). `lastMessage` is set only while it is failing (why). `kind` is `http`, `tcp` or `tls`. `value` is null unless the check judges a value in the answer, and then `{ "path": "power.now", "operator": "<", "expected": "5000", "unit": "W", "last": "120" }` (`last` is what it read the last time, as text).
|
||||||
|
|
||||||
|
`/:id/history?days=` answers `{ check: {id, name}, … points … }`: the availability and answer times over 1–365 days, drawn from each answer (up to 2 days), the hourly figures (up to 120 days) or the daily
|
||||||
|
ones, whichever is fine enough and still kept. A check that judges a value also has `values` — `{ unit, days, points: [{t, v, lo, hi}], latest, lowest, highest }` — over as much of that stretch as results are kept (14 days); otherwise `values` is null.
|
||||||
|
|
||||||
|
### `GET /api/v1/maintenance`
|
||||||
|
|
||||||
|
```json
|
||||||
|
{ "active": [
|
||||||
|
{ "id": 1, "target": { "type": "server", "id": 3, "name": "photos", "kind": "Server" }, "reason": "Kernel update",
|
||||||
|
"startsAt": "2026-10-20T21:45:43.616Z", "endsAt": "2026-10-20T22:45:43.616Z", "createdBy": "Demo Admin" } ],
|
||||||
|
"upcoming": [] }
|
||||||
|
```
|
||||||
|
|
||||||
|
### Examples
|
||||||
|
|
||||||
|
**curl and `jq`: is anything critical?**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -fsS -H "Authorization: Bearer $HLM_TOKEN" https://hm.example.lan/api/v1/alerts \
|
||||||
|
| jq -r '.alerts[] | select(.severity=="critical" and (.silenced|not)) | "\(.source): \(.message)"'
|
||||||
|
```
|
||||||
|
|
||||||
|
**A shell check that fails when something needs attention** (for cron, or a Kuma "push" monitor):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
n=$(curl -fsS -H "Authorization: Bearer $HLM_TOKEN" https://hm.example.lan/api/v1/alerts | jq '.needingAttention.critical')
|
||||||
|
[ "$n" -eq 0 ] || { echo "$n critical alerts"; exit 1; }
|
||||||
|
```
|
||||||
|
|
||||||
|
**Python:**
|
||||||
|
|
||||||
|
```python
|
||||||
|
import os, requests
|
||||||
|
|
||||||
|
BASE = "https://hm.example.lan/api/v1"
|
||||||
|
HEADERS = {"Authorization": f"Bearer {os.environ['HLM_TOKEN']}"}
|
||||||
|
|
||||||
|
def get(path, **params):
|
||||||
|
r = requests.get(f"{BASE}{path}", headers=HEADERS, params=params, timeout=10)
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.json()
|
||||||
|
|
||||||
|
for s in get("/servers", status="offline")["servers"]:
|
||||||
|
print(s["name"], "last seen", s["lastSeenAt"])
|
||||||
|
|
||||||
|
full = [(s["name"], d["mount"], d["usedPercent"]) for s in get("/servers")["servers"] for d in s["disks"] if d["usedPercent"] and d["usedPercent"] > 85]
|
||||||
|
print(full)
|
||||||
|
```
|
||||||
|
|
||||||
|
**Node.js (18+):**
|
||||||
|
|
||||||
|
```js
|
||||||
|
const res = await fetch("https://hm.example.lan/api/v1/endpoints", { headers: { Authorization: `Bearer ${process.env.HLM_TOKEN}` } });
|
||||||
|
if (!res.ok) throw new Error(`${res.status} ${await res.text()}`);
|
||||||
|
const { endpoints } = await res.json();
|
||||||
|
console.log(endpoints.filter((e) => e.status === "down").map((e) => e.name));
|
||||||
|
```
|
||||||
|
|
||||||
|
**Home Assistant** — put the whole header value, including the word `Bearer`, in `secrets.yaml`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# secrets.yaml
|
||||||
|
hlm_auth: "Bearer hlm_p_xxxxxxxxxxxxxxxxxxxxxxxx"
|
||||||
|
```
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# configuration.yaml
|
||||||
|
rest:
|
||||||
|
- resource: https://hm.example.lan/api/v1/alerts
|
||||||
|
scan_interval: 60
|
||||||
|
headers:
|
||||||
|
Authorization: !secret hlm_auth
|
||||||
|
sensor:
|
||||||
|
- name: "Homelab critical alerts"
|
||||||
|
value_template: "{{ value_json.needingAttention.critical }}"
|
||||||
|
- name: "Homelab warnings"
|
||||||
|
value_template: "{{ value_json.needingAttention.warning }}"
|
||||||
|
binary_sensor:
|
||||||
|
- name: "Homelab needs attention"
|
||||||
|
value_template: "{{ value_json.needingAttention.critical > 0 }}"
|
||||||
|
device_class: problem
|
||||||
|
```
|
||||||
|
|
||||||
|
A similar sensor on `/api/v1/servers?status=offline` with `value_template: "{{ value_json.servers | length }}"` counts servers that have gone quiet.
|
||||||
|
|
||||||
|
## Other addresses for machines
|
||||||
|
|
||||||
|
These do not use a session and are not part of the read API. Each is protected by its own token and does one thing.
|
||||||
|
|
||||||
|
### Agent report — `POST /api/agent/report`
|
||||||
|
|
||||||
|
What the agents on the servers call, every 15 minutes by default. The token is the **server's own** (shown once when the server is added under **Servers → Add a server**, and put in the agent by the install
|
||||||
|
one-liner); it can only replace *that* server's report. You can call it yourself from anything that can make an HTTPS request — a router, a NAS, a script — to make a server appear and be monitored.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -fsS -X POST https://hm.example.lan/api/agent/report \
|
||||||
|
-H "Authorization: Bearer <the server's token>" -H "Content-Type: application/json" \
|
||||||
|
-d '{
|
||||||
|
"hostname": "nas",
|
||||||
|
"os_type": "linux",
|
||||||
|
"agent_version": 7,
|
||||||
|
"reported_at": "2026-10-20T21:50:00Z",
|
||||||
|
"system": {
|
||||||
|
"ip_addresses": ["192.168.1.30"],
|
||||||
|
"cpu": { "model": "Intel(R) N100", "cores": 4, "load_percent": 12.4 },
|
||||||
|
"memory": { "total_bytes": 16623411200, "used_bytes": 4210356224 },
|
||||||
|
"disks": [ { "mount": "/", "size_bytes": 245107195904, "used_bytes": 61276798976 } ],
|
||||||
|
"listening_ports": [ { "protocol": "tcp", "port": 22, "address": "0.0.0.0", "process": "sshd" } ],
|
||||||
|
"updates": { "manager": "apt", "count": 5, "security": 2, "reboot_required": false },
|
||||||
|
"health": {
|
||||||
|
"failed_units": [ { "unit": "foo.service", "description": "Foo" } ],
|
||||||
|
"smart": [ { "device": "/dev/sda", "model": "WD Red", "healthy": true, "temperature_c": 38, "wear_percent": null } ],
|
||||||
|
"zfs": [ { "name": "tank", "health": "ONLINE", "capacity_percent": 61 } ]
|
||||||
|
},
|
||||||
|
"hardware": { "manufacturer": "ASUS", "model": "PN42", "serial": "…" },
|
||||||
|
"os": { "id": "debian", "name": "Debian GNU/Linux 12", "version": "12" }
|
||||||
|
},
|
||||||
|
"tasks": [ { "schedule_type": "cron", "name": "backup", "command": "/usr/local/bin/backup.sh", "schedule_expression": "0 3 * * *", "source": "root" } ]
|
||||||
|
}'
|
||||||
|
```
|
||||||
|
|
||||||
|
- **Everything in `system` is optional**, and a part that is malformed is dropped rather than costing the report. Text is trimmed to a sane length. `tasks` may be `[]` (but is required).
|
||||||
|
- Answers: `202` with `{ "ok": true, "taskCount": 1 }` when the report is taken; `401` for no or a wrong token (`missing_token`, `invalid_token`); `400` (`invalid_body`) for a report that is not JSON of this shape. Reports may be up to **4 MB**.
|
||||||
|
- `schedule_type` is `cron`, `systemd_timer` or `windows_task`. `os_type` is recorded only if it is `linux` or `windows`. The same token replaces the previous report's tasks and figures.
|
||||||
|
- `GET /api/agent/report/update?version=<n>` (same token) answers `{ "update": bool, "version": …, "sha256": … }` for an agent that has been told to keep itself up to date; see [AGENT_UPDATE.md](../AGENT_UPDATE.md).
|
||||||
|
|
||||||
|
How the figures are used is in [MONITORING.md](MONITORING.md).
|
||||||
|
|
||||||
|
### Heartbeats, metrics, status pages and calendars
|
||||||
|
|
||||||
|
| Address | What | Token | Documented in |
|
||||||
|
|---|---|---|---|
|
||||||
|
| `GET /api/heartbeat/<token>[/start\|/fail]` | A cron job says it ran | in the address | [HEARTBEATS-AND-CALENDAR.md](HEARTBEATS-AND-CALENDAR.md) |
|
||||||
|
| `GET /metrics` | Prometheus text format | `Authorization: Bearer hlm_m_…` | [PROMETHEUS-GRAFANA.md](PROMETHEUS-GRAFANA.md) |
|
||||||
|
| `GET /api/public/calendar/<token>.ics` | A calendar feed of what is coming up | in the address (a share link) | [HEARTBEATS-AND-CALENDAR.md](HEARTBEATS-AND-CALENDAR.md) |
|
||||||
|
| `GET /api/public/status/<token>`, `/api/public/wall/<token>` | The data behind a shared status page and wall display | in the address (a share link) | Settings → Share links |
|
||||||
|
| `GET /api/alert-actions/<token>`, `POST /api/alert-actions/<token>/<ack\|silence\|snooze>` | The page and the buttons behind a notification | a signed token in the address (24 hours, for the alerts in that message) | [WEBHOOKS.md](WEBHOOKS.md#buttons-on-notifications) |
|
||||||
|
| `GET /health` | Is the app up? | none | answers `{"ok":true}` |
|
||||||
|
|
||||||
|
## Reading the API safely
|
||||||
|
|
||||||
|
- Treat a token like a password: it can read everything its owner can. Give each script its own (so one can be removed alone), choose the shortest life that suits, and keep it in a secret store, not in a repository.
|
||||||
|
- Use HTTPS. The token is sent in a header with every request.
|
||||||
|
- A removed token stops working at once; if one leaks, remove it under **API tokens** and make another.
|
||||||
|
- Every token's last use is shown on the API tokens page, which makes an unused one easy to spot and remove.
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
# Heartbeats and the calendar feed
|
||||||
|
|
||||||
|
Two addresses that are not part of the [read API](API.md) and need no sign-in, because each carries its own token:
|
||||||
|
|
||||||
|
- **Heartbeats** — a cron job or script calls an address when it finishes, and Homelab Manager speaks up when it does not. This is how to monitor things the app cannot see by itself: a backup script,
|
||||||
|
a nightly sync, a certificate renewal.
|
||||||
|
- A **calendar feed** — what is coming up (expiries, warranties, maintenance), for a phone or desktop calendar to subscribe to.
|
||||||
|
|
||||||
|
How a missed heartbeat becomes an alert, a notification and an incident is in [MONITORING.md](MONITORING.md).
|
||||||
|
|
||||||
|
## Heartbeats — for cron jobs and scripts
|
||||||
|
|
||||||
|
### Setting one up
|
||||||
|
|
||||||
|
Administrators: **Monitoring → Heartbeats → Add a heartbeat**.
|
||||||
|
|
||||||
|
| Field | Notes |
|
||||||
|
|---|---|
|
||||||
|
| Name | what the job is, e.g. "Nightly backup" (up to 100 characters) |
|
||||||
|
| **Period** | how often the job should report, from 1 minute to 30 days |
|
||||||
|
| **Grace** | how much later than that is still fine, 0 minutes to a day (so a job that sometimes runs long is not a false alarm) |
|
||||||
|
| Server | optional: the server the job runs on. **Maintenance on that server silences the heartbeat's alerts**, and the alert links to it. |
|
||||||
|
| Notes | up to 500 characters |
|
||||||
|
|
||||||
|
Up to **100** heartbeats. Everyone signed in sees how each job is doing; only administrators see its **address** (with the token) — it is what lets anyone say "I'm alive" for the job. **New address** gives
|
||||||
|
a heartbeat a new token and the old address stops working at once.
|
||||||
|
|
||||||
|
### Calling it
|
||||||
|
|
||||||
|
The job calls its address when it finishes:
|
||||||
|
|
||||||
|
```
|
||||||
|
GET /api/heartbeat/<token> it worked
|
||||||
|
GET /api/heartbeat/<token>/start it began (optional: lets the next call say how long it took)
|
||||||
|
GET /api/heartbeat/<token>/fail it ran and went wrong: an alert at once
|
||||||
|
```
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# in cron, after the job
|
||||||
|
0 3 * * * /usr/local/bin/backup.sh && curl -fsS -m 10 --retry 3 -o /dev/null https://hm.example.lan/api/heartbeat/hb_xxxxxxxxxxxxxxxxxxxxxxxx
|
||||||
|
|
||||||
|
# in a script, with a start, a failure and a note
|
||||||
|
curl -fsS -m 10 -o /dev/null https://hm.example.lan/api/heartbeat/hb_xxxx/start
|
||||||
|
./backup.sh && curl -fsS -m 10 -o /dev/null --data-raw "3.2 GB copied" https://hm.example.lan/api/heartbeat/hb_xxxx \
|
||||||
|
|| curl -fsS -m 10 -o /dev/null --data-raw "exit code $?" https://hm.example.lan/api/heartbeat/hb_xxxx/fail
|
||||||
|
```
|
||||||
|
|
||||||
|
- The token in the address is the whole of the access, and all it allows is to say "I'm alive": no session, no sign-in, and nothing can be read with it. GET, HEAD, POST and PUT all work.
|
||||||
|
Keep the address between the job and you; if it gets out, give the heartbeat a new address (the old one stops working at once).
|
||||||
|
- Anything sent with a call — the body of a POST, or `?msg=…` — is kept as a note, cut to one line of at most 200 characters.
|
||||||
|
- The answer is `200` and `OK`. A token that doesn't exist, a path other than the three, and a token of the wrong shape all get `404 Not found` in the same words. Calls are limited to 120 a minute from one address.
|
||||||
|
- A job is **late** when its period and the grace have passed since its last call (or since the heartbeat was made, if it has never called). That is a warning, announced once and again as cleared when the next call comes;
|
||||||
|
a call to `/fail` is critical and stays until a call that works. A heartbeat that is switched off still answers `200`.
|
||||||
|
- Both are alerts of the *Automation* category, so notification routing, tags, quiet hours and escalation apply, and maintenance on the server a heartbeat is linked to silences them.
|
||||||
|
|
||||||
|
## Calendar feed
|
||||||
|
|
||||||
|
An administrator makes a **calendar feed** under **Settings → Share links**: an address ending in `.ics` for a calendar app to subscribe to (Google Calendar, Apple Calendar, Outlook and Thunderbird all take "add a calendar from a URL").
|
||||||
|
|
||||||
|
```
|
||||||
|
GET /api/public/calendar/<token>.ics
|
||||||
|
```
|
||||||
|
|
||||||
|
- Like any share link, the token is the whole of the access; only its hash is kept, and the address is shown once, when the link is made. It can have an end date and can be removed at any time.
|
||||||
|
A link that doesn't work — never made, ended, removed, or one that opens a wall or a status page instead — gets `404` in the same words. Limited to 60 requests a minute from one address.
|
||||||
|
- It carries what **Coming up** lists, for the next year: all-day events for secrets, domains, certificates, Tailscale keys, warranties, end of support and restore tests, and events with a start and an end for planned and
|
||||||
|
recurring maintenance (every occurrence, which the page folds into one line). Updates that are waiting have no day and are not in it. Something that expired in the last week is kept; older is left out.
|
||||||
|
- With **kinds only**, an event says only what kind of thing it is ("A secret expires") and has no description, so it can go on a shared calendar. With **names** it carries the name and a link back to the app.
|
||||||
|
An event's `UID` is a hash of what it is and its date, the same on every fetch.
|
||||||
|
- The feed asks to be refreshed every six hours (`REFRESH-INTERVAL`); a calendar app decides for itself how often it does.
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
# Languages
|
||||||
|
|
||||||
|
The interface is in English and, for the parts people see every day, in Swedish. This page says how a person chooses, what is covered, and how to translate more.
|
||||||
|
|
||||||
|
## Choosing
|
||||||
|
|
||||||
|
Open the user menu (your name, bottom of the sidebar) and press **English** or **Svenska** under *Language*. The whole screen is drawn again in that language at once, and the choice is **kept on your account**, so it follows you to any browser. Somebody who has never chosen gets the language their browser asks for — Swedish if it is set to Swedish, otherwise English — and that guess is not stored. The sign-in page follows the browser (or what this browser last used) since nobody is signed in yet. `<html lang>` is set to match, for screen readers.
|
||||||
|
|
||||||
|
The choice is one person's; an administrator cannot switch it for anyone else, and a viewer can use Swedish while the others use English.
|
||||||
|
|
||||||
|
## What is translated
|
||||||
|
|
||||||
|
Swedish covers the menu and breadcrumbs, the shared parts (search, tables, pagination, empty lists, dialogs, toasts, "5 min ago"), the sign-in page and the user menu, and the headings, buttons, table columns, filters, labels and messages of the pages used every day: **Dashboard, Alerts, Servers (and a server's own page), Updates (with update runs, restarts and patch plans), Maintenance, My work, Coming up, To do, Integrations, Settings → Display**, and the history of integration figures. Texts that have no Swedish yet show in English, so a page can be half done without anything looking broken.
|
||||||
|
|
||||||
|
Not translated yet:
|
||||||
|
|
||||||
|
- **What the server writes** — alert messages, notifications and their channels, the weekly digest, the monthly report, e-mails, the change log's sentences, audit entries. These are made in many places on the server and are English; translating them is a separate piece of work.
|
||||||
|
- The long explanatory introductions at the top of pages, and the less-used pages and settings, apart from words they share with the above (*Save*, *Cancel*, *Delete* …).
|
||||||
|
- The documentation (these files).
|
||||||
|
|
||||||
|
Dates and times keep following **Settings → Display** (year-month-day by default, which is also the Swedish habit). Numbers use the language's own separators only where a figure is formatted by the page itself.
|
||||||
|
|
||||||
|
## Adding to it (for whoever changes the code)
|
||||||
|
|
||||||
|
English is the language the code is written in, and the English text is its own key:
|
||||||
|
|
||||||
|
```tsx
|
||||||
|
import { t, tn } from "../i18n";
|
||||||
|
|
||||||
|
<button>{t("Save")}</button>
|
||||||
|
toast.success(t("Started updating {name}.", { name }));
|
||||||
|
tn(n, "{n} server", "{n} servers") // the one form for 1, the many form otherwise
|
||||||
|
```
|
||||||
|
|
||||||
|
- The Swedish is an entry in `web/src/i18n/sv.ts`: `"Save": "Spara"`. A text with no entry is shown as written, so nothing breaks while it is waiting for a translation.
|
||||||
|
- **Keep the `{placeholders}` the same** in the Swedish as in the English. A test (`server/test/language.test.ts`) checks that, that no entry is empty, and that every page of the menu has a Swedish name.
|
||||||
|
- Brand and product names (Proxmox, Synology, Docker …) are the same in both languages and have no entry.
|
||||||
|
- Do not call `t()` where a module is loaded (in a constant outside any component or function): it would be fixed to the language at that moment. Keep the English in the constant and call `t(label)` where it is drawn, as the menu (`layout/nav.tsx`) does. Where a constant holds a table of words, a getter does the same without touching where it is drawn: `{ get label() { return t("Running"); } }`. A test (`server/test/language.test.ts`) fails on any `t()` called outside a function.
|
||||||
|
- Do not translate half a sentence that runs on into a link or more English: put the whole sentence in one `t()` with placeholders, or leave it all English.
|
||||||
|
- A local variable called `t` hides the function; call such a variable something else (`todo`, `item`).
|
||||||
|
- Changing the language re-draws the whole app (the tree is keyed on it), so no component needs to subscribe to anything.
|
||||||
|
|
||||||
|
The account setting is `users.language` (`en`, `sv` or null for "follow the browser"); it is read with `GET /api/me` (`user.language`) and changed with `PUT /api/me/language` (`{ "language": "sv" }`, or `null`).
|
||||||
@@ -0,0 +1,734 @@
|
|||||||
|
# How monitoring works
|
||||||
|
|
||||||
|
This page explains what Homelab Manager watches, how often, how a problem becomes an alert, and what happens next — notifications, acknowledgements, maintenance, incidents, history. It is the
|
||||||
|
"why did it do that?" page. Setting things up is in the other pages ([SSO](SSO.md), [API](API.md), [Prometheus and Grafana](PROMETHEUS-GRAFANA.md), [webhooks](WEBHOOKS.md),
|
||||||
|
[heartbeats](HEARTBEATS-AND-CALENDAR.md)); the list of every notification is [NOTIFICATIONS.md](../NOTIFICATIONS.md).
|
||||||
|
|
||||||
|
- [The big picture](#the-big-picture)
|
||||||
|
- [What is watched, and how often](#what-is-watched-and-how-often)
|
||||||
|
- [Servers and the agent](#servers-and-the-agent)
|
||||||
|
- [From a reading to an alert](#from-a-reading-to-an-alert)
|
||||||
|
- [The built-in checks](#the-built-in-checks)
|
||||||
|
- [What is normal for the time](#what-is-normal-for-the-time)
|
||||||
|
- [Service levels](#service-levels)
|
||||||
|
- [Your own alert rules](#your-own-alert-rules)
|
||||||
|
- [Duplicati backups](#duplicati-backups)
|
||||||
|
- [Nextcloud](#nextcloud)
|
||||||
|
- [Hetzner Cloud](#hetzner-cloud)
|
||||||
|
- [Hetzner Robot](#hetzner-robot)
|
||||||
|
- [Game servers (Fragnet)](#game-servers-fragnet)
|
||||||
|
- [Media managers](#media-managers-sonarr-radarr-prowlarr)
|
||||||
|
- [Download clients](#download-clients-qbittorrent-sabnzbd)
|
||||||
|
- [App versions](#app-versions)
|
||||||
|
- [AdGuard Home](#adguard-home)
|
||||||
|
- [Technitium DNS](#technitium-dns)
|
||||||
|
- [Grafana](#grafana)
|
||||||
|
- [Speedtest](#speedtest)
|
||||||
|
- [NetBox](#netbox)
|
||||||
|
- [n8n](#n8n)
|
||||||
|
- [Immich](#immich)
|
||||||
|
- [Paperless-ngx](#paperless-ngx)
|
||||||
|
- [Prometheus](#prometheus)
|
||||||
|
- [Authentik](#authentik)
|
||||||
|
- [Jellyfin](#jellyfin)
|
||||||
|
- [Docker containers that are broken](#docker-containers-that-are-broken)
|
||||||
|
- [Endpoint checks and heartbeats](#endpoint-checks-and-heartbeats)
|
||||||
|
- [Integrations](#integrations)
|
||||||
|
- [Forecasts](#forecasts)
|
||||||
|
- [Dependencies: one outage, one alert](#dependencies-one-outage-one-alert)
|
||||||
|
- [What happens to an alert](#what-happens-to-an-alert) — notify, acknowledge, silence, snooze, flapping, escalate
|
||||||
|
- [Incidents](#incidents)
|
||||||
|
- [History and how long it is kept](#history-and-how-long-it-is-kept)
|
||||||
|
- [Where to see it](#where-to-see-it)
|
||||||
|
- [Tuning, in questions and answers](#tuning-in-questions-and-answers)
|
||||||
|
- [What monitoring does not do](#what-monitoring-does-not-do)
|
||||||
|
|
||||||
|
## The big picture
|
||||||
|
|
||||||
|
```
|
||||||
|
agents ─────────┐
|
||||||
|
integrations ───┤ ┌──► Alerts page ──► acknowledge · snooze · close · silence
|
||||||
|
endpoint checks ┼─► checks ─► conditions ─┼──► notifications ─► Gotify · ntfy · Telegram · Pushover · email · webhook · push
|
||||||
|
heartbeats ─────┤ (rules) "a problem" ├──► event webhooks (machines)
|
||||||
|
your own rules ─┘ ├──► incidents (the record) and history
|
||||||
|
└──► /metrics and the read API
|
||||||
|
```
|
||||||
|
|
||||||
|
Three ideas run through everything:
|
||||||
|
|
||||||
|
1. **A check turns readings into *conditions*.** A condition is "this specific problem exists right now" and has a stable **key** (`offline:server:3`, `disk:server:3:/`) — the same problem always has the same key.
|
||||||
|
2. **Alerts are announced on a change, not on a schedule.** A problem is announced **once** when it appears and **once** when it ends. It is not repeated every time it is looked at.
|
||||||
|
3. **Silence is never "all clear".** When something cannot be *read* (an integration is down, a server's agent is quiet), what depended on it is **held exactly as it was** — neither raised nor cleared —
|
||||||
|
and the Alerts page lists it under *Couldn't check everything*. The app never turns "I couldn't look" into "nothing is wrong".
|
||||||
|
|
||||||
|
## What is watched, and how often
|
||||||
|
|
||||||
|
| What | How it gets to the app | How often |
|
||||||
|
|---|---|---|
|
||||||
|
| **Servers** (Linux, Windows) | The **agent** on each server **pushes** a report: load, memory, disks, listening ports, waiting updates, failed services, SMART and ZFS health, the OS it runs, scheduled tasks. The app never connects *to* a server. | Every **15 minutes** by default (set when the agent is installed) |
|
||||||
|
| **Proxmox**, **Synology** | The health pass **reads** them: node and storage usage, volume and disk status. | Every **15 minutes** |
|
||||||
|
| **UniFi** | A sync **reads** devices, networks, Wi-Fi names and clients and keeps them. | Every **5 minutes** |
|
||||||
|
| **Home Assistant** | A sync **reads** the entities you chose to watch and keeps their state and a history. | Every **5 minutes** |
|
||||||
|
| **UPS** (NUT or SNMP) | A sync **reads** each UPS and keeps its state and a history. | Every **minute** |
|
||||||
|
| **Nginx Proxy Manager** | **Read** when its page is opened and by the alert check, at most once a minute. | On demand, cached 1 minute |
|
||||||
|
| **Semaphore**, **Gitea** | The automation check **reads** the latest run of each template or workflow. | Every **15 minutes** |
|
||||||
|
| **Endpoint checks** (web address, TCP port, TLS certificate) | The app **asks** each one itself. | Each check has its own interval (**5 minutes** by default); the scheduler looks **every minute** for the ones that are due |
|
||||||
|
| **Heartbeats** (cron jobs) | The job **calls** an address when it finishes. | When the job runs; judged every **minute** |
|
||||||
|
| **Secrets**, **certificates**, **domains**, **Tailscale keys** | Their expiry dates, with the certificate fetched live. | **Daily**, at the reminder time in Settings → Notifications |
|
||||||
|
| **Docker image updates**, **Proxmox backup results**, **Proxmox Backup verification** | Read and judged. | **Daily**, at the same time |
|
||||||
|
| **Your own alert rules** | Judged from the history the agents' reports build. | On the **15-minute** health pass |
|
||||||
|
| **Every alert, together** | The Alerts page, the incident recorder and the snooze timer look at the whole picture. | Alerts page: on open, reusing a result up to **1 minute** old · incidents: every **5 minutes** · snoozes: every **minute** · escalation and flapping: every **5 minutes** |
|
||||||
|
|
||||||
|
After the app **restarts**, server-based checks are paused for **20 minutes** so agents get a chance to report before any server is judged.
|
||||||
|
|
||||||
|
## Servers and the agent
|
||||||
|
|
||||||
|
The agent is a small script (a systemd timer on Linux, a scheduled task on Windows) that **only pushes**. Each server has its own token, which can do one thing: replace that server's own report. The app
|
||||||
|
replies "received" and nothing more, so it has no way to make a server run anything. Installation and updating are in the [README](../README.md) and [AGENT_UPDATE.md](../AGENT_UPDATE.md).
|
||||||
|
|
||||||
|
**When is a server "offline"?** When its agent has not reported for longer than the offline limit — **60 minutes** by default (Settings → Notifications → *Servers and disks*), or the server's own limit if it
|
||||||
|
has one (minimum 15). Four reports at the default interval, so one missed report is nothing. A server that **never** reported is "unknown": it has no agent to go quiet, so nothing alerts.
|
||||||
|
|
||||||
|
While a server is offline its other figures are stale, so it is **not also judged** on disks, services or your rules — its offline alert says it all.
|
||||||
|
|
||||||
|
**Per-server settings.** A server's *Settings* section can replace, for that server only: the offline limit, the disk-full percentage, the forecast window, how old its Proxmox guest's backup may be, a limit for
|
||||||
|
one disk or "never alert about this disk", and services never to count as failed (names or patterns such as `gupdate*`). Whatever it does not replace follows the global setting.
|
||||||
|
|
||||||
|
**What the report is used for:** the server's page and history (a sample every 10–15 minutes), the disk forecast, the failed-services and SMART/ZFS checks, new-port detection, the operating-system support
|
||||||
|
check, pending updates, the dependency map, and `/metrics`.
|
||||||
|
|
||||||
|
## From a reading to an alert
|
||||||
|
|
||||||
|
There are two paths, designed so they cannot disagree:
|
||||||
|
|
||||||
|
**1. The state-based checks (the notifications).** Every 15 minutes (endpoint checks and heartbeats: every minute) a check produces the **list of conditions that exist right now**. The app compares that list with
|
||||||
|
the one it remembered from last time:
|
||||||
|
|
||||||
|
- **new** → announced (a notification, an event webhook, a row in the incident record);
|
||||||
|
- **gone** → announced as resolved;
|
||||||
|
- **still there** → nothing;
|
||||||
|
- **its source could not be read this time** → carried over untouched (not resolved, not repeated).
|
||||||
|
|
||||||
|
The remembered list is in the database, so a restart does not forget what was already announced. The first ever run records what is already failing silently, as the baseline, instead of announcing it all.
|
||||||
|
|
||||||
|
**2. The Alerts page.** It asks the same checks, plus the daily ones (expiry, backups, updates, forecasts …), **live**, and turns what they find into one flat list. It ignores the per-event on/off toggles —
|
||||||
|
the page is for looking at, not for being interrupted by — and keeps problems that are under maintenance, marked as silenced instead of dropping them. A result is reused for up to a minute, and every source has a
|
||||||
|
20-second time limit so one hung integration cannot hang the page. A check that fails or times out costs only its own section, and appears under *Couldn't check everything*.
|
||||||
|
|
||||||
|
Each alert has:
|
||||||
|
|
||||||
|
| | |
|
||||||
|
|---|---|
|
||||||
|
| **Severity** | `critical`, `warning` or `info`. Things that are *gone* (offline, a failing SMART disk, a disk over 95%) are critical; things that are *coming* (a forecast, an expiring certificate, an OS going out of support) are warnings or notes. |
|
||||||
|
| **Category** | `offline`, `disk`, `services`, `backup`, `updates`, `expiry`, `automation`, `integration`, `monitoring`, `tickets` — what the filter on the page and the [routing rules](#who-hears-about-it) work on. |
|
||||||
|
| **Source** | Where it comes from: Server, Proxmox, Synology, UniFi, Endpoint, Heartbeat … |
|
||||||
|
| **Target** | What a maintenance window would silence it with (a server, an integration, a device), where it makes sense. |
|
||||||
|
| **Key / id** | Stable identity; the same problem keeps it. Acknowledgements, snoozes, incidents and webhooks all follow it. |
|
||||||
|
|
||||||
|
## The built-in checks
|
||||||
|
|
||||||
|
Every threshold here is a default; where it can be changed, the place is named.
|
||||||
|
|
||||||
|
| Check | Raised when | Default | Changed in |
|
||||||
|
|---|---|---|---|
|
||||||
|
| **Server offline** | The agent has not reported for longer than the limit | 60 min · critical | Settings → Notifications; per server |
|
||||||
|
| **Disk nearly full** | A server disk, a Proxmox node's root filesystem or storage, or a Synology volume is at or over the limit | 90% · warning, **critical from 95%** | Settings → Notifications; per server, per disk |
|
||||||
|
| **Disk filling up** | A disk on a steady climb will be full within the window ([Forecasts](#forecasts)) | 14 days · warning, critical within 3 days | Settings → Notifications; per server |
|
||||||
|
| **Failed services** | A server's agent reports failed services (systemd units on Linux, auto-start services that stopped with an error on Windows); only the ones the server is told to **watch** count | any | per server (watch list) |
|
||||||
|
| **Disk health** | A disk's SMART self-assessment fails (critical) or an SSD has used ≥ 90% of its rated life (warning) | | not configurable |
|
||||||
|
| **ZFS / storage pool** | A ZFS pool (or Windows Storage Spaces pool) is not ONLINE | | not configurable |
|
||||||
|
| **Synology volume / disk** | A volume's status is not "normal"; a disk's status or SMART result is not normal, or it is over the bad-sector or under the remaining-life limit | | not configurable |
|
||||||
|
| **Unusual for the time** | CPU load or memory use over the last hour is far above what is normal for this hour of this weekday ([below](#what-is-normal-for-the-time)) | on · warning | Settings → Notifications |
|
||||||
|
| **New port** | A TCP port starts listening that was not there when the agent first reported and that nobody noted on the server's *Ports* card or marked as expected on **Network → Ports** (which lists every server's new ports in one place, with *Show only new*, *Mark as expected* and *Mark all as expected*) (the container that publishes it, or else a port preset, names it in the message: "8096/tcp jellyfin (container)", "3389/tcp RDP") | on | Settings → Notifications |
|
||||||
|
| **Network device offline** | A UniFi device has been offline too long | 10 min · critical | Settings → Notifications |
|
||||||
|
| **Operating system support** | A server's OS has passed, or is about to pass, its end of support | within 90 days · note; warning once ended | Settings → Notifications |
|
||||||
|
| **Updates waiting** | The agent reports updates waiting (and, where it can say, security ones and a restart needed) | | Updates page |
|
||||||
|
| **Backup stale / failed** | A Proxmox guest's newest backup is older than its limit, the latest run failed, or no job covers it; Proxmox Backup verification failed | 48 h | Settings → Notifications; per server |
|
||||||
|
| **Restore test** | A restore test failed, or none has been done within the window | failure always; age off by default | Settings → Notifications |
|
||||||
|
| **Expiry** | A secret, certificate, domain or Tailscale key is within its warning window or has expired | per item (domains 30 days) | the item; Settings → Notifications |
|
||||||
|
| **Docker updates** | A container image has a newer version | | Settings → Notifications |
|
||||||
|
| **Nextcloud can't be used / disk low** | The database needs upgrading (critical), maintenance mode is on, or little room is left for the data (critical under 5 GB) | on, with the integration | Nextcloud page; [below](#nextcloud) |
|
||||||
|
| **Hetzner server off / traffic / rescue / backup** | A Hetzner Cloud server that was running is off, is stuck migrating or rebuilding, has used 80% of its month's traffic (critical at 100%), has rescue mode left on, or has backups enabled but none recent | on, with the integration | Hetzner page; [below](#hetzner-cloud) |
|
||||||
|
| **Sonarr / Radarr / Prowlarr** | An application reports a health warning or error, has downloads stuck for two hours, or a disk under 10% free | on, with the integration | Media managers page; [below](#media-managers-sonarr-radarr-prowlarr) |
|
||||||
|
| **Hetzner Robot** | A dedicated server or storage box is cancelled and ends within 30 days (critical within 7), a storage box is locked (critical), a rescue system has been on for two hours, or a server is still "in process" after a day | on, with the integration | Hetzner Robot page; [below](#hetzner-robot) |
|
||||||
|
| **qBittorrent / SABnzbd** | Downloads stuck (an error, or stalled for a day), qBittorrent without a network for 15 minutes, SABnzbd paused for six hours with work waiting or downloading nothing for an hour, or a download disk nearly full | on, with the integration | Download clients page; [below](#download-clients-qbittorrent-sabnzbd) |
|
||||||
|
| **AdGuard Home** | DNS has not answered for ten minutes, or its DNS server is not running (critical); protection left off for an hour, or filter lists not updated for a week | on, with the integration | AdGuard Home page; [below](#adguard-home) |
|
||||||
|
| **Technitium DNS** | DNS has not answered for ten minutes, or a secondary zone has expired (critical); a zone cannot sync, blocking left off for an hour, or block lists overdue for an update (warning) | on, with the integration | Technitium DNS page; [below](#technitium-dns) |
|
||||||
|
| **Grafana** | An alert rule is firing (critical when labelled critical or page), a rule cannot be evaluated, or a data source fails its health check | on, with the integration | Grafana page; [below](#grafana) |
|
||||||
|
| **Game server backup / disk / memory** | A Fragnet game server's newest backup is older than the limit (7 days) or none, its disk is over 90% of the plan (critical from 98%), or its memory is over 90% while it runs | on, with the integration | Game servers page; [below](#game-servers-fragnet) |
|
||||||
|
| **Internet slow / speed tests failing** | The middle of the last three speed tests is under the share you set of what the line is sold at (only with those numbers given), or the last three tests all failed | on, with the integration | Speedtest page; [below](#speedtest) |
|
||||||
|
| **NetBox has no worker** | No background worker is running | on, with the integration | NetBox page; [below](#netbox) |
|
||||||
|
| **n8n workflow failed** | A workflow that is switched on failed on its last run | on, with the integration | n8n page; [below](#n8n) |
|
||||||
|
| **Immich disk filling / jobs failing** | The disk under the library is 90 % full (critical at 95 %), or 10 or more background jobs have failed | on, with the integration | Immich page; [below](#immich) |
|
||||||
|
| **Paperless-ngx needs a look** | The worker, database or Redis is in error (critical), documents failed to import, the index or sanity check is in error, or the disk is 90 % full | on, with the integration | Paperless page; [below](#paperless-ngx) |
|
||||||
|
| **Prometheus target down / alert firing** | A scrape target has not answered for five minutes, or one of Prometheus's own alert rules is firing | on, with the integration | Prometheus page; [below](#prometheus) |
|
||||||
|
| **Authentik needs a look** | An outpost that serves apps has gone quiet (critical), a certificate or API token is about to run out (critical close to it), no worker is connected, or sign-ins or flows are failing | on, with the integration | Authentik page; [below](#authentik) |
|
||||||
|
| **Jellyfin needs a look** | Jellyfin wants a restart, a scheduled task (a library scan) failed, or 10 or more sign-ins failed in a day | on, with the integration | Jellyfin page; [below](#jellyfin) |
|
||||||
|
| **Duplicati backup failed / overdue** | A backup's last run failed (critical from the second in a row), or it has not succeeded for longer than it should | per job (2 days by default) | Duplicati page; [below](#duplicati-backups) |
|
||||||
|
| **Docker container broken** | A container keeps restarting (critical), is unhealthy, stopped unexpectedly, or was created and never started; or Dockhand cannot reach a Docker host | on | Settings → Notifications; [below](#docker-containers-that-are-broken) |
|
||||||
|
| **Docker image has critical vulnerabilities** | Dockhand's own scans of a Docker host's images found one or more *critical* findings (high, medium and low are only shown — unless you set a number for high ones, below) | on | Settings → Notifications; [below](#docker-containers-that-are-broken) |
|
||||||
|
| **Docker host has many high vulnerabilities** | The same scans found at least the number of *high* findings you set (*Also warn when a Docker host has this many high vulnerabilities*; 0, the default, never) | off | Settings → Notifications; [below](#docker-containers-that-are-broken) |
|
||||||
|
| **Docker host has old running images** | A Docker host has running containers whose images were built more than the number of days you set ago (*Warn about running Docker images that were built more than …*; 0, the default, never) — scanned or not | off | Settings → Notifications; [below](#docker-containers-that-are-broken) |
|
||||||
|
| **App behind its newest release** | An application has had a newer release out for the number of days set under Operations → App versions (0, the default, never) — needs the lookup of releases switched on | off | App versions page; [below](#app-versions) |
|
||||||
|
| **Automation failed** | A Semaphore template's or Gitea repository's latest run failed (a run still going or cancelled does not count) | | not configurable |
|
||||||
|
| **Integration failing** | An integration's calls have failed several times in a row ([Integrations](#integrations)) | 3 calls | Settings → Notifications |
|
||||||
|
| **Endpoint check failing / certificate running out** | See [below](#endpoint-checks-and-heartbeats) | | per check |
|
||||||
|
| **Heartbeat late / failed** | See [below](#endpoint-checks-and-heartbeats) | | per heartbeat |
|
||||||
|
| **Home Assistant / Nginx Proxy Manager** | An entity unavailable 15 minutes or past your limit; a certificate in use ending within 14 days, or a host NPM cannot set up | | Home Assistant page |
|
||||||
|
| **Service level** | 80 % of a target's error budget used (warning), or overspent (critical), for the month | per target | Service levels page; [below](#service-levels) |
|
||||||
|
| **UPS** | Mains power lost (critical); battery nearly empty (critical); battery to be replaced; load of 90 % or more (warning) | | UPS page; [INTEGRATIONS.md](../INTEGRATIONS.md#ups-nut-or-snmp) |
|
||||||
|
| **Your own rules** | [Below](#your-own-alert-rules) | | Alert settings → Alert rules |
|
||||||
|
|
||||||
|
## What is normal for the time
|
||||||
|
|
||||||
|
A fixed limit cannot say that a server is busy *for a Tuesday at 03:00*. So for each server's **CPU load** and **memory use** the app keeps a baseline for every hour of the week (168 of them, in the **notification time zone**): the **median** of
|
||||||
|
that hour's averages over the previous weeks (up to ten; the hourly figures are kept four months), and a **robust spread** (the median absolute deviation, so one odd week does not stretch it). An hour needs **four weeks** behind it before
|
||||||
|
it has a baseline at all; until then nothing is said. The hour in progress and hours with fewer than two samples are not counted.
|
||||||
|
|
||||||
|
It is **quiet by design**. A reading is a warning only when **all** of these hold, judged on the last hour of samples (at least three, the newest under 30 minutes old):
|
||||||
|
|
||||||
|
| | CPU | Memory |
|
||||||
|
|---|---|---|
|
||||||
|
| The last hour's average is at least | 50 % | 70 % |
|
||||||
|
| …and above the usual by at least | 30 points | 15 points |
|
||||||
|
| …and above it by at least | four spreads (the spread is taken as at least 4 points) | four spreads (at least 2 points) |
|
||||||
|
| …and the lowest sample in that hour is still above the usual by | 15 points | 7.5 points |
|
||||||
|
|
||||||
|
The last line is what separates a **state** from a **spike**: if the figure dropped back toward normal at any time in the hour, it is not said. Only a *high* reading is ever raised — low is never news. What it finds is an ordinary health
|
||||||
|
condition (`baseline:cpu:server:<id>`, `baseline:mem:server:<id>`, *Monitoring*, warning): announced and recovered like the others, held back by maintenance on the server, snoozed, flapping-damped, routed, an event for webhooks — and left out for
|
||||||
|
a server that has stopped reporting (its offline alert says that). Switch it off with *Warn when CPU or memory is unusually high for the time* under Settings → Notifications → What to send → Servers and disks.
|
||||||
|
|
||||||
|
The same baselines draw the grey **usual** line and band on a server's *History → Trends* (up to 30 days; the median, two spreads either side) and the "usually about 8 % now" beside each figure.
|
||||||
|
|
||||||
|
## Service levels
|
||||||
|
|
||||||
|
**Alert settings → Service levels** holds a number against a service: **99.9 % for Gitea**. A target is set on an endpoint check, a server or an integration and is judged over **calendar months** (on the notification time zone's clock). Its **error budget** is the
|
||||||
|
downtime it allows — minutes in the month × (1 − target) — so 99.9 % allows 44.6 minutes of a 31-day month and 99 % about 7½ hours. The page shows, for this month: the figure so far, the downtime used, what is allowed, what is left, the share of the budget used (and
|
||||||
|
where today's pace would end the month, once a day has passed), and how each of the last six months went.
|
||||||
|
|
||||||
|
How downtime is measured, so the figure can be trusted — and where it cannot:
|
||||||
|
|
||||||
|
| Measured on | From | Precision |
|
||||||
|
|---|---|---|
|
||||||
|
| **Endpoint check** | The check's daily figures: each day's share of failed checks, over the part of that day that is in the period (days are UTC days). Days with no checks are not known, and count neither way. | A day's failed share is spread over the whole day |
|
||||||
|
| **Server** | The moments the audit log recorded it stopping reporting and reporting again | To the minute; only from when the server was created |
|
||||||
|
| **Integration** | The moments calls to that *kind* of service started failing and worked again | To the minute; only from when the app began recording them |
|
||||||
|
|
||||||
|
**Planned maintenance is not taken out**: down is down. A thing with no record for the period has no figure, and the page says so rather than guessing.
|
||||||
|
|
||||||
|
When **80 %** of the budget is used it is a **warning**, and when it is overspent a **critical** alert (key `slo:<id>`, *Monitoring*, source *Service level*): announced and recovered like the others, snoozed, flapping-damped, routed, an event for webhooks —
|
||||||
|
and gone when the month turns over and the budget starts again. Each target can have its alert switched off. The figures are in `/metrics` (`homelab_service_level_*`) and in the monthly, quarterly and annual reports ([below](#history-and-how-long-it-is-kept)).
|
||||||
|
|
||||||
|
## Your own alert rules
|
||||||
|
|
||||||
|
Where the built-in checks do not fit, **Alert settings → Alert rules** (administrators write them, everyone sees them and where each is raised) lets you set a limit of your own on one figure of the servers:
|
||||||
|
|
||||||
|
- **What to watch:** CPU load, memory in use, disk use (every disk, or one by name), or disk temperature.
|
||||||
|
- **When:** above or below a limit, and — for load, memory and disk use — **for how long**. "CPU above 90% for 30 minutes" means every reading in the last 30 minutes is above 90%; the readings are the
|
||||||
|
history the agent reports build (one every 10–15 minutes), so a single dip starts the count again. A disk temperature is only known as of the last report, so it is judged right now and has no time.
|
||||||
|
- **How serious:** warning or critical. **For which servers:** all of them, one, or those with a tag.
|
||||||
|
- **Try it now** shows which servers it would be raised on, from current data, before you save it.
|
||||||
|
|
||||||
|
A rule's finding is an **ordinary alert** with the key `rule:<rule id>:server:<server id>[:<disk>]`, in the *Monitoring* category: announced and resolved like the others, held back by maintenance on the
|
||||||
|
server, damped when flapping, routed by the notification rules, and an event for webhooks. Rules are looked at **when the server reports** (right after its report is stored — so a figure that has been past its limit long enough is raised within seconds of the report, not up to fifteen minutes later) and again
|
||||||
|
on every 15-minute health pass, which is also what notices a rule that was deleted or switched off and what judges the servers that have not reported. "For 15 minutes" is still counted in the readings the reports build (one every 10–15 minutes), so it
|
||||||
|
means the readings over 15 minutes, as before. A server that is offline or has not reported for 45 minutes is not judged.
|
||||||
|
|
||||||
|
The check after a report looks at **that server's rules only** and changes nothing else in what is known: the built-in checks, the other servers and the integrations wait for the pass, and what the pass would say is the same. It is not done for
|
||||||
|
a server looked at in the last minute, nor during the 20-minute grace after the app starts, and it is never run at the same moment as a pass (one waits for the other). What is *unusual for the time* (the baselines) still waits for the pass.
|
||||||
|
|
||||||
|
## Duplicati backups
|
||||||
|
|
||||||
|
Duplicati tells this app how each backup went ([INTEGRATIONS.md](../INTEGRATIONS.md#duplicati) says how to set it up), and **Backups → Duplicati** shows every job: when it last succeeded, how the last run ended, how long it took, how big the source is, how many files changed, and where it stands.
|
||||||
|
|
||||||
|
| State | Meaning | Alert |
|
||||||
|
|---|---|---|
|
||||||
|
| **In order** | The last run worked and it is not overdue | — |
|
||||||
|
| **Warnings** | The last run finished with warnings (a locked file, say). It still counts as a backup. The page shows **what the warnings said** (open *N warnings* on the job, or on a run in its history) | none — unless the job is set to **alert on warnings** (below): then a warning |
|
||||||
|
| **Failed** | The last run ended in `Error` or `Fatal` | warning; **critical from the second failure in a row** |
|
||||||
|
| **Overdue** | It has not succeeded for longer than it is **expected** to — 2 days unless you set it per job (6 hours to 90 days) | warning; **critical at three times that** |
|
||||||
|
|
||||||
|
The alert says which job, on which machine, why (the first error Duplicati reported) and when the last good backup was. Only the *Backup* operation counts: a compact, a repair or a verification does not say whether the data is safe. A machine that is heard but has no jobs says why on its card (the last report was another operation, or could not be read, with counts since it was added) — see [INTEGRATIONS.md](../INTEGRATIONS.md#duplicati).
|
||||||
|
A job can be **muted** (no alert; it is still listed),
|
||||||
|
**Alert on warnings** (an operator's button on each job, off until switched on — some jobs always have a few, and a backup with warnings is still a backup) raises a warning-level alert while the job's last run had warnings, naming the job, how many there were and the first thing they said; the next clean run ends it. A job that is muted raises nothing, this included. Maintenance on the server silences it like the others.
|
||||||
|
Two slower things are watched too. **Warnings in a row:** when the last **three runs in a row** all finished with warnings, the job says so (*Warnings 4 runs in a row*) and a warning-level alert is raised (`duplicati-streak:<job>`) — on by default, off per job with *Stop streak alerts*, and not raised for a job that already alerts on every warning run or that has failed or is overdue. **Room:** Duplicati says how big the backups are where they are kept and, for some storage, how much room there is; the *Stored* column shows the size and its growth over the last 180 days (once there are five readings over a week), and where the room is known an alert (`duplicati-growth:<job>`) is raised when the storage will be full within the **disk forecast window** (Settings → Alerts, 14 days; 0 = off) and the growth is steady; critical at three days or less. Storage that does not say how much room it has is never forecast to fill.
|
||||||
|
|
||||||
|
A job can also be muted, and one that no longer exists can be **forgotten** (with its history; if Duplicati reports it again it is learned again). Each machine can be tied to a server, so **maintenance on that server silences its backups' alerts**.
|
||||||
|
|
||||||
|
What is kept: the last result of each job and what its warnings said (up to the first twenty, each cut at 500 characters — Duplicati itself may send fewer than it counted, and the page says so), and each report, with what its warnings said, for 400 days or 400 reports. `/metrics` has `homelab_duplicati_backup_ok`, `homelab_duplicati_last_success_age_seconds` and `homelab_duplicati_source_size_bytes` for each job. A job is only learned from its first report, so a job that has
|
||||||
|
*never* reported is not known to be missing; and because the reports are pushed, **the app being down when a backup finishes** loses that report (the backup shows as overdue only once it is).
|
||||||
|
|
||||||
|
## Nextcloud
|
||||||
|
|
||||||
|
An integration of type *Nextcloud* ([INTEGRATIONS.md](../INTEGRATIONS.md#nextcloud)) is read every five minutes, and **Apps → Nextcloud** shows its version, whether it is usable, and — with an administrator's app password — the room left for the data, the users, the files and the app updates waiting.
|
||||||
|
|
||||||
|
| Alert | When | Severity |
|
||||||
|
|---|---|---|
|
||||||
|
| **Database needs upgrading** | status.php says `needsDbUpgrade` — nobody can use Nextcloud until `occ upgrade` is run | critical |
|
||||||
|
| **Maintenance mode on** | status.php says `maintenance` — nobody can sign in | warning |
|
||||||
|
| **Data disk low** | Less than 20 GB free where the data is kept (needs the login) | warning; critical under 5 GB |
|
||||||
|
| **Background jobs not running** | Nextcloud's own *Cron last run* check says error — the last job ran more than an hour ago (needs the login; Nextcloud 28 or later) | warning |
|
||||||
|
| **Nextcloud's own checks** | One or more of its other setup checks report an error; one alert naming up to three | warning |
|
||||||
|
|
||||||
|
Maintenance on the integration holds them. A reading older than 20 minutes is not believed. Cron is read through Nextcloud's own setup checks (the Monitoring app has no field for it), so it needs the login and Nextcloud **28 or later**; before that it is *not known* and never counted as fine. Warnings from those checks (HTTP headers, a PHP setting) are shown on the page, not alerted.
|
||||||
|
|
||||||
|
## Hetzner Cloud
|
||||||
|
|
||||||
|
An integration of type *Hetzner Cloud* ([INTEGRATIONS.md](../INTEGRATIONS.md#hetzner-cloud)) is one project, read every five minutes, and **Infrastructure → Hetzner Cloud** shows its servers (status, type, place, addresses, this month's traffic against what the type includes, whether backups are on), volumes, snapshots, and roughly what the project costs. Nothing is started, stopped or changed.
|
||||||
|
|
||||||
|
| Alert | When | Severity |
|
||||||
|
|---|---|---|
|
||||||
|
| **Server off** | A server the app saw running is now off (Hetzner keeps billing a server that is off). One off for **a week**, or never seen running by the app, is left alone — as a stopped container is | warning |
|
||||||
|
| **Stuck in a status** | A server has been starting, stopping, migrating, rebuilding or initializing for **half an hour** | warning |
|
||||||
|
| **Traffic** | A server has sent **80%** of its included traffic for the month; at **100%** it is critical (the excess is billed) | warning; critical at 100% |
|
||||||
|
| **Rescue mode left on** | Rescue mode has been switched on for **an hour** without a reboot — the next reboot (a kernel update, say) would come up in the rescue system and the server would not come back | warning |
|
||||||
|
| **Backups late** | A server has backups enabled and the newest is older than the *backup older than* limit (48 hours by default — the same setting as for Proxmox guests; 0 turns it off), or none has been made and the server is older than that | warning |
|
||||||
|
| **Traffic forecast** | A server is **on course to pass its included traffic before the month ends** (by the last week's rate, or the month's average after three days) while it has used under 80% — the alert says on which day. Not raised from 80% on: the traffic alert covers that | warning |
|
||||||
|
| **Load balancer unhealthy** | A load balancer has a target Hetzner judges unhealthy; critical when none of its targets is healthy | warning; critical when all |
|
||||||
|
| **Certificate ending** | A certificate that is in use ends within **14 days** (critical within 3, or past its end), or Hetzner could not issue or renew a managed one. A certificate nothing uses is not raised | warning; critical within 3 days |
|
||||||
|
| **Floating IP unattached** | A floating IP points at no server — it is billed all the same | warning |
|
||||||
|
|
||||||
|
Hetzner does not say when a server went off or when rescue mode was switched on, so "since when" is **when the app first saw it** in a read. Maintenance on the integration holds the alerts; a reading older than 20 minutes is not believed (they are neither raised nor cleared, and the Alerts page says it could not check). Snapshots, volumes (including ones attached to no server, which are billed all the same) and the cost are shown, not alerted.
|
||||||
|
|
||||||
|
**The cost** is an estimate from Hetzner's own price list, including VAT: the servers, their backups (a percentage of the server's price), volumes, snapshots, primary IPs, load balancers and floating IPs. It leaves out traffic over the quota, so it is a floor, not a bill.
|
||||||
|
|
||||||
|
**The traffic forecast** is made from the month's outgoing traffic, written down about every six hours for two weeks: the rate of the last week (at least two days of it) or, with too little, the month's average so far (after three days). It assumes the month is the calendar month in UTC — not checked against a live project — and a counter that goes down starts the history over. The page shows under each server's traffic bar either the day the quota is passed or what the month is on course to come to.
|
||||||
|
|
||||||
|
**Load balancers, floating IPs and certificates** are listed on the Hetzner page — a load balancer with how many of its targets are healthy, a floating IP with the server it points at, a certificate with when it ends (and the ones nothing uses) — and their prices are in the cost estimate. `/metrics` has each load balancer's unhealthy targets, each certificate's end and each server's forecast month.
|
||||||
|
|
||||||
|
**On a server's page**, a *Hetzner Cloud* card appears when one of the project's servers is that server — matched by its public or private address, else by name — with its status, type, newest backup and traffic. A Hetzner server with no agent is listed as *not in the app's servers*. `/metrics` has `homelab_hetzner_server_running`, `homelab_hetzner_traffic_outgoing_bytes`, `homelab_hetzner_traffic_included_bytes` and `homelab_hetzner_monthly_cost`.
|
||||||
|
|
||||||
|
## Hetzner Robot
|
||||||
|
|
||||||
|
An integration of type *Hetzner Robot* ([INTEGRATIONS.md](../INTEGRATIONS.md#hetzner-robot)) is one Hetzner account's dedicated servers and storage boxes, read every fifteen minutes (Robot allows 200 requests an hour), and **Infrastructure → Hetzner Robot** shows them: each server's product, data centre, addresses, traffic plan, status, rescue system and what is paid until (or, if cancelled, when it ends and how many days are left), and each storage box's product, location, linked server, whether it is locked and what is paid until. Nothing is reset, ordered or cancelled.
|
||||||
|
|
||||||
|
| Alert | When | Severity |
|
||||||
|
|---|---|---|
|
||||||
|
| **Shutting down** | A server (or storage box) is **cancelled** and its last day is within **30 days**. The date Robot gives is the last day it runs; the server and its data go with it. Not-cancelled ones are never raised, however near their renewal | warning; critical within 7 days |
|
||||||
|
| **Storage box locked** | A storage box is locked — its data cannot be reached | critical |
|
||||||
|
| **Rescue system left on** | A server's rescue system has been switched on, in every read, for **two hours**: its next boot is into the rescue system, not its own | warning |
|
||||||
|
| **Stuck being set up** | A server has been something other than "ready" ("in process") in every read for **a day** | warning |
|
||||||
|
|
||||||
|
"Since when" is what the app saw: the time of the first read that showed it, carried from read to read. Maintenance on the integration holds the alerts; a reading older than 45 minutes is not believed (they are neither raised nor cleared, and the Alerts page says it could not check). A server's rescue state that could not be asked is *unknown*, not off, and raises nothing.
|
||||||
|
|
||||||
|
## Media managers (Sonarr, Radarr, Prowlarr)
|
||||||
|
|
||||||
|
An integration of type *Sonarr*, *Radarr*, *Prowlarr*, *Lidarr* or *Readarr* ([INTEGRATIONS.md](../INTEGRATIONS.md#sonarr-radarr-and-prowlarr)) is read every five minutes, and **Apps → Media managers** shows, for each: what the application says is wrong with itself, its download queue (the stuck ones first, with why), its disks, how many wanted items have no file (Sonarr, Radarr) and, for Prowlarr, which indexers it has switched off after failures. Nothing is searched, grabbed or changed.
|
||||||
|
|
||||||
|
| Alert | When | Severity |
|
||||||
|
|---|---|---|
|
||||||
|
| **It says it is unwell** | One of the application's own health checks is a **warning** or an **error** (a download client that cannot be reached, a root folder gone, indexers unavailable…). The alert quotes it. A *notice* — an update being available — is shown on the page, not raised | warning |
|
||||||
|
| **Downloads stuck** | Downloads that have been in the queue for **two hours or more** and are judged warning or error by the application, have failed, or are blocked from importing. One alert for all of an application's, naming the first three, how long the oldest has been there, and the reason it gives | warning |
|
||||||
|
| **Disk nearly full** | A disk the application uses has under **10%** free, critical under 5%. Folders on one disk count as one (the same size and free space), and a disk under 2 GB is not a media disk | warning; critical under 5% |
|
||||||
|
|
||||||
|
Maintenance on the integration holds the alerts; a reading older than 20 minutes is not believed (they are neither raised nor cleared, and the Alerts page says it could not check). A finding is raised under a key made from the check and its words with the numbers left out, so "3 indexers unavailable" becoming "4" is the same alert, and different words are a new one.
|
||||||
|
|
||||||
|
## Download clients (qBittorrent, SABnzbd)
|
||||||
|
|
||||||
|
An integration of type *qBittorrent* or *SABnzbd* ([INTEGRATIONS.md](../INTEGRATIONS.md#qbittorrent-and-sabnzbd)) is read every five minutes, and **Apps → Download clients** shows, for each: how fast it is downloading (and, for qBittorrent, uploading), the items in its queue and their states (the ones needing a look first), whether qBittorrent can reach the network or SABnzbd is paused, the room its download folder has, what failed in the last day (SABnzbd) and SABnzbd's own warnings. Nothing is added, paused, resumed or removed.
|
||||||
|
|
||||||
|
| Alert | When | Severity |
|
||||||
|
|---|---|---|
|
||||||
|
| **Downloads stuck** | A download is in an **error** state (qBittorrent's `error` and `missingFiles`), or is **stalled** — nobody to get it from — with nothing having moved for **a day** (its last activity, or when it was added). One alert for all of a client's, naming the first three and why for the worst | warning |
|
||||||
|
| **No connection** | qBittorrent has torrents loaded and has said `disconnected` in every read for **15 minutes** — typically a VPN it is bound to has dropped. `firewalled` (no incoming connections) is shown, not raised: it still downloads | warning |
|
||||||
|
| **Paused and forgotten** | SABnzbd's whole queue has been paused for **six hours** with downloads waiting | warning |
|
||||||
|
| **Downloading nothing** | SABnzbd is not paused, has downloads waiting, and its speed has been zero for **an hour** — a news server that does not answer or refuses the login | warning |
|
||||||
|
| **Disk nearly full** | A download folder has under **10%** free (SABnzbd, which says how big its disks are), critical under 5%; or, for qBittorrent, which says only what is free, under **20 GB**, critical under 5 GB. A disk under 2 GB is not a download disk | warning; critical under 5% / 5 GB |
|
||||||
|
|
||||||
|
"Since when" is what the app saw: the time of the first read that showed it, carried from read to read, so a restart of the app does not start the clock again. A failed download in SABnzbd's history is shown for the last day, not raised: failures are normal on Usenet, and Sonarr and Radarr try another release (they raise it themselves when downloads stay stuck — see [Media managers](#media-managers-sonarr-radarr-prowlarr)).
|
||||||
|
|
||||||
|
Maintenance on the integration holds the alerts; a reading older than 20 minutes is not believed (they are neither raised nor cleared, and the Alerts page says it could not check).
|
||||||
|
|
||||||
|
## Game servers (Fragnet)
|
||||||
|
|
||||||
|
An integration of type *Fragnet game servers* ([INTEGRATIONS.md](../INTEGRATIONS.md#fragnet-game-servers)) is one account on Fragnet's game panel (a Pterodactyl fork), read every five minutes, and **Infrastructure → Game servers** shows its servers: the game, every address and port, whether it runs, memory and disk against the plan, the newest backup, and (opened) the backups, schedules and SFTP address. Operators can start, stop, restart and kill a server there, and ask for a backup now (*Back up* on the row, or *Back up now* beside the server's backups): the panel makes it in the background, the page reads the account again every twenty seconds until it is done, and a successful one counts for the *backup late* alert like any other. A backup that is still "being made" does not count until it has finished.
|
||||||
|
|
||||||
|
| Alert | When | Severity |
|
||||||
|
|---|---|---|
|
||||||
|
| **Backup late** | The server's plan allows backups and its newest **successful** backup is older than *warn when the newest backup is older than* (7 days by default; 0 turns it off), or it has none and the app has known the server for that long. A backup that failed does not count; the message says when the newest attempt failed. A server can be left out by name | warning |
|
||||||
|
| **Disk nearly full** | The server has used 90% of its disk (*warn when disk or memory is fuller than*); critical from 98% — a full disk stops the game saving | warning; critical from 98% |
|
||||||
|
| **Memory nearly full** | The server uses 90% of its memory limit **while it runs** — past the limit the host stops the game | warning |
|
||||||
|
|
||||||
|
Each of the three can be **switched off on its own** in the integration's settings (Administration → Integrations → edit: *Do not alert about disk space / memory / backups*) — for example to hear about disk and backups but not memory. The page keeps showing the figures; a bar for something switched off is blue instead of green, yellow or red, and the backups tile says *not judged*. There is no CPU alert: the CPU is shown on the server's row only.
|
||||||
|
|
||||||
|
Each game server's row opens to a **power history** (how much of the last 30 days it was on, a seven-day bar and the latest changes, with who pressed the button when it was done from here — see [INTEGRATIONS.md](../INTEGRATIONS.md#fragnet-game-servers)).
|
||||||
|
|
||||||
|
When a server is **turned on or off** it is not an alert but a **notification** — see [NOTIFICATIONS.md](../NOTIFICATIONS.md): one message, through the configured channels, when a read shows it changed since the last one (switch it off under Settings → Notifications → What to send).
|
||||||
|
|
||||||
|
A server that is **stopped** is not an alert (game servers are often stopped on purpose), and one that is suspended or being installed is not judged. "Since when" a server has been in its state is **when the app first saw it** in a read. Maintenance on the integration holds the alerts; a reading older than 20 minutes is not believed (they are neither raised nor cleared, and the Alerts page says it could not check). `/metrics` has the running state, disk and memory against their limits and when the newest backup finished.
|
||||||
|
|
||||||
|
## AdGuard Home
|
||||||
|
|
||||||
|
An integration of type *AdGuard Home* ([INTEGRATIONS.md](../INTEGRATIONS.md#adguard-home)) is read every two minutes, and **Network → AdGuard Home** shows whether DNS answers (a real question over UDP, with how long it took), whether protection is on, the queries and how many were blocked, the names blocked most and the clients asking most, and its filter lists. Nothing is changed.
|
||||||
|
|
||||||
|
| Alert | When | Severity |
|
||||||
|
|---|---|---|
|
||||||
|
| **DNS server not running** | AdGuard Home says its DNS server is not running — nothing that uses it can look up a name | critical |
|
||||||
|
| **DNS not answering** | The test question has gone unanswered (or been answered with a server failure or a refusal) in every read for **ten minutes**. "No such name" counts as an answer | critical |
|
||||||
|
| **Protection off** | Protection has been switched off for **an hour** — someone paused it and forgot; nothing is being blocked | warning |
|
||||||
|
| **Filter lists stale** | Lists that are switched on have not been updated for **a week** (or never) while filtering is on — one alert for all of them | warning |
|
||||||
|
|
||||||
|
"Since when" is **when the app first saw it** in a read. Maintenance on the integration holds the alerts; a reading older than ten minutes is not believed (they are neither raised nor cleared, and the Alerts page says it could not check). `/metrics` has whether it runs, protects and answers, how long an answer took, and the queries and blocked counts.
|
||||||
|
|
||||||
|
## Technitium DNS
|
||||||
|
|
||||||
|
An integration of type *Technitium DNS* ([INTEGRATIONS.md](../INTEGRATIONS.md#technitium-dns)) is read every two minutes, and **Network → Technitium DNS** shows whether DNS answers (a real question over UDP, with how long it took), whether blocking is on, how long the server has run, the last day's queries and how they ended, the clients and names asked about and blocked most, the cache against its limit, the block lists and when they are next updated, the zones that have a problem and the DHCP leases. Nothing is changed. It does not report the CPU, memory or disk of the machine it runs on — Technitium's API has none; a server agent does.
|
||||||
|
|
||||||
|
| Alert | When | Severity |
|
||||||
|
|---|---|---|
|
||||||
|
| **DNS not answering** | The test question has gone unanswered (or been answered with a server failure or a refusal) in every read for **ten minutes**. "No such name" counts as an answer | critical |
|
||||||
|
| **Zone expired** | A secondary zone has expired — it no longer answers. One alert for all the zones with a problem; it is critical if any of them has expired | critical |
|
||||||
|
| **Zone cannot sync** | Secondary zones that cannot sync with their primary and are going stale (none has expired yet) | warning |
|
||||||
|
| **Blocking off** | Blocking has been switched off for **an hour** while block lists are set up — someone paused it and forgot; nothing is being blocked. With no block lists, off is how it is and nothing is said | warning |
|
||||||
|
| **Block lists overdue** | The block lists' next update was due more than **six hours** ago and has not happened | warning |
|
||||||
|
| **Cluster node unreachable** | In a cluster (Technitium 14 and later), the node this app reads says another node is *unreachable*, and has for **ten minutes** (by the node's own *last seen*, or since this app first saw it so). One alert for each node. The cluster keeps answering without it, so a **secondary** is a warning; the **primary** is critical — the other nodes can take no configuration changes while it is down | warning; critical for the primary |
|
||||||
|
| **Cluster configuration out of step** | The node this app reads is a **secondary** and has not taken the configuration from the primary for three of its own refresh periods (15 minutes by default, so 45), and at least half an hour | warning |
|
||||||
|
|
||||||
|
**The cluster** is read from `/api/admin/cluster/state`, which needs the token's user to be allowed to **view administration** (without it the rest is read as before and the page says what is missing). A server that is in no cluster, or is older than 14, shows nothing of it. Point the integration at one node of the cluster, or one integration at each node to judge them from both sides: a node can only say what *it* can reach. Nothing is changed.
|
||||||
|
|
||||||
|
The server's own internal zones and zones that are switched off are not judged. "Since when" is **when the app first saw it** in a read. Maintenance on the integration holds the alerts; a reading older than ten minutes is not believed (they are neither raised nor cleared, and the Alerts page says it could not check). `/metrics` has whether it answers and how long an answer took, whether blocking is on, how long it has run, the last day's counts, the cache, the zones with a problem and the DHCP leases.
|
||||||
|
|
||||||
|
## Grafana
|
||||||
|
|
||||||
|
An integration of type *Grafana* ([INTEGRATIONS.md](../INTEGRATIONS.md#grafana)) is read every five minutes, and **Monitoring → Grafana** shows which alert rules are firing, pending or cannot be evaluated, and how each data source tests. Nothing is silenced or changed.
|
||||||
|
|
||||||
|
| Alert | When | Severity |
|
||||||
|
|---|---|---|
|
||||||
|
| **Rule firing** | A Grafana-managed alert rule is firing (a pending one is only shown). The alert names the rule, its folder and group, since when, and its summary | critical when the rule's `severity` label is `critical` or `page`, else warning |
|
||||||
|
| **Rule cannot be evaluated** | Grafana reports the rule in an error state — an alert that cannot alert is worse than a quiet one — with the error | warning |
|
||||||
|
| **Data source failing** | A data source fails Grafana's own health check. One whose type has no check is *not tested*, never failing | warning |
|
||||||
|
|
||||||
|
Grafana can notify about its own alerts as well; with both switched on you hear twice. Maintenance on the integration holds the alerts; a reading older than 20 minutes is not believed. `/metrics` has the rules, those firing, those in error and the failing data sources.
|
||||||
|
|
||||||
|
## App versions
|
||||||
|
|
||||||
|
**Operations → App versions** lists each connected application whose version is known, with the newest release of it and how far behind it is: *a patch* (the third number), *a minor update* (the second), *a major version* (the first; for Authentik, whose versions are dates, a new year). A build number after the third number is ignored, and prefixes (`v`, `n8n@`, `version/`) are taken off. What is newer here than the newest release (a beta) is *ahead*.
|
||||||
|
|
||||||
|
The newest release is asked of **GitHub** (`GET /repos/{owner}/{repo}/releases/latest`, which leaves out pre-releases and drafts) **once a day, and only for an application whose version has been read** — and **only while an administrator has switched the lookup on**. It sends nothing about you or your systems but the name of the repository, and is the only thing this app asks of an address that is not yours. GitHub limits requests without a sign-in (60 an hour); a lookup that is refused keeps the release it had, says why on the page and is tried again at the next pass.
|
||||||
|
|
||||||
|
| Alert | When | Severity |
|
||||||
|
|---|---|---|
|
||||||
|
| **App behind its newest release** | A newer release has been out for the number of days set (*Warn when a newer release has been out for … days*; 0, the default, never) and the application has not caught up. The alert says which release, how long, and whether it is a patch, a minor update or a new major version | warning |
|
||||||
|
|
||||||
|
Maintenance on the integration holds the alert. It is under *Updates* on the Alerts page. `/metrics` has `homelab_app_update_available` (1 if a newer release is out) for each application, while the lookup is on.
|
||||||
|
|
||||||
|
## Speedtest
|
||||||
|
|
||||||
|
An integration of type *Speedtest Tracker* ([INTEGRATIONS.md](../INTEGRATIONS.md#speedtest-tracker)) is read every ten minutes, and **Monitoring → Speedtest** shows the newest test, the middle of the last three and charts of download, upload and ping over up to 90 days. This app never starts a test: Speedtest Tracker runs them on its own schedule.
|
||||||
|
|
||||||
|
| Alert | When | Severity |
|
||||||
|
|---|---|---|
|
||||||
|
| **The internet is slow** | The middle of the newest three completed tests is under the share (60% unless you set another) of what you pay for — download and upload each judged only if you gave its number | warning |
|
||||||
|
| **Speed tests failing** | The newest three tests that completed or failed all failed — the internet is down, or Speedtest Tracker cannot reach Ookla | warning |
|
||||||
|
|
||||||
|
The *middle* (the median) of three is judged so that one test that ran while somebody was streaming does not raise an alert. Without *the download you pay for* and *the upload you pay for* (on the integration) nothing is judged slow — a speed alone says nothing — and only failing tests raise an alert. Maintenance on the integration holds the alerts; a reading older than 45 minutes is not believed. Results are kept 90 days.
|
||||||
|
|
||||||
|
## NetBox
|
||||||
|
|
||||||
|
An integration of type *NetBox* ([INTEGRATIONS.md](../INTEGRATIONS.md#netbox)) is read every five minutes, and **Network → NetBox** shows its workers, what it holds and — the other half — **which of your servers are not in it**.
|
||||||
|
|
||||||
|
| Alert | When | Severity |
|
||||||
|
|---|---|---|
|
||||||
|
| **No background worker** | `/api/status/` says no RQ worker is running — webhooks, scripts, jobs and reports are not run, although the pages load | warning |
|
||||||
|
|
||||||
|
The list of servers with no address in NetBox is not an alert: it is the app's servers (from what their agents report) against NetBox's addresses, leaving out Docker bridge addresses and the other ranges excluded on the Consistency page. A server counts as missing only if it has addresses and none is in NetBox.
|
||||||
|
|
||||||
|
## n8n
|
||||||
|
|
||||||
|
An integration of type *n8n* ([INTEGRATIONS.md](../INTEGRATIONS.md#n8n)) is read every five minutes, and **Automation → n8n** lists the workflows with their last run and the failures of the last day.
|
||||||
|
|
||||||
|
| Alert | When | Severity |
|
||||||
|
|---|---|---|
|
||||||
|
| **Workflow failed** | A workflow that is switched on failed (or crashed) on its last run, in the last week. One alert per workflow; it clears when the workflow next succeeds | warning |
|
||||||
|
|
||||||
|
A workflow that is switched off says nothing, and what each workflow last did is judged from n8n's **newest 250 runs**. If n8n gives its workflows but not its runs, the last outcomes are unknown and nothing is claimed. Maintenance on the integration holds the alerts.
|
||||||
|
|
||||||
|
## Immich
|
||||||
|
|
||||||
|
An integration of type *Immich* ([INTEGRATIONS.md](../INTEGRATIONS.md#immich)) is read every five minutes, and **Apps → Immich** shows the photo and video counts, what the library takes, the disk under it and each background job queue with how many failed. Photos cannot be made again, and Immich fails quietly:
|
||||||
|
|
||||||
|
| Alert | When | Severity |
|
||||||
|
|---|---|---|
|
||||||
|
| **Disk under the library filling** | 90 % full | warning; critical from 95 % |
|
||||||
|
| **Background jobs failed** | 10 or more failed jobs, all queues together; the three worst queues are named | warning |
|
||||||
|
|
||||||
|
Failed jobs stay counted until they are run again or cleared in Immich (Administration → Jobs), so that alert stays until you do. A queue with a backlog of *waiting* jobs is not an alert — a big import has one for days. Maintenance on the integration holds them. A reading older than 20 minutes is not believed.
|
||||||
|
|
||||||
|
## Paperless-ngx
|
||||||
|
|
||||||
|
An integration of type *Paperless-ngx* ([INTEGRATIONS.md](../INTEGRATIONS.md#paperless-ngx)) is read every five minutes, and **Apps → Paperless** shows the documents and the inbox, the disk, Paperless's own system status and the imports that failed in the last week.
|
||||||
|
|
||||||
|
| Alert | When | Severity |
|
||||||
|
|---|---|---|
|
||||||
|
| **Database, Redis or the worker in error** | The system status says `ERROR` — a scan dropped in the folder just sits there when the worker is down | critical |
|
||||||
|
| **Index or classifier in error** | The system status says `ERROR` | warning |
|
||||||
|
| **Sanity check found problems** | The system status says `WARNING` for the sanity check | warning |
|
||||||
|
| **Documents failed to import** | One or more unacknowledged failures in the last week; the files and the latest error are named | warning |
|
||||||
|
| **Disk filling** | 90 % full | warning; critical from 95 % |
|
||||||
|
|
||||||
|
A classifier that has not been trained yet reports a *warning* in Paperless and is not shown as a problem. The system status needs a superuser's token; with another, the page says it is unknown and only the failed imports and the counts are judged. Maintenance on the integration holds the alerts.
|
||||||
|
|
||||||
|
## Prometheus
|
||||||
|
|
||||||
|
An integration of type *Prometheus* ([INTEGRATIONS.md](../INTEGRATIONS.md#prometheus)) is read every two minutes, and **Monitoring → Prometheus** shows its targets by job (how many up, how many down), the ones not answering and the alerts its rules have firing. It makes what you already measure
|
||||||
|
there part of the same list, with the same notifications, silencing and incidents as the rest.
|
||||||
|
|
||||||
|
| Alert | When | Severity |
|
||||||
|
|---|---|---|
|
||||||
|
| **Targets not answering** | One or more targets of a job have been down for five minutes. One alert per job, naming up to three instances and the first scrape error | warning |
|
||||||
|
| **Alert firing** | One of Prometheus's rules is firing. One alert per alert name ("DiskFull (3 firing)"), with the first summary | critical if any has `severity="critical"` (or `page`), else warning |
|
||||||
|
|
||||||
|
Pending alerts, `info` / `none` severities and the heartbeats every pipeline carries (`Watchdog`, `DeadMansSwitch`, `InfoInhibitor`) are not shown as problems. Maintenance on the integration holds them. A reading older than ten minutes is not believed. Alerts that
|
||||||
|
Grafana manages itself are not in Prometheus's list; to see one here, write it as a Prometheus rule instead. Nothing from Prometheus is exported back to `/metrics`.
|
||||||
|
|
||||||
|
## Authentik
|
||||||
|
|
||||||
|
An integration of type *Authentik* ([INTEGRATIONS.md](../INTEGRATIONS.md#authentik)) is read every five minutes, and **Apps → Authentik** shows its version, workers, each outpost (and whether it is reporting), the certificates and API tokens that run out — soonest first — and the failed sign-ins of the last day. The certificate alerts can be switched off for one Authentik (the tick box **Do not alert about certificates** on the integration) — useful for one that is not in use at the moment; the certificates are still listed and in `/metrics`, and the other alerts are unchanged.
|
||||||
|
Authentik is the front door to everything behind it, so what is raised early is what would shut it:
|
||||||
|
|
||||||
|
| Alert | When | Severity |
|
||||||
|
|---|---|---|
|
||||||
|
| **Outpost has not reported in** | An outpost that serves applications had no instance report within five minutes | critical |
|
||||||
|
| **Outpost runs another version** | An instance runs a version other than the one Authentik expects | warning |
|
||||||
|
| **Certificate runs out** | Within 14 days (a certificate Authentik renews itself is left out) | warning; critical within 3 days and after |
|
||||||
|
| **API token or app password runs out** | Within 7 days | warning; critical within 2 days. The token this app reads Authentik with counts too |
|
||||||
|
| **No worker connected** | The workers endpoint answers with none | warning |
|
||||||
|
| **Failed sign-ins** | 20 or more in the last day | warning |
|
||||||
|
| **Configuration errors** | 3 or more in the last day, with the newest message | warning |
|
||||||
|
|
||||||
|
Maintenance on the integration holds them. A reading older than 20 minutes is not believed: nothing is raised or cleared from it, and the Alerts page says Authentik could not be checked. A token that does not expire, a verification token (they live for minutes) and an outpost with no applications are never alerts.
|
||||||
|
|
||||||
|
## Jellyfin
|
||||||
|
|
||||||
|
An integration of type *Jellyfin* ([INTEGRATIONS.md](../INTEGRATIONS.md#jellyfin)) is read every two minutes, and **Apps → Jellyfin** shows whether it is up, who is **streaming** right now (and whether that is transcoded, which is what loads the machine), the scheduled tasks that failed, and a chart of the most that were streaming at once.
|
||||||
|
It raises three warnings: **a restart is wanted**, **a scheduled task failed**, **10 or more sign-ins failed in a day** (the last is the one to look at if the server is open to the internet). Maintenance on the integration holds them.
|
||||||
|
|
||||||
|
Its other use is **not cutting off a film**. When the Jellyfin runs on a server the app knows, an update run on that server asks first if people are streaming (the update drawer says who; *Update anyway* goes ahead), and a patch plan waits for them for up to two hours before it updates that server.
|
||||||
|
Only a recent reading counts, so a Jellyfin that cannot be read never holds an update up. The match is by address: the Jellyfin's address must be the server's IP address or host name as the agent reports it (a Jellyfin behind a proxy on another machine is not matched).
|
||||||
|
|
||||||
|
## Docker containers that are broken
|
||||||
|
|
||||||
|
If you use Dockhand, the health pass (every 15 minutes) also reads each container's **state** and **status text** — the same text the Docker page shows — and raises an alert (category *Failing services*, source *Docker*) for:
|
||||||
|
|
||||||
|
| What | When | Severity |
|
||||||
|
|---|---|---|
|
||||||
|
| **Restart loop** | Docker says the container is restarting ("Restarting (127) 39 seconds ago") | critical |
|
||||||
|
| **Unhealthy** | The container's own health check fails ("Up 6 days (unhealthy)") | warning |
|
||||||
|
| **Stopped unexpectedly** | It is exited or dead, and it was **running at the last look**, or it has never been seen and stopped **within the last day** (from "Exited (0) 15 minutes ago") | warning |
|
||||||
|
| **Never started** | It was created and has stayed "Created" for **an hour** | warning |
|
||||||
|
| **Docker host unreachable** | Dockhand could not read one of its environments (a Hawser or Docker agent that is down) | warning |
|
||||||
|
|
||||||
|
The hard part is *stopped*: many containers are stopped on purpose (a stack that is switched off, a one-shot job, a watchtower that ran and exited). So a stop is only news when it was **seen happen** — a container that has been down all along (a stack
|
||||||
|
that exited two weeks ago, one that exited three months ago) is left alone. Once raised, a stop stays raised until the container runs again, is removed, or is left alone. Only what the app has seen is judged, so after switching it on nothing alerts for what was already
|
||||||
|
long stopped. The scanner containers Dockhand starts (`dockhand-grype-*`, `dockhand-trivy-*`) are always left alone.
|
||||||
|
|
||||||
|
(Settings → Notifications → *What to send* shows one group at a time — *Servers and disks*, *Secrets, keys and domains*, *Backups and updates*, *Integrations and automation*, *DNS changes* — and the settings search opens the group a setting is in.)
|
||||||
|
|
||||||
|
**Leaving a container alone:** the *Leave alone* button on the Docker page (administrators; *Watch again* undoes it), or the list under Settings → Notifications → What to send → *Containers to leave alone* — one per line, a name or `host/name`, `*` for anything.
|
||||||
|
Maintenance on the Docker integration holds these alerts like its other ones. If Dockhand itself cannot be read the containers' alerts are held rather than cleared (and the integration is raised as failing, as any other). Switch the whole thing off with *Warn when a
|
||||||
|
Docker container is broken*. What the app has seen of each container is kept between looks (in `settings`, key `_internal:containerMemory`).
|
||||||
|
|
||||||
|
**Critical vulnerabilities.** Dockhand can scan images (Grype or Trivy) — by hand, after a pull, or for a whole host. Every fifteen minutes the app reads **what Dockhand has already stored** (never starting a scan, so it costs Dockhand nothing) and raises a *warning* for each Docker host whose scanned images have one or more **critical** findings, naming the images and how many of the findings have a fix. High, medium and low findings are shown on the Docker page (*Vulnerabilities*, with the critical ones listed: CVE, package, installed and fixed version, image, and the containers that use it) but are never alerted. An image that was never scanned is not known to be clean — the page says how many of a host's images have a scan. A host whose count cannot be read is *unknown*, not clean, and says nothing. A reading older than 45 minutes is not believed: the alert is neither raised nor cleared and the Alerts page says it could not check. Maintenance on the Docker integration holds it, and an alert can be closed or snoozed like any other.
|
||||||
|
**High findings** are never alerted unless you set a number under *Also warn when a Docker host has this many high vulnerabilities* (0, the default, is never): a host whose scanned images have at least that many high findings then gets a warning of its own ("Jackie: 12 high vulnerabilities in the scanned images"), beside the critical one. Scanners find high-severity issues in almost every image, so a low number is mostly noise. **To switch it off:** Settings → Notifications → What to send → *Warn about critical vulnerabilities in Docker images* (on by default; it turns the high ones off too). Dockhand can also send its own vulnerability notifications; with both on you hear twice. `/metrics` has `homelab_docker_vulnerabilities` (by `severity`) and `homelab_docker_images_scanned` for each host.
|
||||||
|
|
||||||
|
## Endpoint checks and heartbeats
|
||||||
|
|
||||||
|
**Endpoint checks** (Monitoring → Endpoints) are for the simple "is it up?" that does not need a monitoring server of its own. Three kinds:
|
||||||
|
|
||||||
|
| Kind | What it asks |
|
||||||
|
|---|---|
|
||||||
|
| `http` | Fetches an `http(s)://` address. Counts as up if the status is in the accepted set (200–399 unless you say otherwise, e.g. `200-299, 301`), and — if you ask — the page contains a keyword, and/or **a value in a JSON answer is within its limit** (see below). Follows up to 5 redirects when it has a keyword or value to look for. |
|
||||||
|
| `tcp` | Opens a connection to a host and port. |
|
||||||
|
| `tls` | Connects and reads the certificate; the answer is the days left. |
|
||||||
|
|
||||||
|
Per check: **interval** (1–1440 minutes, default 5), **timeout** (default 10 s), **failures before an alert** (default 2 in a row — one blip is not an alert), **certificate warning days** (default 14), an
|
||||||
|
optional **server it is linked to** (maintenance on that server silences it, and the alert links to it), and *accept a self-signed certificate*. Each answer — time, result, message — is kept, with hourly and
|
||||||
|
daily summaries for the charts and uptime figures.
|
||||||
|
|
||||||
|
- A check **fails an alert** (critical) after the configured number of failures in a row, and is announced as resolved when it works again. A **certificate** within its warning days is its own alert (`endpoint-cert:…`).
|
||||||
|
- **Checking a value, not just "is it up".** A web check can read a number or a word out of a JSON answer and judge it. Say **where** it is — `power.now`, `state`, `items[2].name`, or for a Prometheus query
|
||||||
|
`data.result[0].value[1]` (`[0]` is the first place in a list; `["a.b"]` for a name with a dot in it) — and **how it should compare**: *is below*, *is at most*, *is above*, *is at least*, *is equal to*, *is anything but*,
|
||||||
|
*contains*, or *is there*. A number given as text (`"42.5"`, which is how Prometheus gives them) counts as a number; equality compares numbers as numbers and anything else as the same text, capital letters
|
||||||
|
included. An optional **unit** (`W`, `%`) is shown after a number. The status is judged first, then the value: a value outside its limit is a failure that says so ("power.now is 5200 W, but should be below 5000 W"),
|
||||||
|
and the alert reads *Meter — power.now is 5200 W, but …* rather than "isn't answering". It is also a failure if the answer isn't JSON, the path isn't in it ("it stops at power"), or the answer is more than 64 KB.
|
||||||
|
The value read is remembered with each result: shown in the *Answer* column, charted under **History…** (as far back as results are kept, 14 days), in `/metrics` as `homelab_endpoint_value` (numbers only) and in the read API.
|
||||||
|
Examples: Home Assistant's `/api/states/sensor.power` (path `state`), a Plex or Frigate status, an inverter's output, a Prometheus query (`/api/v1/query?query=…`).
|
||||||
|
- **A header, for an API that wants a token.** A web check can send one header with its request (usually `Authorization: Bearer …`). The value is stored **encrypted** when `CREDENTIALS_ENCRYPTION_KEY` is set, is never sent back
|
||||||
|
to the browser (the form says *Saved — leave blank to keep it*), and goes only to the address you gave — **not to another host a redirect leads to**.
|
||||||
|
- What an administrator types in cannot be just anything: loopback, link-local and cloud-metadata addresses are refused, and the connection goes to the one address that passed the check. LAN and public hosts are fine.
|
||||||
|
- **Apps in your containers.** With Dockhand connected, an image the app recognises (Immich, Paperless-ngx, Nextcloud, Vaultwarden, Authentik, Gitea, Grafana, Prometheus, Jellyfin, n8n, ntfy, Gotify, Nginx Proxy Manager, Open WebUI, Ollama, SearXNG, Wiki.js,
|
||||||
|
BookStack, Homebox, Semaphore, Meilisearch, and a few dozen with just a web page) is offered as a check on **that app's own health address**, with what a good answer looks like set up as a [value check](#endpoint-checks-and-heartbeats): Immich's
|
||||||
|
`/api/server/ping` must say `res` = `pong`, Nextcloud's `/status.php` `maintenance` = `false`, Gitea's `/api/healthz` `status` = `pass`, Grafana's `/api/health` `database` = `ok`, Prometheus's `/-/healthy` must contain "Healthy". It is asked on the Docker
|
||||||
|
host's address — the server the app knows under the Docker environment's name in Dockhand, first usable IPv4 — at the **port the container publishes** (from Dockhand; if Dockhand does not say, the one the host's agent sees listening that the app is known to use).
|
||||||
|
Only running containers, only hosts the app knows, and only what no check covers already is offered; where an app is recognised the plain "something answers on 8080" suggestion for that port is replaced by it. The paths are those the projects publish for the purpose, **not tried against live
|
||||||
|
instances here**: a wrong one is a failing check you can edit. Uptime Kuma's monitors are not looked at, so something you already watch there may be offered — *Not for me* stops it.
|
||||||
|
- The app suggests checks it can already work out (a web port a server's agent sees open, a secret with a host) under *Endpoints → Suggested*, and proxied hosts Nginx Proxy Manager serves.
|
||||||
|
|
||||||
|
**Heartbeats** (Monitoring → Heartbeats) are the other direction: a cron job or script **calls an address when it finishes**, and the app alerts when it does *not*. A heartbeat has a **period** and a **grace**; it is
|
||||||
|
**late** when both have passed since the last call (a *warning*), and a call to `/fail` is **critical** at once. Details and examples: [HEARTBEATS-AND-CALENDAR.md](HEARTBEATS-AND-CALENDAR.md).
|
||||||
|
|
||||||
|
Both are judged **every minute**, with the same remember-and-compare bookkeeping as the other checks.
|
||||||
|
|
||||||
|
## Integrations
|
||||||
|
|
||||||
|
Integrations (Proxmox, Synology, UniFi, Semaphore, Gitea, Docker, Tailscale, Uptime Kuma, osTicket, phpIPAM, Proxmox Backup, Nginx Proxy Manager, Home Assistant, and the DNS providers) are **read** by the app,
|
||||||
|
using credentials an administrator enters under *Integrations*; what each needs is in [INTEGRATIONS.md](../INTEGRATIONS.md). Every call is recorded in the **Diagnostic Log**.
|
||||||
|
|
||||||
|
- **An integration is "failing"** when its last **3 calls in a row** (Settings → Notifications) failed. Failures are counted per *kind* of service (all Proxmox integrations share a streak), in memory, so a
|
||||||
|
restart starts the count again. A *down* notification goes out at the threshold and a *recovered* one on the next success. The moments are written to the audit log, and they feed the uptime record on the
|
||||||
|
integration's page.
|
||||||
|
- **When an integration cannot be read, what it reported is held, not cleared.** A Proxmox node that cannot be read does not make its storage alerts disappear (or appear); the Alerts page says it could not check.
|
||||||
|
The synced ones (UniFi, Home Assistant) are only judged while their last good read is recent (**20 minutes**; a UPS, **5**).
|
||||||
|
- **An integration can explain other alerts.** A service that stops answering because the integration it needs is failing is folded under it ([Dependencies](#dependencies-one-outage-one-alert)).
|
||||||
|
|
||||||
|
## Forecasts
|
||||||
|
|
||||||
|
A forecast says "this will be full in N days" from a **straight line through the recent history**, and is deliberately cautious:
|
||||||
|
|
||||||
|
- It needs **at least six readings covering at least three days**, and a **steady** climb — the line has to explain at least 90% of the movement (R² ≥ 0.9). A big copy that then gets deleted, or a disk that
|
||||||
|
wobbles, is **not forecast**. A slope under 0.05 points a day counts as flat.
|
||||||
|
- It looks at the **last two weeks**; if those are flat it looks at the **last ninety days**, so a slow, steady climb is seen too.
|
||||||
|
- A disk that is **already over its limit** is left to the ordinary "nearly full" alert.
|
||||||
|
- It is shown as an alert when it falls **within the window** (Settings → Notifications → *Warn when a disk will fill within*, 14 days by default; per server; 0 turns it off), as a line in the weekly digest, and as
|
||||||
|
`homelab_server_disk_days_to_full` / `homelab_storage_days_to_full` in [`/metrics`](PROMETHEUS-GRAFANA.md). It is **not** sent as a notification the moment it appears.
|
||||||
|
- **Servers:** from the agents' history. **Proxmox storages and Synology volumes:** the app writes down what each health pass reads, one reading an hour, kept 400 days, and forecasts those the same way; it is
|
||||||
|
shown beside each storage on the Proxmox page and each volume on the Synology page. Until there are a few days of readings the page says it is collecting history.
|
||||||
|
|
||||||
|
## Dependencies: one outage, one alert
|
||||||
|
|
||||||
|
If a Proxmox host goes down, its fifteen guests go quiet too. Fifteen alerts about the symptom would bury the one about the cause. The app keeps a **graph of what needs what**:
|
||||||
|
|
||||||
|
- a server **runs on** a host (set by hand, or worked out from its Proxmox node);
|
||||||
|
- a UniFi device is **connected through** another, and a **server is plugged into** a switch or access point (matched by IP address);
|
||||||
|
- **dependencies you write** (*Map → Dependencies*: "Gitea needs the server it is installed on", "the NAS needs the rack switch"), and the app offers suggestions it can work out.
|
||||||
|
|
||||||
|
When something is down and what depends on it is down too, the dependants' alerts are **folded under** the first: shown nested, marked *caused by …*, not announced on their own, and the host's message says
|
||||||
|
"— also takes down web and db". If the host comes back and a dependant **stays** down, it is announced then, as new: **an outage is never swallowed.**
|
||||||
|
|
||||||
|
## What happens to an alert
|
||||||
|
|
||||||
|
Several things can quiet an alert. They are different, and it matters which you use:
|
||||||
|
|
||||||
|
| | Promise about the thing? | Hides it? | Notifications | Ends |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| **Acknowledge** ("I'm on it") | No — it says someone has seen it, why, who has it | No: still listed, still counted in `/metrics`; leaves the *needs attention* count | Stops [escalation](#escalation) | When you choose (an hour … 30 days), or when the alert has been gone half an hour |
|
||||||
|
| **Snooze** ("remind me tomorrow 09:00") | No | Yes, held back like maintenance | Raise/clear messages held; **one reminder** at the end if it is still there | At the time you set (up to 90 days), or earlier with *Wake now* |
|
||||||
|
| **Close** ("dealt with, or accepted") | Yes — with who and why, and the reason is written on the incident | Yes, held back like a snooze | Raise/clear messages held; **one message** only if it **gets worse** | Never by time: when it gets more serious than it was, when it has been gone half an hour, or with *Open again* |
|
||||||
|
| **Maintenance window** | Yes — "I'm working on this" (5 minutes to a week) | Yes, marked *silenced* | Held for the server, integration, DNS provider or device | At its end; what is still there is then announced as **new** |
|
||||||
|
| **Folded under a host** | — automatic | Nested under its cause | Not announced on its own | When the cause ends |
|
||||||
|
| **Flapping** | — automatic | No | Raise and clear messages held; one "flapping" message, one "settled" message | After an hour of calm |
|
||||||
|
|
||||||
|
### Who hears about it
|
||||||
|
|
||||||
|
Notifications go to the channels that are on (**Gotify, ntfy, Telegram, Pushover, email, a webhook, push to phones and browsers**), narrowed by **routing rules** (first match wins: by kind, by severity, by the server's tags) and each
|
||||||
|
browser's own wishes. **Quiet hours** hold messages and send one digest at the end of the window (critical alerts can be let through); **Away** mode does the same for a stretch you set, "until Sunday". Every message the app can send, and what toggles it, is in [NOTIFICATIONS.md](../NOTIFICATIONS.md).
|
||||||
|
**Event webhooks** are a separate way of being told, for machines: they hear every raise and clear, flapping and snoozed included ([WEBHOOKS.md](WEBHOOKS.md)).
|
||||||
|
|
||||||
|
### Flapping
|
||||||
|
|
||||||
|
An alert **raised three times within an hour** (Settings → Notifications; 0 = off) is called *flapping*: one message says so, its further raise and clear messages are held, and it is marked on the page. After
|
||||||
|
an hour with no raise or clear, one *settled* message says whether it is still there. The alert itself is untouched: still raised, still counted. *Alerts → Noise* lists what keeps coming back.
|
||||||
|
|
||||||
|
### Escalation
|
||||||
|
|
||||||
|
An alert raised for a while with **nobody on it** can be told again in up to three steps ("after 30 minutes: push; after 2 hours: email the neighbour who has the key"), with its own channels and an optional extra
|
||||||
|
email address. An acknowledgement stops the rest; problems under maintenance, folded under a host or only informational never escalate. Looked at every five minutes.
|
||||||
|
|
||||||
|
### Maintenance
|
||||||
|
|
||||||
|
*Maintenance* (a window of 5 minutes to a week, started by hand, in bulk, from a **recurring schedule**, or mirrored from Uptime Kuma) silences what is about the target: a server's offline/disk/service alerts and
|
||||||
|
its endpoint checks and heartbeats; an integration's storage, backup and automation alerts; "integration down" for that service type; a UniFi device's offline alert. What depends on it can be put under
|
||||||
|
maintenance with it. A silenced problem is **not remembered as known**, so one still there when the window ends **alerts then, as new**.
|
||||||
|
|
||||||
|
## Incidents
|
||||||
|
|
||||||
|
Every time an alert was raised and later went away is an **incident**: when it opened and closed, how bad it got, what it was about, what explained it, who acknowledged or was handed it, and — on its
|
||||||
|
timeline — what was done to the thing in the hour around it (maintenance, a restart, a settings change). People can add notes. The recorder looks at the alerts **every five minutes** whether or not anyone has
|
||||||
|
the page open; the same alert coming back is a **new** incident. Resolved incidents are kept **400 days**. They are on **Alerts → History**, in the read API (`/api/v1/incidents`) and in the
|
||||||
|
monthly report.
|
||||||
|
|
||||||
|
### What changed before an incident
|
||||||
|
|
||||||
|
Each incident page has a **What changed before this?** card (and `GET /api/v1/incidents/:id/before`): the things the app knows happened in the 30 minutes, 2, 6 or 24 hours before the alert began,
|
||||||
|
ranked. The sources are the audit log (settings, DNS, dependencies, integrations, a server being changed, a neighbour going offline or filling up), maintenance windows that began or ended, update runs and patch
|
||||||
|
plans, other alerts raised first, and a step in the server's own figures (a disk that jumped, memory that fell away).
|
||||||
|
|
||||||
|
Each is scored from three things:
|
||||||
|
|
||||||
|
| | How it counts |
|
||||||
|
|---|---|
|
||||||
|
| **How related** | The **same thing** the incident is about counts most; then **something it needs** (the host it runs on, the switch it is plugged into, a dependency you wrote — the nearest count more); then a **change that touches everything** (a setting, DNS); then **something that needs it**; and last something with **no link** the app knows of. |
|
||||||
|
| **What kind** | A maintenance window ending or a host going down is likelier than a change of settings, which is likelier than a note being written. Logins, notes, runbooks, tags and the like are not listed at all. A setting that bears on this kind of alert (thresholds, routing) counts for more. |
|
||||||
|
| **How close** | A change five minutes before counts for much more than one two hours before — the weight falls off over about an hour and never reaches zero inside the window. |
|
||||||
|
|
||||||
|
They are shown as *Likely* (55 and over), *Possible* (25 and over) and *Background* (the rest, behind a button); under 8 is left out, and at most ten are shown. The audit entry "pve1 stopped reporting"
|
||||||
|
and the alert about it are one event and shown once. It is a **list of suspects, not a diagnosis**: it says what happened and when, not why, and a change made on the machine itself — a package installed by
|
||||||
|
hand, a config file edited — is not something the app can see.
|
||||||
|
|
||||||
|
## History and how long it is kept
|
||||||
|
|
||||||
|
| What | Kept | Grain |
|
||||||
|
|---|---|---|
|
||||||
|
| A server's load, memory and each disk | **35 days** of samples (one every 10–15 minutes), **120 days** of hourly summaries, **~800 days** of daily summaries | summaries carry the lowest and highest |
|
||||||
|
| Endpoint check answers | **14 days** of every answer, hourly summaries for **120 days**, daily summaries for years | |
|
||||||
|
| Uptime of servers and integrations | The last weeks, worked out from the moments each stopped and resumed (in the audit log) | one figure a day |
|
||||||
|
| Proxmox storage and Synology volume use | **400 days**, one reading an hour | |
|
||||||
|
| Home Assistant watched numbers | **35 days**, one reading every few minutes | |
|
||||||
|
| A UPS's charge, load and mains voltage | **35 days**, one reading every five minutes (every minute on battery) | |
|
||||||
|
| Incidents | **400 days** after they resolve | |
|
||||||
|
| Messages sent to event webhooks | The last **100 per webhook**, none older than **14 days** — see [WEBHOOKS.md](WEBHOOKS.md#the-delivery-log-and-sending-again) | |
|
||||||
|
| The audit log and the Diagnostic Log | Until automatic purging is switched on under *Settings → Data* (then 90 days by default, configurable) | |
|
||||||
|
|
||||||
|
A chart asks for 24 hours, 7, 30, 90 days or a year and is drawn from whichever of these is fine enough and still there.
|
||||||
|
|
||||||
|
**Is it being trimmed?** *Settings → Data → Storage* measures the database, its backups and sessions, and every table by the room it takes, and for each history says how far back it reaches against how long it is meant to be kept:
|
||||||
|
*trimmed* (at its limit), *filling up* (not yet old enough to be trimmed), *not trimmed* (it holds far more than it should, so the trimming is not running or has not caught up) or *no limit* (the logs, until purging is
|
||||||
|
switched on). The last two are listed at the top as needing a look. Nothing there changes anything.
|
||||||
|
|
||||||
|
## Where to see it
|
||||||
|
|
||||||
|
| | Shows |
|
||||||
|
|---|---|
|
||||||
|
| **Dashboard**, **My work**, the menu badge | What needs attention now; what is yours |
|
||||||
|
| **Alerts** (Now · History · Noise) | Every alert, with acknowledge, snooze, silence; the incident record; what keeps coming back |
|
||||||
|
| A **server's page** | Its figures, history, health, ports, tasks, notes, jobs, what it runs on and what runs on it |
|
||||||
|
| **Endpoints**, **Heartbeats**, **Proxmox**, **Synology**, **Network devices**, **Home Assistant**, **UPS**, **Proxy hosts** | Each source's own view |
|
||||||
|
| **Service levels** | Each target and what is left of this month's error budget |
|
||||||
|
| **Weekly digest** and **monthly, quarterly and annual reports** | The week, the month, the quarter and the year in one message |
|
||||||
|
| **`/metrics`** → Prometheus, Grafana | [PROMETHEUS-GRAFANA.md](PROMETHEUS-GRAFANA.md) |
|
||||||
|
| **Read API** (`/api/v1`) | [API.md](API.md) |
|
||||||
|
| **Event webhooks** | [WEBHOOKS.md](WEBHOOKS.md) |
|
||||||
|
| **Status page**, **wall display**, **calendar feed** | Read-only views for others; made under *Settings → Share links* |
|
||||||
|
|
||||||
|
## Tuning, in questions and answers
|
||||||
|
|
||||||
|
**I am about to reboot a server.** Start maintenance on it (Maintenance page, or *Silence* on an alert): its alerts are held and, if it is still down at the end, announced as new.
|
||||||
|
|
||||||
|
**An alert is noisy because it keeps coming and going.** Let it flap (it will be damped after three raises an hour), then look at *Alerts → Noise* — it says what usually helps: a longer grace, a different limit.
|
||||||
|
For your own rules, raise the limit or lengthen the time.
|
||||||
|
|
||||||
|
**One server legitimately runs at 92% disk.** Give that server (or that one disk) its own limit — or *never alert about this disk* — in the server's *Settings* section.
|
||||||
|
|
||||||
|
**I want to be reminded about something later, not silence it.** Snooze it.
|
||||||
|
|
||||||
|
**I have dealt with it, or I accept it, and I do not want to be reminded.** Close it (*Close…* on the Alerts page, with a reason). It stays closed until it gets worse than it was, or the problem goes away for half an hour and returns.
|
||||||
|
|
||||||
|
**I do not want to be woken for warnings.** Add a routing rule: kind or severity → the channels you want (a rule with no channels tells nobody; the Alerts page still shows it), and set quiet hours.
|
||||||
|
|
||||||
|
**A job on a server has to run every night.** Make a heartbeat for it and call it from the end of the script.
|
||||||
|
|
||||||
|
**I want my own limit, such as "memory above 85% for an hour".** Add an alert rule.
|
||||||
|
|
||||||
|
**An integration's alerts disappeared.** Check *Couldn't check everything* on the Alerts page: the integration could not be read, so its alerts are held. The Diagnostic Log says why.
|
||||||
|
|
||||||
|
**The app restarted and nothing about servers is alerting.** For 20 minutes after a restart servers are not judged, so the agents can report first.
|
||||||
|
|
||||||
|
**I want these figures in my own monitoring.** `/metrics` for Prometheus, the read API for scripts, event webhooks to be told.
|
||||||
|
|
||||||
|
## What monitoring does not do
|
||||||
|
|
||||||
|
- **It does not fix anything.** The agents only push, and nothing an alert says is done automatically. (Updates and playbooks can be run from the app by an operator, on purpose.)
|
||||||
|
- **It does not see inside a server that has no agent** — only what an endpoint check, a heartbeat or an integration can say about it.
|
||||||
|
- **It is not a replacement for a second pair of eyes on the app itself.** If Homelab Manager is down, nothing here speaks. Watch it from outside: a Prometheus rule on `up{job="homelab-manager"}`
|
||||||
|
([PROMETHEUS-GRAFANA.md](PROMETHEUS-GRAFANA.md#alerting-rules-to-start-from)), or a check from another tool against `GET /health`.
|
||||||
|
- **It is as fresh as its sources.** A server's figures are as old as its last report (up to 15 minutes), a forecast needs days of history, and an integration's state is the last good read.
|
||||||
@@ -0,0 +1,401 @@
|
|||||||
|
# Prometheus and Grafana
|
||||||
|
|
||||||
|
Homelab Manager can be scraped by **Prometheus** at `/metrics`, and ships a ready-made **Grafana dashboard** built on those series. This page sets up both, from nothing, and lists every series.
|
||||||
|
|
||||||
|
- [How it fits together](#how-it-fits-together)
|
||||||
|
- [1. Make a metrics token](#1-make-a-metrics-token)
|
||||||
|
- [2. Check it by hand](#2-check-it-by-hand)
|
||||||
|
- [3. Set up Prometheus](#3-set-up-prometheus)
|
||||||
|
- [4. Set up Grafana](#4-set-up-grafana)
|
||||||
|
- [A complete example with Docker Compose](#a-complete-example-with-docker-compose)
|
||||||
|
- [Alerting rules to start from](#alerting-rules-to-start-from)
|
||||||
|
- [Every series](#every-series)
|
||||||
|
- [Useful queries](#useful-queries)
|
||||||
|
- [Troubleshooting](#troubleshooting)
|
||||||
|
|
||||||
|
## How it fits together
|
||||||
|
|
||||||
|
```
|
||||||
|
agents, checks, integrations ──► Homelab Manager ──/metrics──► Prometheus ──► Grafana
|
||||||
|
(reads its own database; (stores the history,
|
||||||
|
calls nothing outside) draws the dashboard)
|
||||||
|
```
|
||||||
|
|
||||||
|
- `/metrics` is a **view of what the app already knows**. A scrape reads the database and the app's last alert result (reused for up to a minute); it **never calls out** — not to Proxmox, not to a server, not
|
||||||
|
to an integration. Scraping every 15 seconds costs a few database reads.
|
||||||
|
- The app keeps its own history (a year of server load and disks, endpoint answer times, uptime), so Prometheus is **not needed** to see trends in the app itself. Use Prometheus and Grafana when you
|
||||||
|
want these figures next to the rest of your monitoring, longer retention, or your own alert rules in Alertmanager.
|
||||||
|
- **Every series is a gauge.** Labels are escaped as Prometheus requires.
|
||||||
|
|
||||||
|
## 1. Make a metrics token
|
||||||
|
|
||||||
|
Administrators: **Settings → API and webhooks → Metrics for Prometheus → Make a token**.
|
||||||
|
|
||||||
|
- The token (it starts with `hlm_m_`) is **shown once**; only its SHA-256 is kept.
|
||||||
|
- Up to **ten** can exist — make one per Prometheus, so one can be removed without touching the others. A removed token **stops working at once**.
|
||||||
|
- Each shows when it was last used, so an unused one is easy to find.
|
||||||
|
- The token goes in the **`Authorization: Bearer …` header only**. A token in the address (`?token=…`) is **not accepted** (addresses end up in logs).
|
||||||
|
- No token, a wrong one and a removed one all get the same `401`. Requests are limited to **120 a minute** per address (`429` beyond).
|
||||||
|
- The answer is Prometheus's text format: `Content-Type: text/plain; version=0.0.4; charset=utf-8`.
|
||||||
|
|
||||||
|
A metrics token can read `/metrics` and nothing else. It is a different thing from the *personal API tokens* used for the [read API](API.md), and neither can sign in.
|
||||||
|
|
||||||
|
## 2. Check it by hand
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -fsS -H "Authorization: Bearer hlm_m_xxxxxxxxxxxxxxxxxxxxxxxx" https://hm.example.lan/metrics | head -30
|
||||||
|
```
|
||||||
|
|
||||||
|
You should see something like this (trimmed; the `# HELP` lines are left out here):
|
||||||
|
|
||||||
|
```
|
||||||
|
# TYPE homelab_info gauge
|
||||||
|
homelab_info{environment=""} 1
|
||||||
|
# TYPE homelab_server_up gauge
|
||||||
|
homelab_server_up{server="wiki-host"} 1
|
||||||
|
# TYPE homelab_server_last_seen_timestamp_seconds gauge
|
||||||
|
homelab_server_last_seen_timestamp_seconds{server="wiki-host"} 1792532670
|
||||||
|
# TYPE homelab_storage_days_to_full gauge
|
||||||
|
homelab_storage_days_to_full{pool="Lab PVE / pve1: storage \"tank\""} 21
|
||||||
|
# TYPE homelab_alerts gauge
|
||||||
|
homelab_alerts{severity="critical"} 1
|
||||||
|
homelab_alerts{severity="warning"} 3
|
||||||
|
homelab_alerts{severity="info"} 0
|
||||||
|
```
|
||||||
|
|
||||||
|
Without the header you get `401` — that is the right answer:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -i https://hm.example.lan/metrics # HTTP/1.1 401 Unauthorized, WWW-Authenticate: Bearer realm="metrics"
|
||||||
|
```
|
||||||
|
|
||||||
|
## 3. Set up Prometheus
|
||||||
|
|
||||||
|
Put the token in a file of its own, readable only by Prometheus, with **no trailing newline**:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
printf '%s' 'hlm_m_xxxxxxxxxxxxxxxxxxxxxxxx' > /etc/prometheus/homelab-manager.token
|
||||||
|
chmod 600 /etc/prometheus/homelab-manager.token
|
||||||
|
```
|
||||||
|
|
||||||
|
`prometheus.yml`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
global:
|
||||||
|
scrape_interval: 30s
|
||||||
|
|
||||||
|
scrape_configs:
|
||||||
|
- job_name: homelab-manager
|
||||||
|
scheme: https
|
||||||
|
metrics_path: /metrics
|
||||||
|
authorization:
|
||||||
|
type: Bearer
|
||||||
|
credentials_file: /etc/prometheus/homelab-manager.token
|
||||||
|
static_configs:
|
||||||
|
- targets: ["hm.example.lan"] # host[:port] of APP_BASE_URL, without https://
|
||||||
|
# If the certificate comes from your own CA, point at it (rather than skipping verification):
|
||||||
|
# tls_config:
|
||||||
|
# ca_file: /etc/prometheus/my-ca.pem
|
||||||
|
```
|
||||||
|
|
||||||
|
- **30–60 seconds is plenty.** The app's own data changes at the pace of the agents (every 15 minutes or so) and the checks (minutes).
|
||||||
|
- Prometheus accepts the token inline too (`credentials: hlm_m_…`), but a file keeps it out of the configuration you share.
|
||||||
|
- Behind a reverse proxy, use the address the proxy serves (`scheme: https`, port 443). Reaching the container directly is `scheme: http`, target `host:3000` — fine on a trusted network.
|
||||||
|
- Reload Prometheus (`kill -HUP`, or `POST /-/reload` if enabled) and open **Status → Targets**: `homelab-manager` should be **UP**.
|
||||||
|
- **Several installations** (production and a dev copy): the series `homelab_info{environment="…"}` carries the name set by `ENVIRONMENT_NAME`, so you can tell them apart; add an `instance`/`env` label in the scrape
|
||||||
|
config as well.
|
||||||
|
|
||||||
|
## 4. Set up Grafana
|
||||||
|
|
||||||
|
1. **Add Prometheus as a data source** — *Connections → Data sources → Add → Prometheus*, URL e.g. `http://prometheus:9090`, *Save & test*.
|
||||||
|
2. **Download the dashboard** — in Homelab Manager: **Settings → API and webhooks → Download the dashboard** (administrators). It is a file, `homelab-manager-dashboard.json`. (The same file is served to a signed-in
|
||||||
|
administrator at `GET /api/metrics-tokens/grafana-dashboard.json`.)
|
||||||
|
3. **Import it** — in Grafana: **Dashboards → New → Import**, upload the file, and **choose your Prometheus** when asked. The file never names a data source, so it is not tied to any installation.
|
||||||
|
|
||||||
|
The dashboard has a **Server** picker (filled from `homelab_server_up`) and five rows:
|
||||||
|
|
||||||
|
| Row | What it has |
|
||||||
|
|---|---|
|
||||||
|
| **Needs attention** | Critical alerts, warnings, acknowledged, open incidents, servers down and up, integrations failing, maintenance, stale and failed backups, failed services, updates waiting, restarts needed; alerts over time; each integration's state. |
|
||||||
|
| **Servers** | One table (up, CPU, memory, updates, security updates, failed services, agent behind), a tile per server for how long it has been up since it last started (for servers whose agent reports it), CPU and memory over time, 30-day uptime, time since each agent last reported. |
|
||||||
|
| **Disks** | Usage with a 90% line, disks filling up (days to full), SMART health, SSD life used, disk temperature, ZFS pools. |
|
||||||
|
| **Backups** | Guests by backup state, the age of each newest backup, and the age, last result and count of Homelab Manager's own backups. |
|
||||||
|
| **Secrets, power and the app itself** | Secrets and certificates by days left, the inventory's power draw, how long `/metrics` takes to build. |
|
||||||
|
|
||||||
|
The dashboard is **built by the app from the same code that serves `/metrics`**, and a test checks that it only asks for series that are really served. So **after an upgrade that adds series, download it again**
|
||||||
|
to get panels for them. It was written for Grafana 9 and later and has been checked for structure (every panel on the grid, every query well formed, every series real), **but it has not been opened in a Grafana
|
||||||
|
by the author** — if a panel looks wrong, the series table below says what each query should be.
|
||||||
|
|
||||||
|
## A complete example with Docker Compose
|
||||||
|
|
||||||
|
Prometheus and Grafana side by side, on the same host as Homelab Manager or elsewhere. Files in one folder:
|
||||||
|
|
||||||
|
```
|
||||||
|
monitoring/
|
||||||
|
├─ docker-compose.yml
|
||||||
|
├─ prometheus.yml (as above, with credentials_file: /etc/prometheus/homelab-manager.token)
|
||||||
|
├─ homelab-manager.token (the token only, no newline)
|
||||||
|
└─ rules/
|
||||||
|
└─ homelab-manager.yml (the alerting rules below, optional)
|
||||||
|
```
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# docker-compose.yml
|
||||||
|
services:
|
||||||
|
prometheus:
|
||||||
|
image: prom/prometheus:latest
|
||||||
|
restart: unless-stopped
|
||||||
|
command:
|
||||||
|
- --config.file=/etc/prometheus/prometheus.yml
|
||||||
|
- --storage.tsdb.retention.time=400d # a year and a bit: the dashboard's long ranges are then real
|
||||||
|
- --web.enable-lifecycle
|
||||||
|
volumes:
|
||||||
|
- ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||||
|
- ./homelab-manager.token:/etc/prometheus/homelab-manager.token:ro
|
||||||
|
- ./rules:/etc/prometheus/rules:ro
|
||||||
|
- prometheus-data:/prometheus
|
||||||
|
ports: ["9090:9090"]
|
||||||
|
|
||||||
|
grafana:
|
||||||
|
image: grafana/grafana:latest
|
||||||
|
restart: unless-stopped
|
||||||
|
volumes:
|
||||||
|
- grafana-data:/var/lib/grafana
|
||||||
|
ports: ["3001:3000"]
|
||||||
|
depends_on: [prometheus]
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
prometheus-data:
|
||||||
|
grafana-data:
|
||||||
|
```
|
||||||
|
|
||||||
|
With the rules, add to `prometheus.yml`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
rule_files:
|
||||||
|
- /etc/prometheus/rules/*.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
In Grafana the data source URL is then `http://prometheus:9090` (the service name, since they share a compose network).
|
||||||
|
|
||||||
|
## Alerting rules to start from
|
||||||
|
|
||||||
|
Homelab Manager already sends its own notifications (see [MONITORING.md](MONITORING.md)). **Use these in addition only if you want the same things in Alertmanager**, otherwise you will be told twice.
|
||||||
|
They are the ones worth having *outside* the app — above all, the app itself going quiet.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# rules/homelab-manager.yml
|
||||||
|
groups:
|
||||||
|
- name: homelab-manager
|
||||||
|
rules:
|
||||||
|
# The one thing the app cannot tell you about itself.
|
||||||
|
- alert: HomelabManagerDown
|
||||||
|
expr: up{job="homelab-manager"} == 0
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical }
|
||||||
|
annotations: { summary: "Homelab Manager cannot be scraped" }
|
||||||
|
|
||||||
|
- alert: HomelabCriticalAlerts
|
||||||
|
expr: homelab_alerts{severity="critical"} > 0
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical }
|
||||||
|
annotations: { summary: "{{ $value }} critical alert(s) in Homelab Manager" }
|
||||||
|
|
||||||
|
- alert: HomelabServerDown
|
||||||
|
expr: homelab_server_up == 0
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical }
|
||||||
|
annotations: { summary: "{{ $labels.server }} has stopped reporting" }
|
||||||
|
|
||||||
|
- alert: HomelabDiskFilling
|
||||||
|
expr: homelab_server_disk_days_to_full < 7 or homelab_storage_days_to_full < 7
|
||||||
|
for: 1h
|
||||||
|
labels: { severity: warning }
|
||||||
|
annotations: { summary: "A disk or storage will be full within a week ({{ $labels.mount }}{{ $labels.pool }})" }
|
||||||
|
|
||||||
|
- alert: HomelabDiskUsageHigh
|
||||||
|
expr: homelab_server_disk_used_percent > 90
|
||||||
|
for: 15m
|
||||||
|
labels: { severity: warning }
|
||||||
|
annotations: { summary: "{{ $labels.server }}: {{ $labels.mount }} is {{ $value | printf \"%.0f\" }}% full" }
|
||||||
|
|
||||||
|
- alert: HomelabSelfBackupStale
|
||||||
|
expr: time() - homelab_selfbackup_last_success_timestamp_seconds > 2 * 86400
|
||||||
|
labels: { severity: warning }
|
||||||
|
annotations: { summary: "Homelab Manager's own backup is more than two days old" }
|
||||||
|
|
||||||
|
- alert: HomelabEndpointDown
|
||||||
|
expr: homelab_endpoint_up == 0
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: warning }
|
||||||
|
annotations: { summary: "Endpoint check {{ $labels.name }} is failing" }
|
||||||
|
|
||||||
|
- alert: HomelabCertificateExpiring
|
||||||
|
expr: homelab_endpoint_certificate_days_left < 14 or homelab_secret_expiry_days < 14
|
||||||
|
labels: { severity: warning }
|
||||||
|
annotations: { summary: "A certificate or secret expires in under two weeks ({{ $labels.name }})" }
|
||||||
|
|
||||||
|
- alert: HomelabHeartbeatMissed
|
||||||
|
expr: homelab_heartbeat_up == 0
|
||||||
|
labels: { severity: warning }
|
||||||
|
annotations: { summary: "Job {{ $labels.name }} has not reported in time, or failed" }
|
||||||
|
|
||||||
|
- alert: HomelabIntegrationFailing
|
||||||
|
expr: homelab_integration_up == 0
|
||||||
|
for: 10m
|
||||||
|
labels: { severity: warning }
|
||||||
|
annotations: { summary: "Integration {{ $labels.integration }} ({{ $labels.type }}) keeps failing" }
|
||||||
|
```
|
||||||
|
|
||||||
|
A series that is **left out** is not zero: `homelab_integration_up`, `homelab_endpoint_up` and `homelab_selfbackup_last_attempt_ok` do not exist until something has been tried. That is why the rules above
|
||||||
|
compare against a condition rather than use `absent()`.
|
||||||
|
|
||||||
|
## Every series
|
||||||
|
|
||||||
|
All gauges. The same table is checked by a test against what `/metrics` really serves.
|
||||||
|
|
||||||
|
| Series | Labels | Meaning |
|
||||||
|
|---|---|---|
|
||||||
|
| `homelab_info` | `environment` | Always 1; says which installation this is. |
|
||||||
|
| `homelab_server_up` | `server` | 1 if the agent has reported within the offline threshold (Settings → Notifications, or the server's own limit if it has one), else 0. Servers that never reported are left out. |
|
||||||
|
| `homelab_server_last_seen_timestamp_seconds` | `server` | When the agent last reported. |
|
||||||
|
| `homelab_server_boot_time_seconds` | `server` | When the server last started, as a Unix time, worked out from the uptime its agent reports (Linux agent 8, Windows agent 7 and later). How long it has been up is `time() - homelab_server_boot_time_seconds`; the value only moves when the server reboots. |
|
||||||
|
| `homelab_server_cpu_load_percent` | `server` | CPU load as the agent reports it. |
|
||||||
|
| `homelab_server_memory_used_percent` | `server` | Memory in use. |
|
||||||
|
| `homelab_server_disk_used_percent` | `server`, `mount` | How full each disk is. |
|
||||||
|
| `homelab_server_uptime_ratio_30d` | `server` | Share of the last 30 days the server was up, 0–1 (from the uptime record). |
|
||||||
|
| `homelab_server_pending_updates` | `server` | Updates waiting to be installed, when the agent reports them. |
|
||||||
|
| `homelab_server_pending_security_updates` | `server` | Of those, the security fixes, when the package manager can say. |
|
||||||
|
| `homelab_server_reboot_required` | `server` | 1 if a restart is due. |
|
||||||
|
| `homelab_server_agent_outdated` | `server` | 1 if the agent is older than the one this installation would install now. |
|
||||||
|
| `homelab_server_failed_units` | `server` | Services (systemd units) that have failed, when a Linux agent reports them. |
|
||||||
|
| `homelab_server_disk_healthy` | `server`, `device` | 1 if the disk's own SMART self-assessment passes, 0 if it fails. |
|
||||||
|
| `homelab_server_disk_wear_percent` | `server`, `device` | How much of an SSD's rated life is used, where the disk says. |
|
||||||
|
| `homelab_server_disk_temperature_celsius` | `server`, `device` | A disk's temperature. |
|
||||||
|
| `homelab_server_zfs_pool_online` | `server`, `pool` | 1 if a ZFS pool (or Windows Storage Spaces pool) is ONLINE, 0 in any other state. |
|
||||||
|
| `homelab_server_disk_days_to_full` | `server`, `mount` | Days until the disk is full at its present rate of filling; only for disks with a steady climb over the last days (left out for the rest). |
|
||||||
|
| `homelab_storage_days_to_full` | `pool` | Days until a Proxmox storage or Synology volume is full at its present rate of filling; only for those with a steady climb. |
|
||||||
|
| `homelab_alerts` | `severity` | Active alerts: `critical`, `warning`, `info` (not counting silenced ones). |
|
||||||
|
| `homelab_alerts_acknowledged` | | Alerts somebody has acknowledged (and that aren't silenced). Still counted in `homelab_alerts`. |
|
||||||
|
| `homelab_alerts_silenced` | | Alerts held back by a maintenance window or a snooze. |
|
||||||
|
| `homelab_alerts_grouped` | | Alerts folded under a server that is down. |
|
||||||
|
| `homelab_incidents_open` | | Incidents that have begun and not yet ended. |
|
||||||
|
| `homelab_selfbackup_snapshots` | | How many backups of Homelab Manager's own database are kept. |
|
||||||
|
| `homelab_selfbackup_last_success_timestamp_seconds` | | When the newest of them was made, as a Unix time. |
|
||||||
|
| `homelab_selfbackup_last_attempt_ok` | | 1 if the last attempt to back it up worked, 0 if it failed (left out until one has been tried). |
|
||||||
|
| `homelab_integration_up` | `integration`, `type` | 1 if the integration's latest calls work, 0 if failing; left out until something has called it. |
|
||||||
|
| `homelab_maintenance_active_windows` | | Maintenance windows running now. |
|
||||||
|
| `homelab_secret_expiry_days` | `name` | Days until a secret expires (negative once it has). |
|
||||||
|
| `homelab_inventory_power_watts` | | Typical draw of everything in the inventory. |
|
||||||
|
| `homelab_backup_guests` | `status` | Proxmox guests by how well they are backed up: `ok`, `stale`, `failed`, `none`, `unknown`. From what the last backup check read; a scrape never goes to Proxmox or Backup Server itself, and the series is left out until something has been read. |
|
||||||
|
| `homelab_backup_last_age_seconds` | `guest`, `vmid`, `integration` | How long ago a guest's newest backup was. |
|
||||||
|
| `homelab_endpoint_up` | `name`, `kind` | 1 if the endpoint check's last answer was fine, else 0; left out until it has been asked. |
|
||||||
|
| `homelab_endpoint_response_milliseconds` | `name`, `kind` | How long it took to answer the last time. |
|
||||||
|
| `homelab_endpoint_value` | `name`, `kind` | The number a value check last read from the answer (checks that judge a value only, and only when it is a number). |
|
||||||
|
| `homelab_endpoint_certificate_days_left` | `name`, `kind` | Days until the certificate it shows expires (negative once it has). |
|
||||||
|
| `homelab_service_level_target_percent` | `name`, `subject`, `type` | The availability a service-level target asks for. |
|
||||||
|
| `homelab_service_level_availability_percent`, `homelab_service_level_error_budget_remaining_minutes` | `name`, `subject`, `type` | How available the thing has been so far this calendar month, and the downtime still allowed before the target is missed (negative once it is). Left out for a target with nothing recorded this month. |
|
||||||
|
| `homelab_duplicati_backup_ok` | `backup`, `source` | 1 if the Duplicati backup's last run worked and it is not overdue, else 0. |
|
||||||
|
| `homelab_duplicati_last_success_age_seconds`, `homelab_duplicati_source_size_bytes` | `backup`, `source` | Seconds since it last succeeded (a run with warnings counts), and the size of what its last report examined. |
|
||||||
|
| `homelab_nextcloud_cron_ok` | `name` | 1 if Nextcloud's own check says its background jobs ran within the last hour, else 0; absent when that could not be asked (no login, or before Nextcloud 28). |
|
||||||
|
| `homelab_nextcloud_maintenance_mode`, `homelab_nextcloud_app_updates` | `name` | 1 if Nextcloud is in maintenance mode, else 0; and how many apps have an update waiting (left out when not asked). Left out for a Nextcloud not read in the last twenty minutes. |
|
||||||
|
| `homelab_nextcloud_free_bytes`, `homelab_nextcloud_active_users_24h` | `name` | Bytes free where the data is kept, and the users active in the last day (both need an administrator's app password). |
|
||||||
|
| `homelab_docker_vulnerabilities` | `integration`, `host`, `severity` | The vulnerabilities in the scanned images of a Docker host, by severity (`critical`, `high`, `medium`, `low`), from Dockhand's stored scans. Left out for a host whose count could not be read, and when the reading is older than 45 minutes. |
|
||||||
|
| `homelab_docker_images_scanned` | `integration`, `host` | How many of a Docker host's images have a scan in Dockhand (the rest are not known to be clean). |
|
||||||
|
| `homelab_speedtest_download_bits_per_second`, `homelab_speedtest_upload_bits_per_second` | `name` | The download and upload speed of the newest completed speed test of a Speedtest Tracker, in bits per second. Left out while the reading is older than 45 minutes. |
|
||||||
|
| `homelab_speedtest_ping_seconds` | `name` | The ping of that test, in seconds. |
|
||||||
|
| `homelab_hetzner_server_running` | `project`, `server` | 1 if the Hetzner Cloud server is running, else 0. |
|
||||||
|
| `homelab_hetzner_traffic_outgoing_bytes`, `homelab_hetzner_traffic_included_bytes` | `project`, `server` | This month's outgoing traffic of a Hetzner Cloud server, and what its type includes per month (bytes). |
|
||||||
|
| `homelab_hetzner_monthly_cost` | `project`, `currency` | The estimated monthly cost of a Hetzner Cloud project at list prices, including VAT (an estimate: it has the load balancers and floating IPs, not traffic over the quota). |
|
||||||
|
| `homelab_hetzner_traffic_forecast_bytes` | `project`, `server` | What a Hetzner Cloud server's outgoing traffic is forecast to come to by the end of the month; left out when there is too little to say. |
|
||||||
|
| `homelab_hetzner_load_balancer_unhealthy_targets` | `project`, `load_balancer` | How many targets of a Hetzner Cloud load balancer are unhealthy. |
|
||||||
|
| `homelab_hetzner_certificate_expiry_timestamp_seconds` | `project`, `certificate` | When a Hetzner Cloud certificate ends, in seconds since 1970. |
|
||||||
|
| `homelab_hetzner_robot_servers` | `name` | How many dedicated servers a Hetzner Robot account has. |
|
||||||
|
| `homelab_hetzner_robot_rescue_active` | `name`, `server`, `server_name` | 1 if a dedicated server's rescue system is on for the next boot, else 0; left out when it could not be asked. |
|
||||||
|
| `homelab_hetzner_robot_days_left` | `name`, `server`, `server_name` — or `storage_box`, `storage_box_name` | Calendar days left of a cancelled dedicated server or storage box: 0 on its last day, negative after. Only for cancelled ones. |
|
||||||
|
| `homelab_hetzner_robot_storage_box_locked` | `name`, `storage_box`, `storage_box_name` | 1 if a storage box is locked, else 0. |
|
||||||
|
| `homelab_arr_health_issues` | `app`, `name` | How many warnings or errors Sonarr, Radarr or Prowlarr reports about itself. |
|
||||||
|
| `homelab_arr_queue_items`, `homelab_arr_queue_stuck_items` | `app`, `name` | The downloads in the queue of Sonarr or Radarr, and how many are stuck (judged wrong, failed or blocked for two hours or more). |
|
||||||
|
| `homelab_arr_wanted_missing` | `app`, `name` | Wanted items Sonarr or Radarr has no file for. |
|
||||||
|
| `homelab_arr_disk_free_bytes` | `app`, `name`, `path` | Free space of a disk Sonarr, Radarr or Prowlarr uses. |
|
||||||
|
| `homelab_download_speed_bytes` | `client`, `name`, `direction` | How fast qBittorrent or SABnzbd is downloading (`down`) or uploading (`up`), in bytes per second; SABnzbd does not say `up`. |
|
||||||
|
| `homelab_download_items` | `client`, `name`, `state` | How many items the download client has in each state (downloading, queued, stalled, seeding …); a state with none is left out except downloading and queued. |
|
||||||
|
| `homelab_download_stuck_items` | `client`, `name` | How many items are stuck (in an error state, or stalled with nothing moving for a day). |
|
||||||
|
| `homelab_download_paused`, `homelab_download_connected` | `client`, `name` | 1 if the whole queue is paused (SABnzbd), and 1 if the client can reach the network, i.e. is not `disconnected` (qBittorrent only), else 0. |
|
||||||
|
| `homelab_download_failed_recent` | `client`, `name` | Downloads that failed in the last day (SABnzbd only). |
|
||||||
|
| `homelab_download_disk_free_bytes` | `client`, `name`, `folder` | Free space of a folder the download client uses. |
|
||||||
|
| `homelab_app_update_available` | `app`, `name`, `running`, `latest` | 1 if a newer release of the application is out than the one it runs, else 0. Only while Operations → App versions is switched on, and for an application whose newest release has been looked up. |
|
||||||
|
| `homelab_adguard_running`, `homelab_adguard_protection_enabled`, `homelab_adguard_dns_up` | `name` | 1 if AdGuard Home's DNS server runs, if its protection is on, and if it answered the app's test DNS question; else 0. |
|
||||||
|
| `homelab_adguard_dns_probe_seconds` | `name` | How long it took to answer that question; left out when it did not. |
|
||||||
|
| `homelab_adguard_queries`, `homelab_adguard_blocked`, `homelab_adguard_avg_processing_seconds` | `name` | The queries it answered in its statistics period, how many it blocked, and its average processing time. |
|
||||||
|
| `homelab_technitium_dns_up`, `homelab_technitium_blocking_enabled` | `name` | 1 if Technitium's DNS server answered the app's test DNS question, and if its blocking is on; else 0. |
|
||||||
|
| `homelab_technitium_dns_probe_seconds` | `name` | How long it took to answer that question; left out when it did not. |
|
||||||
|
| `homelab_technitium_uptime_seconds` | `name` | How long the DNS server has run, since it last started; left out when it did not say. |
|
||||||
|
| `homelab_technitium_block_lists_overdue` | `name` | 1 if the block lists' update was due more than six hours ago, else 0. Only where block lists are set up. |
|
||||||
|
| `homelab_technitium_queries`, `homelab_technitium_blocked`, `homelab_technitium_cached`, `homelab_technitium_recursive`, `homelab_technitium_server_failures`, `homelab_technitium_clients` | `name` | The last day's queries, how many were blocked, answered from the cache, resolved by the server and ended in a server failure, and how many different clients asked. Gauges over a rolling day, not counters. |
|
||||||
|
| `homelab_technitium_cache_entries` | `name` | Entries in its DNS cache now. |
|
||||||
|
| `homelab_technitium_zones`, `homelab_technitium_zone_problems` | `name` | The zones it holds (its internal ones not counted), and how many have expired or cannot sync. |
|
||||||
|
| `homelab_technitium_dhcp_leases` | `name` | The DHCP leases it holds. Left out when it has no DHCP server. |
|
||||||
|
| `homelab_grafana_rules`, `homelab_grafana_rules_firing`, `homelab_grafana_rules_error` | `name` | How many alert rules Grafana has, how many are firing and how many cannot be evaluated. |
|
||||||
|
| `homelab_grafana_datasources_failing` | `name` | How many of Grafana's data sources fail its health check. |
|
||||||
|
| `homelab_fragnet_server_running` | `account`, `server`, `game` | 1 if the Fragnet game server is running, else 0. |
|
||||||
|
| `homelab_fragnet_disk_used_bytes`, `homelab_fragnet_disk_limit_bytes` | `account`, `server`, `game` | The disk a Fragnet game server uses, and the disk its plan allows (left out when unlimited). |
|
||||||
|
| `homelab_fragnet_memory_used_bytes`, `homelab_fragnet_memory_limit_bytes` | `account`, `server`, `game` | The memory a Fragnet game server uses (0 while stopped), and its limit (left out when unlimited). |
|
||||||
|
| `homelab_fragnet_newest_backup_timestamp_seconds` | `account`, `server`, `game` | When the newest successful backup of a game server finished, in seconds since 1970; left out for one with no backup. |
|
||||||
|
| `homelab_netbox_workers`, `homelab_netbox_devices`, `homelab_netbox_ip_addresses` | `name` | The background workers running, and the devices and IP addresses NetBox holds; left out for a NetBox not read in the last twenty minutes. |
|
||||||
|
| `homelab_n8n_workflows`, `homelab_n8n_workflows_active`, `homelab_n8n_workflows_failing` | `name` | How many workflows n8n has, how many are switched on, and how many of those failed on their last run (absent when the runs could not be read). |
|
||||||
|
| `homelab_immich_photos`, `homelab_immich_videos`, `homelab_immich_library_bytes` | `name` | How many photos and videos Immich holds and what the library takes; left out for an Immich not read in the last twenty minutes. |
|
||||||
|
| `homelab_immich_disk_used_percent`, `homelab_immich_failed_jobs` | `name` | How full the disk under the library is, and how many background jobs have failed (all queues). |
|
||||||
|
| `homelab_paperless_documents`, `homelab_paperless_inbox`, `homelab_paperless_failed_imports` | `name` | The documents Paperless-ngx holds, those waiting in its inbox, and the imports that failed in the last week and were not dismissed. |
|
||||||
|
| `homelab_paperless_component_ok` | `name`, `component` | 1 if that part of Paperless (database, redis, celery, index, classifier, sanity_check) is in order, else 0; absent when the system status could not be read. |
|
||||||
|
| `homelab_authentik_outpost_up` | `name`, `outpost` | 1 if the Authentik outpost has had an instance report in the last five minutes, else 0; left out for an Authentik not read in the last twenty minutes. |
|
||||||
|
| `homelab_authentik_workers`, `homelab_authentik_failed_logins_24h` | `name` | How many Authentik workers are connected, and how many sign-ins failed in the last day (100 means at least that). |
|
||||||
|
| `homelab_authentik_certificate_expiry_days` | `name`, `certificate` | Whole days until a certificate (one Authentik does not renew itself) runs out; negative once it has. |
|
||||||
|
| `homelab_jellyfin_streams`, `homelab_jellyfin_transcodes` | `name` | How many are streaming from Jellyfin right now, and how many of those it is transcoding; left out for a Jellyfin not read in the last ten minutes. |
|
||||||
|
| `homelab_jellyfin_pending_restart` | `name` | 1 if Jellyfin is waiting to be restarted, else 0. |
|
||||||
|
| `homelab_ups_on_battery` | `name` | 1 if the UPS has lost mains power and runs on its battery, else 0; left out for a UPS that has not been read in the last five minutes. |
|
||||||
|
| `homelab_ups_battery_charge_percent`, `homelab_ups_runtime_seconds`, `homelab_ups_load_percent`, `homelab_ups_input_volts` | `name` | How full the battery is, how long it would last at the present load, how much of its capacity the load uses, and the mains voltage it sees — each left out when the UPS does not say. |
|
||||||
|
| `homelab_heartbeat_up` | `name` | 1 if the job has reported in time and not said it failed, else 0; switched-off heartbeats are left out. |
|
||||||
|
| `homelab_heartbeat_last_ping_timestamp_seconds` | `name` | When the job last reported that it finished (Unix time). |
|
||||||
|
| `homelab_heartbeat_last_duration_seconds` | `name` | How long the job said its last run took (only when it sent `/start`). |
|
||||||
|
| `homelab_restore_test_age_days` | `subject` | Days since a restore of this was last tested; things marked "no need to test" are left out. |
|
||||||
|
| `homelab_restore_test_passed` | `subject` | 1 if the last restore test passed (fully or partly), 0 if it failed. |
|
||||||
|
| `homelab_scrape_duration_seconds` | | How long building the answer took. |
|
||||||
|
| `homelab_scrape_timestamp_seconds` | | When the answer was built. |
|
||||||
|
|
||||||
|
## Useful queries
|
||||||
|
|
||||||
|
```promql
|
||||||
|
# Servers that have gone quiet
|
||||||
|
homelab_server_up == 0
|
||||||
|
|
||||||
|
# The five fullest disks
|
||||||
|
topk(5, homelab_server_disk_used_percent)
|
||||||
|
|
||||||
|
# Disks that will be full within a month, soonest first (and the same for storage: homelab_storage_days_to_full)
|
||||||
|
sort(homelab_server_disk_days_to_full < 30)
|
||||||
|
|
||||||
|
# Endpoints slower than half a second
|
||||||
|
homelab_endpoint_response_milliseconds > 500
|
||||||
|
|
||||||
|
# How many guests are not well backed up
|
||||||
|
sum(homelab_backup_guests{status=~"stale|failed|none"})
|
||||||
|
|
||||||
|
# Updates waiting across all servers, and how many are security fixes
|
||||||
|
sum(homelab_server_pending_updates), sum(homelab_server_pending_security_updates)
|
||||||
|
|
||||||
|
# Is the dev copy up as well as production? (several installations: filter on the label you added)
|
||||||
|
homelab_info
|
||||||
|
```
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
| Symptom | Cause |
|
||||||
|
|---|---|
|
||||||
|
| Target is **DOWN**, `401 Unauthorized` | The token is wrong, was removed, or has a trailing newline in its file (use `printf '%s'`). The same `401` is given for no token and a bad one. |
|
||||||
|
| Target is **DOWN**, `429 Too many requests` | More than 120 requests a minute from one address — a second Prometheus, or a very short scrape interval. |
|
||||||
|
| Target is **DOWN**, certificate error | Prometheus does not trust the certificate. Give it the CA with `tls_config.ca_file` (or `insecure_skip_verify: true` on a trusted LAN, knowing what that gives up). |
|
||||||
|
| Target **UP** but some series are missing | A series that has nothing to say is **left out** rather than reported as 0 — an integration nobody has called yet, a check never asked, a server whose agent reports no SMART data. That is expected. |
|
||||||
|
| A dashboard panel is empty | The series it asks for is left out (as above), or the app is older than the dashboard file. Download the dashboard again after an upgrade. |
|
||||||
|
| Numbers differ from the Alerts page | `homelab_alerts` counts active alerts and leaves out silenced ones; the page shows both. The app's own result is reused for up to a minute. |
|
||||||
|
| Names with odd characters look wrong | Label values are escaped (`\"`, `\\`, `\n`) as the Prometheus format requires; Prometheus and Grafana unescape them. |
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
# Documentation
|
||||||
|
|
||||||
|
Everything about running and integrating with Homelab Manager is in Markdown files in this repository. This folder holds the **guides**; the **references** (who can do what, every notification, every table,
|
||||||
|
the credentials each integration needs) are next to the main [README](../README.md).
|
||||||
|
|
||||||
|
## Guides — this folder
|
||||||
|
|
||||||
|
| Guide | Read it when you want to … |
|
||||||
|
|---|---|
|
||||||
|
| **[SSO.md](SSO.md)** | Set up sign-in with Authentik: what is needed (HTTPS, reverse proxy, a signing key), the provider and `.env` settings, how roles can follow groups, and what to check when sign-in fails. |
|
||||||
|
| **[API.md](API.md)** | Call Homelab Manager from a script, Home Assistant or another tool: personal tokens, every `/api/v1` address with its answer, examples in curl, Python, Node and Home Assistant, and the other machine-facing addresses (agent reports, heartbeats, share links). |
|
||||||
|
| **[PROMETHEUS-GRAFANA.md](PROMETHEUS-GRAFANA.md)** | Scrape `/metrics` with Prometheus, import the Grafana dashboard, copy a working Docker Compose and alerting rules, and look up every series. |
|
||||||
|
| **[WEBHOOKS.md](WEBHOOKS.md)** | Be told when something happens: signed event webhooks (alert opened/resolved, maintenance started/ended) with receivers in Node and Python, and the chat-style notification webhook. |
|
||||||
|
| **[HEARTBEATS-AND-CALENDAR.md](HEARTBEATS-AND-CALENDAR.md)** | Monitor a cron job (it calls an address when it finishes) or subscribe a calendar to what is coming up. |
|
||||||
|
| **[MONITORING.md](MONITORING.md)** | Understand how monitoring works: what is watched and how often, how a reading becomes an alert, what acknowledge, snooze, maintenance and flapping each do, incidents, history, forecasts, and how to tune it. |
|
||||||
|
|
||||||
|
## References — next to the README
|
||||||
|
|
||||||
|
| File | What it is |
|
||||||
|
|---|---|
|
||||||
|
| [README.md](../README.md) | What the app is, every feature, installation (Docker, arm64, development). |
|
||||||
|
| [ROLES.md](../ROLES.md) | Viewer, operator, administrator: who sees which page and who can change what. |
|
||||||
|
| [NOTIFICATIONS.md](../NOTIFICATIONS.md) | Every notification the app sends, what triggers it, what toggles it, routing, quiet hours, escalation. |
|
||||||
|
| [INTEGRATIONS.md](../INTEGRATIONS.md) | What each integration and DNS provider needs (credentials, permissions) and what it does. |
|
||||||
|
| [DATABASE.md](../DATABASE.md) | The database tables, columns and relationships. |
|
||||||
|
| [BACKUP.md](../BACKUP.md) | What to back up, how (also while it runs), how to restore, and how to check a backup. |
|
||||||
|
| [AGENT_UPDATE.md](../AGENT_UPDATE.md) | Keeping the agents on the servers up to date. |
|
||||||
|
|
||||||
|
## Where to start
|
||||||
|
|
||||||
|
- **A new installation:** [README](../README.md) (install) → [SSO.md](SSO.md) (sign-in) → add servers and integrations in the app → [MONITORING.md](MONITORING.md) (what to expect) → [BACKUP.md](../BACKUP.md).
|
||||||
|
- **Getting figures into Grafana:** [PROMETHEUS-GRAFANA.md](PROMETHEUS-GRAFANA.md).
|
||||||
|
- **Getting alerts into something else:** [WEBHOOKS.md](WEBHOOKS.md) for pushes, [API.md](API.md) for asking.
|
||||||
|
- **"Why did / didn't it alert?":** [MONITORING.md](MONITORING.md), then [NOTIFICATIONS.md](../NOTIFICATIONS.md).
|
||||||
|
|
||||||
|
## Conventions in these pages
|
||||||
|
|
||||||
|
- Examples use **`https://hm.example.lan`** for your own address (`APP_BASE_URL`), and tokens like `hlm_p_xxxxxxxx…` that you will replace.
|
||||||
|
- A **menu path** such as *Settings → API and webhooks* is in the web interface; who may open it is in [ROLES.md](../ROLES.md).
|
||||||
|
- Sample answers have the shapes the app really sends (several were captured from a test installation and trimmed); the values are only illustrative.
|
||||||
|
- Things that **have not been checked against the real thing** (a third-party system, a Grafana version, another identity provider) are said so where they come up.
|
||||||
+230
@@ -0,0 +1,230 @@
|
|||||||
|
# Single sign-on (SSO)
|
||||||
|
|
||||||
|
Homelab Manager has no passwords of its own. People sign in through an identity provider with **OpenID Connect (OIDC)**, and the app only ever sees who they are,
|
||||||
|
never a password. It is written for **[Authentik](https://goauthentik.io/)** and the settings are named after it; any provider that speaks standard OIDC *can* work
|
||||||
|
(see [Other identity providers](#other-identity-providers)), but only Authentik has been used.
|
||||||
|
|
||||||
|
This page covers what the setup needs, how to configure Authentik and the app, how roles are decided, and what to check when sign-in fails.
|
||||||
|
|
||||||
|
- [How sign-in works](#how-sign-in-works)
|
||||||
|
- [What you need before you start](#what-you-need-before-you-start)
|
||||||
|
- [1. Set up Authentik](#1-set-up-authentik)
|
||||||
|
- [2. Set up the app](#2-set-up-the-app)
|
||||||
|
- [3. The first sign-in](#3-the-first-sign-in)
|
||||||
|
- [Roles](#roles) — by hand, or from groups
|
||||||
|
- [Sessions and signing out](#sessions-and-signing-out)
|
||||||
|
- [What does *not* use SSO](#what-does-not-use-sso)
|
||||||
|
- [Troubleshooting](#troubleshooting)
|
||||||
|
- [If you are locked out](#if-you-are-locked-out)
|
||||||
|
- [Security notes](#security-notes)
|
||||||
|
|
||||||
|
## How sign-in works
|
||||||
|
|
||||||
|
1. A person opens the app and is sent to `/auth/login`. The app starts an *authorization code* flow with **PKCE** (a one-time code verifier and challenge) and a random `state`,
|
||||||
|
and redirects the browser to Authentik.
|
||||||
|
2. The person signs in at Authentik (with whatever Authentik asks for: password, passkey, two-factor …).
|
||||||
|
3. Authentik sends the browser back to **`<APP_BASE_URL>/auth/callback`** with a one-time code.
|
||||||
|
4. The app, from its own server, swaps the code for tokens at Authentik, checks the **ID token's signature** against the keys Authentik publishes (not only its contents), and
|
||||||
|
asks Authentik's *userinfo* address for the person's name, email, picture and groups.
|
||||||
|
5. The app finds the person by their **`sub`** (Authentik's stable id for them) or makes an account, decides their role ([below](#roles)), writes the sign-in to the audit log
|
||||||
|
and starts a session with a **new session id**.
|
||||||
|
|
||||||
|
The *client* is **confidential**: the app holds a client secret, and the code exchange happens server to server. The scopes asked for are `openid email profile`.
|
||||||
|
|
||||||
|
## What you need before you start
|
||||||
|
|
||||||
|
| You need | Why |
|
||||||
|
|---|---|
|
||||||
|
| **An Authentik instance** | The identity provider. |
|
||||||
|
| **The app reachable by browsers at a fixed address — `APP_BASE_URL`** — and **over HTTPS** in production | The redirect address Authentik sends people back to is built from it. The session cookie is marked `Secure` when the app runs in production (`NODE_ENV=production`, which the supplied Docker setup does), and browsers only keep a `Secure` cookie from an HTTPS page — so behind plain `http://` on a LAN address, sign-in appears to work and then sends you straight back to the login page. (`http://localhost` is the exception browsers allow.) Put the app behind a reverse proxy with a certificate. |
|
||||||
|
| **A reverse proxy that passes `X-Forwarded-For` and `X-Forwarded-Proto`** (one proxy in front) | The app trusts one proxy hop, so the sign-in log and the rate limit see each person's address instead of the proxy's. |
|
||||||
|
| **The app able to reach Authentik, server to server** | It calls Authentik's discovery, token, userinfo and key (JWKS) addresses itself, from wherever it runs — from inside the Docker network that may not be the name your browser uses. |
|
||||||
|
| **Authentik's certificate trusted by the app** | If Authentik uses a certificate from your own CA, give Node the CA with `NODE_EXTRA_CA_CERTS=/path/to/ca.pem`. Do **not** switch certificate checking off with `NODE_TLS_REJECT_UNAUTHORIZED=0`. |
|
||||||
|
| **Reasonably correct clocks** on both machines | Tokens have times in them; a minute of drift is tolerated, more is refused. |
|
||||||
|
| **Two secrets generated by you** | `SESSION_SECRET` (signs the session cookie) and `CREDENTIALS_ENCRYPTION_KEY` (encrypts stored integration credentials). Not part of SSO itself, but the app will not run in production without the first. |
|
||||||
|
|
||||||
|
Generate both with:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
node -e "console.log(require('crypto').randomBytes(32).toString('hex'))"
|
||||||
|
```
|
||||||
|
|
||||||
|
## 1. Set up Authentik
|
||||||
|
|
||||||
|
The names of the menus differ a little between Authentik versions; what has to be true does not.
|
||||||
|
|
||||||
|
**Create the provider** — *Applications → Providers → Create → OAuth2/OpenID Provider*:
|
||||||
|
|
||||||
|
| Setting | Value |
|
||||||
|
|---|---|
|
||||||
|
| Name | anything, e.g. `Homelab Manager` |
|
||||||
|
| Authorization flow | your usual one (explicit or implicit consent, as you prefer) |
|
||||||
|
| **Client type** | **Confidential** |
|
||||||
|
| Client ID / Client secret | generated for you — you will copy both into the app's `.env` |
|
||||||
|
| **Redirect URIs** | `https://hm.example.lan/auth/callback` — your `APP_BASE_URL` followed by `/auth/callback`, as an **exact (strict) match** |
|
||||||
|
| **Signing key** | **choose a certificate (an RSA key), such as *authentik Self-signed Certificate*.** With no signing key Authentik signs ID tokens with the client secret (HS256) and publishes no keys, and the app refuses such a token. |
|
||||||
|
| Scopes | `openid`, `email`, `profile` (the defaults) |
|
||||||
|
| Subject mode | leave the default (*Based on the hashed User ID*) — see the warning below |
|
||||||
|
| Issuer mode | leave the default (*Each provider has a different issuer*) |
|
||||||
|
|
||||||
|
> **Do not change the subject mode later.** The app recognises a person by their `sub`. Changing how Authentik builds it makes every person a *new* person: a new account
|
||||||
|
> (a viewer, with none of the old account's role) and the old one left behind.
|
||||||
|
|
||||||
|
**Create the application** — *Applications → Applications → Create*: give it a name and a **slug** (e.g. `homelab-manager`), choose the provider above, and decide **who may use it**
|
||||||
|
(a policy or group binding on the application). Authentik decides who can *sign in*; the app's roles decide what they can *do* once in.
|
||||||
|
|
||||||
|
The **issuer URL** you need next is on the provider's page — normally `https://authentik.example.lan/application/o/<slug>/`, **with the trailing slash**.
|
||||||
|
|
||||||
|
## 2. Set up the app
|
||||||
|
|
||||||
|
Put these in the `.env` file next to `docker-compose.yml` (a template is in `.env.example`):
|
||||||
|
|
||||||
|
```ini
|
||||||
|
# Where people reach the app. HTTPS, no trailing slash needed. Used to build the redirect address.
|
||||||
|
APP_BASE_URL=https://hm.example.lan
|
||||||
|
|
||||||
|
# From the Authentik provider.
|
||||||
|
AUTHENTIK_ISSUER_URL=https://authentik.example.lan/application/o/homelab-manager/
|
||||||
|
AUTHENTIK_CLIENT_ID=…
|
||||||
|
AUTHENTIK_CLIENT_SECRET=…
|
||||||
|
|
||||||
|
# Generated by you (see above). Changing SESSION_SECRET signs everyone out.
|
||||||
|
SESSION_SECRET=…
|
||||||
|
CREDENTIALS_ENCRYPTION_KEY=…
|
||||||
|
```
|
||||||
|
|
||||||
|
| Variable | Meaning |
|
||||||
|
|---|---|
|
||||||
|
| `APP_BASE_URL` | The address people type. The redirect address is `APP_BASE_URL` + `/auth/callback`; it must match what the Authentik provider allows, character for character (scheme, host, port). Also the address used in links in notifications and in the read API. |
|
||||||
|
| `AUTHENTIK_ISSUER_URL` | The provider's issuer URL. The app *discovers* the rest from `<issuer>/.well-known/openid-configuration`, so a wrong or unreachable issuer is the usual reason sign-in fails. |
|
||||||
|
| `AUTHENTIK_CLIENT_ID`, `AUTHENTIK_CLIENT_SECRET` | From the provider. The secret is read from the environment only; it is not stored in the database. |
|
||||||
|
| `SESSION_SECRET` | Signs the session cookie. In production the app **refuses to start** without it, and warns if it is still the placeholder from `.env.example` or shorter than 32 characters. |
|
||||||
|
| `SESSION_DIR` | Where sessions are kept on disk (default `data/sessions`, inside the data volume). |
|
||||||
|
| `NODE_ENV` | `production` in the Docker image: makes the cookie `Secure` and requires `SESSION_SECRET`. |
|
||||||
|
|
||||||
|
If any of the three `AUTHENTIK_*` values is missing the app still starts — so the rest of it can be looked at — but `/auth/login` fails until they are set. Restart the container after
|
||||||
|
changing `.env`.
|
||||||
|
|
||||||
|
## 3. The first sign-in
|
||||||
|
|
||||||
|
**The first person to sign in becomes the administrator**, so sign in yourself first on a new installation. Everyone after that starts as a **viewer** until an administrator
|
||||||
|
changes their role under **Administration → Users** — or until their groups decide it ([below](#roles)).
|
||||||
|
|
||||||
|
Only people Authentik lets into the application can sign in at all, which is why the application's access policy matters: the first account is whoever you let in first. A person an administrator makes from **Apps → Authentik → Users and groups** is a new Authentik user, no more: to sign in here they still have to be allowed into the application (its group binding or policy), and they start as a viewer like anyone else.
|
||||||
|
|
||||||
|
## Roles
|
||||||
|
|
||||||
|
There are three: **viewer** (sees everything, changes nothing), **operator** (acknowledges and silences alerts, starts maintenance, runs updates, writes notes and jobs …) and
|
||||||
|
**admin** (everything, including settings, integrations and users). [ROLES.md](../ROLES.md) lists exactly who can see and do what.
|
||||||
|
|
||||||
|
### By hand (the default)
|
||||||
|
|
||||||
|
Administrators change roles on **Administration → Users**. The only administrator cannot be demoted — there is always at least one.
|
||||||
|
|
||||||
|
### From groups
|
||||||
|
|
||||||
|
Instead of setting roles by hand, let the groups Authentik reports decide them. In Authentik, the user's groups come in a **`groups`** claim that is part of the **`profile`** scope the app
|
||||||
|
already asks for, so there is nothing to add on the Authentik side except the groups themselves (e.g. `homelab-admins`, `homelab-ops`) and who is in them.
|
||||||
|
|
||||||
|
In the app: **Settings → Sign-in → Roles from groups**.
|
||||||
|
|
||||||
|
| Setting | Meaning |
|
||||||
|
|---|---|
|
||||||
|
| Set roles from groups at each sign-in | The switch. Off, roles are by hand. |
|
||||||
|
| Claim that holds the groups | `groups` for Authentik. |
|
||||||
|
| Admin groups | Names (one per line or comma separated; case does not matter). In any of them → **admin**. |
|
||||||
|
| Operator groups | In any of them (and in no admin group) → **operator**. |
|
||||||
|
| Someone in none of those groups | **becomes a viewer**, or **keeps the role they already have**. |
|
||||||
|
|
||||||
|
The rules are applied **at every sign-in**, so a change in Authentik shows up the next time the person signs in (or when their session next starts), not instantly.
|
||||||
|
|
||||||
|
Safeguards:
|
||||||
|
|
||||||
|
- If a sign-in carries **no groups claim at all** (a wrong claim name, a scope that is missing), roles are left exactly as they are — "we weren't told" is never treated as "in no groups".
|
||||||
|
- **The only administrator is never demoted** by a group rule.
|
||||||
|
- **The first person to sign in is still the administrator**, whatever the groups say.
|
||||||
|
- While roles follow groups, **roles cannot be set by hand** (a role set on the Users page would be undone at the next sign-in); the Users page says so.
|
||||||
|
- Every role change made from groups is written to the audit log (`user` / `role_from_groups`, and `role_from_groups_blocked` when the last administrator was protected).
|
||||||
|
|
||||||
|
Before switching it on, press **Check against the groups people had last time**: it changes nothing and shows what each person *would* become, from the groups they had at their last sign-in.
|
||||||
|
The groups each person had are also shown on the Users page, so you can see the names before relying on them.
|
||||||
|
|
||||||
|
### Profile pictures
|
||||||
|
|
||||||
|
Optional, off by default (**Settings → Sign-in → Profile pictures**): the picture Authentik gives, and/or a Gravatar. Either one means *the server* fetches the picture from an outside
|
||||||
|
address; people's browsers never do, and the app serves the pictures from itself.
|
||||||
|
|
||||||
|
## Sessions and signing out
|
||||||
|
|
||||||
|
- A session lasts up to **7 days**. The cookie is `HttpOnly`, `SameSite=Lax` and, in production, `Secure`.
|
||||||
|
- **Disabling someone in Authentik does not end an app session that is already running.** To cut someone off at once, end their session under **Users → Signed in now**
|
||||||
|
(administrators); it ends immediately, and they cannot start another once Authentik refuses them. Accounts themselves are not deleted in the app — a person who no longer signs in
|
||||||
|
simply has no session.
|
||||||
|
- **Signing out** of the app also ends the person's Authentik session when Authentik supports *RP-initiated logout* (it does), so the next visit asks for a password again.
|
||||||
|
Ending a session from *Signed in now* ends only the app's own session, not the one at Authentik.
|
||||||
|
- Sign-ins and sign-outs are in the audit log, with the address they came from.
|
||||||
|
|
||||||
|
## What does *not* use SSO
|
||||||
|
|
||||||
|
Some addresses are meant for machines and use their own tokens instead of a session. None of them can be used to sign in, and each stops working the moment its token is removed.
|
||||||
|
|
||||||
|
| Address | Used by | Token |
|
||||||
|
|---|---|---|
|
||||||
|
| `/api/agent/report` | the agent on each server | one per server, shown once when the server is added |
|
||||||
|
| `/metrics` | Prometheus | a *metrics token* (Settings → API and webhooks) — see [PROMETHEUS-GRAFANA.md](PROMETHEUS-GRAFANA.md) |
|
||||||
|
| `/api/v1/…` | scripts, Home Assistant | a *personal API token* (user menu → API tokens) — see [API.md](API.md) |
|
||||||
|
| `/api/heartbeat/<token>` | cron jobs | the heartbeat's own address — see [HEARTBEATS-AND-CALENDAR.md](HEARTBEATS-AND-CALENDAR.md) |
|
||||||
|
| `/api/public/…` | status pages, the wall display, the calendar feed | a *share link* (Settings → Share links) |
|
||||||
|
| `/api/alert-actions/<token>…` | the buttons and links on a notification | a signed token for the alerts in one message, valid 24 hours — see [WEBHOOKS.md](WEBHOOKS.md#buttons-on-notifications) |
|
||||||
|
| `/health` | container health checks | none (answers `{"ok":true}`) |
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
A failed sign-in shows a plain page ("Sign-in failed") and nothing more — deliberately, since the reason would tell an attacker what to adjust. **The reason is in the server log and in
|
||||||
|
the audit log** (*Change log → Raw audit log*, category `session`, action `login_failed`, with the address and a short reason).
|
||||||
|
|
||||||
|
| Reason in the log | What it usually means |
|
||||||
|
|---|---|
|
||||||
|
| `provider_unreachable` | The app could not reach Authentik to read its configuration: a wrong `AUTHENTIK_ISSUER_URL` (missing trailing slash, wrong slug, `http` instead of `https`), a name the *container* cannot resolve, a firewall, or a certificate the app does not trust (see `NODE_EXTRA_CA_CERTS`). It retries on the next sign-in; no restart is needed. |
|
||||||
|
| `provider_rejected` | Authentik answered but the exchange failed. Most often: **the redirect URI** in the provider is not exactly `APP_BASE_URL/auth/callback`; the **client secret** is wrong; the provider has **no signing key** (HS256 tokens are refused); a clock that is several minutes off; or the browser took too long and the one-time code ran out. |
|
||||||
|
| `no_identity` | Authentik signed the person in but sent no `sub`. Check the provider's scopes include `openid`. |
|
||||||
|
| *(not recorded)* `no_pending_login` | A page left open too long, a bookmarked callback address, or the session cookie was not kept. If it happens every time, the cookie is the problem — see the next row. |
|
||||||
|
|
||||||
|
Other symptoms:
|
||||||
|
|
||||||
|
| Symptom | Likely cause |
|
||||||
|
|---|---|
|
||||||
|
| You sign in at Authentik and land on the login page again, over and over | The session cookie is not being kept. In production it is `Secure`, so the site must be opened over **HTTPS**; behind a reverse proxy, check it sends `X-Forwarded-Proto`. Also check `APP_BASE_URL` is the address in the browser's address bar. |
|
||||||
|
| Everyone is signed out after a restart | `SESSION_SECRET` changed (or is generated fresh each start), or the `data/sessions` folder is not on the persistent volume. |
|
||||||
|
| New people are all viewers although roles from groups is on | The sign-in carried no groups: check the claim name, and that the Authentik user is in the groups. Look at the Users page — the groups seen at each person's last sign-in are shown there. |
|
||||||
|
| Roles from groups did nothing to an administrator | Working as intended if they are the only administrator (a group rule never removes the last one); the audit log says `role_from_groups_blocked`. |
|
||||||
|
| "Cannot remove the only admin" | Same safeguard, on the Users page. Make someone else an administrator first. |
|
||||||
|
| A person with a different login at Authentik (renamed, merged) shows up as a new person | The app matches on `sub`, not on name or email. See the subject-mode warning above. |
|
||||||
|
|
||||||
|
## If you are locked out
|
||||||
|
|
||||||
|
There is no local password, so if the identity provider is down or misconfigured nobody can sign in. People who are **already signed in** keep working until their session ends.
|
||||||
|
To get back in, fix the provider or the `AUTHENTIK_*` settings and restart the container.
|
||||||
|
|
||||||
|
If the only administrator's account was changed so that nobody is an administrator any more, promote someone directly in the database (stop the app, or accept that it is a
|
||||||
|
single quick write; **take a copy of the data folder first** — see [BACKUP.md](../BACKUP.md)):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sqlite3 data/homelab-manager.sqlite "UPDATE users SET role = 'admin' WHERE email = 'you@example.lan';"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Security notes
|
||||||
|
|
||||||
|
- **PKCE and `state`** protect the code exchange; the callback is refused without the pending sign-in that the same browser started.
|
||||||
|
- **The ID token's signature is verified** against Authentik's published keys, so the login does not depend on the connection to Authentik alone.
|
||||||
|
- **A new session id is issued at sign-in**, so an id planted in the browser beforehand cannot be used.
|
||||||
|
- **Requests that change something are refused when they come from another origin** (the cookie is `SameSite=Lax`, which does not stop another app on the same parent domain), including `GET /auth/logout`.
|
||||||
|
- **Sign-in is rate-limited** to 20 attempts a minute per address, and each failure is audited.
|
||||||
|
- Every response carries `frame-ancestors 'none'`, `X-Frame-Options: DENY`, `nosniff` and a same-origin referrer policy.
|
||||||
|
|
||||||
|
## Other identity providers
|
||||||
|
|
||||||
|
The code uses standard OIDC discovery (`<issuer>/.well-known/openid-configuration`), the authorization code flow with PKCE (`S256`), a confidential client, **RS256-signed ID tokens
|
||||||
|
with a published JWKS**, and the `sub`, `email`, `name`/`preferred_username`, `picture` and a groups claim (name configurable) — nothing Authentik-specific. Keycloak, Authelia and
|
||||||
|
similar providers should therefore work by putting *their* issuer URL, client id and secret in the `AUTHENTIK_*` variables and setting the groups claim name under **Settings → Sign-in**.
|
||||||
|
This has **not been tried**; if you try one, make sure it signs with RS256 and that the redirect address is registered exactly.
|
||||||
@@ -0,0 +1,337 @@
|
|||||||
|
# Webhooks
|
||||||
|
|
||||||
|
Homelab Manager can **tell other systems when something happens** — Home Assistant, n8n, a status board, your own script — instead of making them ask. There are two different things called a webhook
|
||||||
|
here, and they are easy to mix up:
|
||||||
|
|
||||||
|
| | **Event webhooks** | **The notification webhook** |
|
||||||
|
|---|---|---|
|
||||||
|
| For | **Machines**: a structured, signed JSON message per event | **People**: the same sentence a human would read, posted to a chat-style endpoint |
|
||||||
|
| Where | **Settings → API and webhooks → Event webhooks** | **Settings → Notifications → Channels → Webhook** |
|
||||||
|
| Sent when | An alert opens or resolves, a maintenance window starts or ends | Whenever a notification is sent to a channel (it is one of the five channels) |
|
||||||
|
| Body | `{ event, id, createdAt, source, data }` | `{ "content": "**Title**\nmessage" }` |
|
||||||
|
| Authenticity | `X-HLM-Signature` — an HMAC-SHA256 of the body | a shared secret in `X-Webhook-Secret` (not a signature) |
|
||||||
|
| Held back by routing, quiet hours, flapping, snooze | **No** (it is a separate way of being told) | **Yes** (it is a notification like the others) |
|
||||||
|
|
||||||
|
- [Event webhooks](#event-webhooks)
|
||||||
|
- [Setting one up](#setting-one-up)
|
||||||
|
- [Events](#events)
|
||||||
|
- [The message](#the-message)
|
||||||
|
- [Checking the signature](#checking-the-signature)
|
||||||
|
- [Delivery: what to rely on and what not to](#delivery-what-to-rely-on-and-what-not-to)
|
||||||
|
- [Alert keys](#alert-keys)
|
||||||
|
- [A small receiver](#a-small-receiver)
|
||||||
|
- [Home Assistant and n8n](#home-assistant-and-n8n)
|
||||||
|
- [The notification webhook](#the-notification-webhook)
|
||||||
|
- [Buttons on notifications](#buttons-on-notifications)
|
||||||
|
- [Troubleshooting](#troubleshooting)
|
||||||
|
|
||||||
|
## Event webhooks
|
||||||
|
|
||||||
|
### Setting one up
|
||||||
|
|
||||||
|
Administrators: **Settings → API and webhooks → Event webhooks → Add a webhook**.
|
||||||
|
|
||||||
|
| Field | Notes |
|
||||||
|
|---|---|
|
||||||
|
| Name | up to 60 characters; what the list shows |
|
||||||
|
| Address | `http://` or `https://`, **no user name or password in it**. The address is not restricted to public hosts — a receiver on your LAN is the normal case. |
|
||||||
|
| Secret | optional, up to 200 characters. With one, every message is **signed**. Stored encrypted when `CREDENTIALS_ENCRYPTION_KEY` is set; **never shown again** after saving. |
|
||||||
|
| Events | at least one of the four below |
|
||||||
|
|
||||||
|
- **Up to 20 webhooks.** Each can be **switched off** and on, **removed**, and **tested**. The secret cannot be edited afterwards: to change it, remove the webhook and make a new one.
|
||||||
|
- **Test** sends `{"event": "test", …}` right away, signed like the real thing, whether or not the webhook is switched on or wants that event, and says how it went (`200`, or what went wrong).
|
||||||
|
- The list shows when the last message was sent and its result — a status code, `timed out`, or the error.
|
||||||
|
- Making, switching and removing a webhook are in the audit log with its **name and events — never its address or secret** (the address can carry a token of its own).
|
||||||
|
|
||||||
|
### Events
|
||||||
|
|
||||||
|
| Event | Sent when |
|
||||||
|
|---|---|
|
||||||
|
| `alert.opened` | A problem begins. |
|
||||||
|
| `alert.resolved` | A problem ends. |
|
||||||
|
| `maintenance.started` | A maintenance window begins — by hand, in bulk, from a recurring schedule, or mirrored from Uptime Kuma. |
|
||||||
|
| `maintenance.ended` | A maintenance window finishes, on its own or ended early. |
|
||||||
|
| `game_server.started` | A game server (Infrastructure → Game servers) is seen turned on — starting or running after being off. |
|
||||||
|
| `game_server.stopped` | A game server is seen turned off — stopping or offline after being on. |
|
||||||
|
|
||||||
|
**Game servers.** The accounts are read every five minutes, so `at` is to the nearest five minutes and a server that is switched off and on again between two reads is not seen at all. *Starting* counts as on and *stopping* as off, so a start is announced once, when it begins, not again when it has finished. A server read for the first time, one being installed or moved, and one whose state is unknown are not announced. Unlike the chat notification, these are **not** held back by the *Game servers turned on or off* switch in Settings → Notifications: a machine hears every change. A receiver that itself sent the power signal can tell its own doing by `signal`/`by`.
|
||||||
|
|
||||||
|
**What counts as an alert here.** `alert.opened` and `alert.resolved` are sent for the alerts that go through the health-style bookkeeping — server offline, disks, failed services, SMART and ZFS problems, new
|
||||||
|
ports, network devices offline, your own [alert rules](MONITORING.md#your-own-alert-rules), endpoint checks, heartbeats, and automation failures — on the **same terms as a notification**: a problem
|
||||||
|
under a maintenance window, or folded under a server that is down, is **not announced until it stands alone**. Two things that *do* hold back a notification do **not** hold back the webhook: a problem
|
||||||
|
being **flapping**, and one that is **snoozed** — a machine should hear every raise and clear. Summaries (an expiring secret, a stale backup, Docker updates, the weekly digest) are not events; read
|
||||||
|
those from the [read API](API.md) or [`/metrics`](PROMETHEUS-GRAFANA.md).
|
||||||
|
|
||||||
|
### The message
|
||||||
|
|
||||||
|
```
|
||||||
|
POST <your address>
|
||||||
|
Content-Type: application/json
|
||||||
|
User-Agent: HomelabManager-Webhook
|
||||||
|
X-HLM-Event: alert.opened
|
||||||
|
X-HLM-Delivery: 5b0c3e0a-1f4e-4a55-9d0e-6c1c1c1c1c1c (the same for a retry of the same message)
|
||||||
|
X-HLM-Signature: sha256=9f2c… (only if the webhook has a secret)
|
||||||
|
```
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"event": "alert.opened",
|
||||||
|
"id": "5b0c3e0a-1f4e-4a55-9d0e-6c1c1c1c1c1c",
|
||||||
|
"createdAt": "2026-10-17T21:04:11.482Z",
|
||||||
|
"source": "homelab-manager",
|
||||||
|
"data": {
|
||||||
|
"alert": {
|
||||||
|
"key": "offline:server:3",
|
||||||
|
"source": "server",
|
||||||
|
"severity": "critical",
|
||||||
|
"message": "pve1 hasn't reported for 1 h — also takes down web and db"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
| `data` | Fields |
|
||||||
|
|---|---|
|
||||||
|
| `alert.opened` | `alert.key` (stable for the same problem — see [Alert keys](#alert-keys)), `alert.source` (where it came from, e.g. `server`, `proxmox:3:pve1`, `endpoint`, `heartbeat`), `alert.severity` (`critical` or `warning`), `alert.message` |
|
||||||
|
| `alert.resolved` | `alert.key`, `alert.message` (what it said when it opened) |
|
||||||
|
| `maintenance.started` | `window`: `id`, `targetType` (`server`, `integration`, `dns_provider`, `device`), `targetId`, `targetName`, `reason`, `startedAt`, `endsAt`, `source` (`null` for one started by hand, `schedule:…` for a recurring one, `kuma:…` for one mirrored from Uptime Kuma) |
|
||||||
|
| `maintenance.ended` | the same `window`, plus `endedAt` — one ended early has its `endsAt` brought forward |
|
||||||
|
| `game_server.started` / `game_server.stopped` | `gameServer`: `id` (the panel's id), `name`, `game`, `integrationId`, `integrationName`, `on` (`true` for started), `at` (when the read saw it), `signal` (`start`, `stop`, `restart` or `kill` when the change followed a power signal sent from this app, else `null`) and `by` (who sent that signal, else `null`) |
|
||||||
|
| `test` | `message` |
|
||||||
|
|
||||||
|
A captured test message, as a receiver sees it (headers lower-cased by the receiving server):
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"headers": {
|
||||||
|
"x-hlm-event": "test",
|
||||||
|
"x-hlm-delivery": "72118185-66dd-49d8-8886-aed8bdc40d61",
|
||||||
|
"x-hlm-signature": "sha256=c10f409aac74524e3bc84c0c062f536aa44b98c3bede082f970dfa496fb81f57",
|
||||||
|
"user-agent": "HomelabManager-Webhook",
|
||||||
|
"content-type": "application/json"
|
||||||
|
},
|
||||||
|
"body": {
|
||||||
|
"event": "test",
|
||||||
|
"id": "72118185-66dd-49d8-8886-aed8bdc40d61",
|
||||||
|
"createdAt": "2026-10-20T21:46:21.349Z",
|
||||||
|
"source": "homelab-manager",
|
||||||
|
"data": { "message": "This is a test from Homelab Manager." }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Checking the signature
|
||||||
|
|
||||||
|
With a secret, `X-HLM-Signature` is `sha256=` followed by the hex **HMAC-SHA256 of the exact body bytes**, with the secret as the key. Compute it over the **raw body as received** (not over parsed and
|
||||||
|
re-serialised JSON), compare in **constant time**, and reject a message that does not match.
|
||||||
|
|
||||||
|
```js
|
||||||
|
import { createHmac, timingSafeEqual } from "node:crypto";
|
||||||
|
|
||||||
|
export function isFromHomelabManager(secret, rawBody, signatureHeader) {
|
||||||
|
const expected = Buffer.from(`sha256=${createHmac("sha256", secret).update(rawBody).digest("hex")}`);
|
||||||
|
const given = Buffer.from(signatureHeader ?? "");
|
||||||
|
return given.length === expected.length && timingSafeEqual(given, expected);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
```python
|
||||||
|
import hmac, hashlib
|
||||||
|
|
||||||
|
def is_from_homelab_manager(secret: str, raw_body: bytes, signature_header: str) -> bool:
|
||||||
|
expected = "sha256=" + hmac.new(secret.encode(), raw_body, hashlib.sha256).hexdigest()
|
||||||
|
return hmac.compare_digest(expected, signature_header or "")
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Quick check at a shell, with the body saved to a file:
|
||||||
|
printf 'sha256=%s\n' "$(openssl dgst -sha256 -hmac "$SECRET" -hex < body.json | sed 's/^.* //')"
|
||||||
|
```
|
||||||
|
|
||||||
|
The signature proves the message came from someone with the secret; it does **not** stop a captured message being sent again later. If that matters, remember the `id` (or `X-HLM-Delivery`) of what you
|
||||||
|
have handled and refuse repeats, and reject a `createdAt` that is far from your clock.
|
||||||
|
|
||||||
|
### Delivery: what to rely on and what not to
|
||||||
|
|
||||||
|
- Delivery is **best-effort and never holds anything up.** One try; if it does not get a `2xx` answer, **one more after two seconds** (the same message, the same `id` and `X-HLM-Delivery`); then the
|
||||||
|
result is written on the webhook and in its [delivery log](#the-delivery-log-and-sending-again).
|
||||||
|
- A request is given **five seconds**. **Redirects are not followed** (a redirect counts as a failure) — give the final address.
|
||||||
|
- **There is no queue and no automatic replay.** A receiver that is down for longer than those two tries **misses the event** — until someone sends it again from the delivery log. Messages for different events can arrive in any order. If you need to be certain, treat the
|
||||||
|
webhook as a *nudge* and reconcile with the [read API](API.md) (`GET /api/v1/alerts`, `/api/v1/maintenance`), which always says what is true now.
|
||||||
|
- Because a retry reuses the delivery id, **handle each id once** (idempotently).
|
||||||
|
- Answer quickly (`200` is enough) and do slow work afterwards.
|
||||||
|
|
||||||
|
### The delivery log and sending again
|
||||||
|
|
||||||
|
Every message sent to a webhook is written down: when, which event, whether it got through, what came back (`200`, or what went wrong, such as `connect ECONNREFUSED` or `500`), whether it needed its second try, and the
|
||||||
|
message itself. **Settings → API and webhooks → Event webhooks → Sent** opens the log for one webhook, newest first, and **Show the message** unfolds the JSON that was sent.
|
||||||
|
|
||||||
|
**Send again** sends a message from the log **exactly as it was** — the same bytes, the same `id` and `X-HLM-Delivery`, a signature made over the same body — so a receiver that already handled it can recognise it by its id and
|
||||||
|
ignore it, and one that missed it handles it now. The new attempt is logged as a delivery of its own, marked *sent again by hand*, and updates the webhook's *Last* result. Like the first send it is tried twice.
|
||||||
|
It is for when the receiver was down or the fault was at its end; it does not repair a wrong address.
|
||||||
|
|
||||||
|
- **The last 100 per webhook, and none older than 14 days,** are kept. Removing a webhook removes its log.
|
||||||
|
- A message larger than **16 000 characters** is not kept in full, so it is not shown and cannot be sent again (`409 too_large`). The messages the app sends are far smaller.
|
||||||
|
- **Administrators only** — the log holds what was sent, which names servers and says what is wrong with them. Each resend is in the audit log (*webhook › retry*) and in the **Change log**.
|
||||||
|
|
||||||
|
```
|
||||||
|
GET /api/event-webhooks/<id>/deliveries?limit=50 the log (limit 1–100)
|
||||||
|
POST /api/event-webhooks/<id>/deliveries/<delivery row id>/retry send one again → { "ok": true, "result": "200" }
|
||||||
|
```
|
||||||
|
|
||||||
|
These are the app's own addresses (they need an administrator's sign-in, not an API token), listed for completeness.
|
||||||
|
|
||||||
|
### Alert keys
|
||||||
|
|
||||||
|
`alert.key` is the stable identity of a problem: the same problem has the same key every time, so you can match an `opened` with its `resolved`. The shapes:
|
||||||
|
|
||||||
|
| Key | About |
|
||||||
|
|---|---|
|
||||||
|
| `offline:server:<id>` | A server has stopped reporting |
|
||||||
|
| `offline:device:<id>` | A UniFi device has been offline too long |
|
||||||
|
| `disk:server:<id>:<mount>` | A disk is nearly full |
|
||||||
|
| `disk:proxmox:<integration>:…`, `disk:synology:<integration>:<volume>` | A Proxmox storage or Synology volume is nearly full |
|
||||||
|
| `synology-volume:<integration>:<volume>`, `synology-disk:<integration>:<disk>` | A Synology volume or disk is not healthy |
|
||||||
|
| `failed-units:server:<id>` | Services have failed on a server |
|
||||||
|
| `smart:server:<id>:<device>`, `zfs:server:<id>:<pool>` | A disk says it is failing, a pool is not online |
|
||||||
|
| `new-port:server:<id>:<proto>:<port>` | A new TCP port started listening |
|
||||||
|
| `rule:<rule id>:server:<id>[:<disk>]` | One of your own [alert rules](MONITORING.md#your-own-alert-rules) |
|
||||||
|
| `endpoint:<check id>`, `endpoint-cert:<check id>` | An endpoint check is failing, or its certificate is running out |
|
||||||
|
| `heartbeat:<id>`, `heartbeat-fail:<id>` | A job reported late, or said it failed |
|
||||||
|
| `automation:semaphore:…`, `automation:gitea:…` | A Semaphore task or a Gitea Actions run failed |
|
||||||
|
|
||||||
|
Treat unknown shapes as opaque; match on the whole key.
|
||||||
|
|
||||||
|
### A small receiver
|
||||||
|
|
||||||
|
Node with Express — the one thing to get right is to verify the signature against the **raw** body:
|
||||||
|
|
||||||
|
```js
|
||||||
|
import express from "express";
|
||||||
|
import { createHmac, timingSafeEqual } from "node:crypto";
|
||||||
|
|
||||||
|
const SECRET = process.env.HLM_WEBHOOK_SECRET;
|
||||||
|
const seen = new Set(); // use something persistent for real
|
||||||
|
|
||||||
|
const app = express();
|
||||||
|
app.post("/hlm", express.raw({ type: "application/json" }), (req, res) => {
|
||||||
|
const expected = Buffer.from("sha256=" + createHmac("sha256", SECRET).update(req.body).digest("hex"));
|
||||||
|
const given = Buffer.from(req.get("x-hlm-signature") ?? "");
|
||||||
|
if (given.length !== expected.length || !timingSafeEqual(given, expected)) return res.sendStatus(401);
|
||||||
|
|
||||||
|
const msg = JSON.parse(req.body.toString("utf8"));
|
||||||
|
if (seen.has(msg.id)) return res.sendStatus(200); // a retry of something already handled
|
||||||
|
seen.add(msg.id);
|
||||||
|
|
||||||
|
if (msg.event === "alert.opened") console.log(`[${msg.data.alert.severity}] ${msg.data.alert.message}`);
|
||||||
|
else if (msg.event === "alert.resolved") console.log(`resolved: ${msg.data.alert.message}`);
|
||||||
|
res.sendStatus(200);
|
||||||
|
});
|
||||||
|
app.listen(8080);
|
||||||
|
```
|
||||||
|
|
||||||
|
Python with Flask:
|
||||||
|
|
||||||
|
```python
|
||||||
|
import hmac, hashlib, os
|
||||||
|
from flask import Flask, request, abort
|
||||||
|
|
||||||
|
SECRET = os.environ["HLM_WEBHOOK_SECRET"].encode()
|
||||||
|
app = Flask(__name__)
|
||||||
|
|
||||||
|
@app.post("/hlm")
|
||||||
|
def hook():
|
||||||
|
raw = request.get_data() # the exact bytes
|
||||||
|
expected = "sha256=" + hmac.new(SECRET, raw, hashlib.sha256).hexdigest()
|
||||||
|
if not hmac.compare_digest(expected, request.headers.get("X-HLM-Signature", "")):
|
||||||
|
abort(401)
|
||||||
|
msg = request.get_json()
|
||||||
|
if msg["event"] == "alert.opened":
|
||||||
|
print(msg["data"]["alert"]["severity"], msg["data"]["alert"]["message"])
|
||||||
|
return "", 200
|
||||||
|
```
|
||||||
|
|
||||||
|
### Home Assistant and n8n
|
||||||
|
|
||||||
|
- **Home Assistant** — an automation with a *webhook trigger* can take the message as it comes. Home Assistant's webhook URL has an unguessable id in it (`/api/webhook/<id>`), which is the access control, so a
|
||||||
|
secret here is optional; if you want the signature, check it in a small receiver like the ones above and forward.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
automation:
|
||||||
|
- alias: "Homelab Manager critical alert"
|
||||||
|
trigger:
|
||||||
|
- platform: webhook
|
||||||
|
webhook_id: homelab_manager_alerts # then the address is https://ha.example.lan/api/webhook/homelab_manager_alerts
|
||||||
|
allowed_methods: [POST]
|
||||||
|
local_only: true
|
||||||
|
condition: "{{ trigger.json.event == 'alert.opened' and trigger.json.data.alert.severity == 'critical' }}"
|
||||||
|
action:
|
||||||
|
- service: notify.mobile_app_phone
|
||||||
|
data:
|
||||||
|
title: "Homelab alert"
|
||||||
|
message: "{{ trigger.json.data.alert.message }}"
|
||||||
|
```
|
||||||
|
|
||||||
|
- **n8n** — a *Webhook* node (method POST) gives you the address; add a *Code* or *IF* node on `{{$json.body.event}}`. To verify the signature in n8n, enable the node's *Raw Body* option and compute the HMAC
|
||||||
|
in a Code node as above.
|
||||||
|
|
||||||
|
## The notification webhook
|
||||||
|
|
||||||
|
A notification channel (beside Gotify, ntfy, Telegram, Pushover, email and push) posts each notification to a URL you give it, as the sentence a person would read — handy for chat tools that accept an incoming-webhook post.
|
||||||
|
|
||||||
|
**Settings → Notifications → Channels → Webhook**: switch it on, give the address, and optionally a secret.
|
||||||
|
|
||||||
|
```
|
||||||
|
POST <your address>
|
||||||
|
Content-Type: application/json
|
||||||
|
X-Webhook-Secret: <the secret> (only if one is set)
|
||||||
|
|
||||||
|
{ "content": "**Homelab Manager — Health Alert**\n1 new issue:\n\n⚠ pve1 hasn't reported for 1 h" }
|
||||||
|
```
|
||||||
|
|
||||||
|
- The body is exactly `{ "content": "<title in bold>\n<message>" }` — the shape Discord's incoming webhooks take (Slack and Mattermost expect `text`, so they need a small adapter in between).
|
||||||
|
- `X-Webhook-Secret` is the secret **as given**, not a signature — it only lets a receiver reject strangers. For something verifiable, use an event webhook.
|
||||||
|
- It follows everything a notification follows: the **routing rules** (which channels hear what), **quiet hours**, **maintenance**, **flapping** and **snooze**, and the toggle for each kind of notification. It
|
||||||
|
is delivered once with no retry; a failure is written to the server log.
|
||||||
|
- The secret is stored encrypted when `CREDENTIALS_ENCRYPTION_KEY` is set. The **Send test** button on the channel sends a one-off message through that channel only.
|
||||||
|
- Which messages exist, and when each is sent, is in [NOTIFICATIONS.md](../NOTIFICATIONS.md).
|
||||||
|
|
||||||
|
## Buttons on notifications
|
||||||
|
|
||||||
|
A notification can carry buttons that act on the alert it is about, from the phone's lock screen, without signing in: **Acknowledge**, **Silence for an hour** and **Snooze until tomorrow at 09:00**. An administrator switches
|
||||||
|
it on under **Settings → Notifications → Who hears what → Buttons on notifications**; it is off until then, and off means no link works. How each channel shows them is in [NOTIFICATIONS.md](../NOTIFICATIONS.md#buttons-on-notifications).
|
||||||
|
|
||||||
|
The addresses behind them, for anything that wants to build on it:
|
||||||
|
|
||||||
|
```
|
||||||
|
GET /api/alert-actions/<token> a page listing the alerts with a button for each action — opening it changes nothing
|
||||||
|
POST /api/alert-actions/<token>/ack acknowledge ("Acknowledged from a notification", until the alert is gone)
|
||||||
|
POST /api/alert-actions/<token>/silence start an hour of maintenance on what the alert is about (only if it is about something a window can silence)
|
||||||
|
POST /api/alert-actions/<token>/snooze snooze until tomorrow 09:00 in the notification time zone
|
||||||
|
add ?key=<alert key> (or a form field "key") when the token is for more than one alert
|
||||||
|
```
|
||||||
|
|
||||||
|
```json
|
||||||
|
{ "ok": true, "message": "Acknowledged." }
|
||||||
|
```
|
||||||
|
|
||||||
|
- **The token is the access.** It is signed (HMAC-SHA256, with a key derived from `SESSION_SECRET`), is for the **alerts named in it only** (the alert keys of that one message, up to 20), and **ends after 24 hours**. Nothing
|
||||||
|
is stored. A forged, altered or ended link is `404`/`410`. A `key` that is not in the token is `400`; an alert that has cleared is `404` ("it has cleared"); a silence for something with nothing to silence it with is `409`.
|
||||||
|
- **It can do these three things and nothing else** — no reading, no settings, no other alert. Everything it does is in the audit log, said to be done *from a notification link*, with the IP address it came from.
|
||||||
|
- **A POST from a browser form gets a page back; anything else gets JSON.** `GET` never acts, so a link preview, a chat unfurl or a mail scanner cannot press a button. Limited to 60 requests a minute from one address.
|
||||||
|
- **Anyone who can read the notification can use its buttons** for the 24 hours. That is the trade for not having to sign in: keep the ntfy topic private (a self-hosted ntfy, or a topic with a password), do not point this at a public
|
||||||
|
topic, and remember the link is in an email's text. Changing `SESSION_SECRET` or switching the setting off ends every link at once.
|
||||||
|
- The links are built from `APP_BASE_URL`, so it has to be an address your phone can open.
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
| Symptom | Cause |
|
||||||
|
|---|---|
|
||||||
|
| **Test** says `timed out` | The receiver did not answer in five seconds, or is not reachable *from the app* (containers: `localhost` is the container itself). |
|
||||||
|
| **Test** says `401`/`403` | Your receiver rejected it — often a signature check against the wrong secret, or one computed over parsed rather than raw JSON. |
|
||||||
|
| **Test** says `301`/`302`/`308` | Redirects are not followed. Use the final address (`https://` instead of `http://`, the trailing slash). |
|
||||||
|
| **Test** works but real events never arrive | The webhook is switched off, or does not have that event ticked, or the alert is under maintenance / folded under a host (not announced until it stands alone). |
|
||||||
|
| The same alert arrives twice | A retry — the `id` is the same. Handle each `id` once. |
|
||||||
|
| An alert I snoozed or that is flapping still arrived | By design: machines hear every raise and clear. |
|
||||||
|
| A secret about to expire / a stale backup did not produce an event | Summaries are not events. Read them from `/api/v1` or `/metrics`. |
|
||||||
|
| I lost the secret | It cannot be shown again. Remove the webhook and make a new one. |
|
||||||
Generated
+173
-2
@@ -1805,6 +1805,12 @@
|
|||||||
"win32"
|
"win32"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
"node_modules/@socket.io/component-emitter": {
|
||||||
|
"version": "3.1.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/@socket.io/component-emitter/-/component-emitter-3.1.2.tgz",
|
||||||
|
"integrity": "sha512-9BCxFwvbGg/RsZK9tjXd8s4UcwR0MWeFQ1XEKIQVVvAGJyINdrqKMcTRyLoK8Rse1GjzLV9cwjWV1olXRWEXVA==",
|
||||||
|
"license": "MIT"
|
||||||
|
},
|
||||||
"node_modules/@tabler/core": {
|
"node_modules/@tabler/core": {
|
||||||
"version": "1.5.1",
|
"version": "1.5.1",
|
||||||
"resolved": "https://registry.npmjs.org/@tabler/core/-/core-1.5.1.tgz",
|
"resolved": "https://registry.npmjs.org/@tabler/core/-/core-1.5.1.tgz",
|
||||||
@@ -1913,6 +1919,16 @@
|
|||||||
"@types/node": "*"
|
"@types/node": "*"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/@types/cors": {
|
||||||
|
"version": "2.8.19",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.19.tgz",
|
||||||
|
"integrity": "sha512-mFNylyeyqN93lfe/9CSxOGREz8cpzAhH+E93xJ4xWQf62V8sQ/24reV2nyzUWM6H6Xji+GGHpkbLe7pVoUEskg==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"@types/node": "*"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/@types/estree": {
|
"node_modules/@types/estree": {
|
||||||
"version": "1.0.9",
|
"version": "1.0.9",
|
||||||
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz",
|
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz",
|
||||||
@@ -2372,6 +2388,24 @@
|
|||||||
"integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==",
|
"integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==",
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
|
"node_modules/cors": {
|
||||||
|
"version": "2.8.6",
|
||||||
|
"resolved": "https://registry.npmjs.org/cors/-/cors-2.8.6.tgz",
|
||||||
|
"integrity": "sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"object-assign": "^4",
|
||||||
|
"vary": "^1"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">= 0.10"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"type": "opencollective",
|
||||||
|
"url": "https://opencollective.com/express"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/cron-parser": {
|
"node_modules/cron-parser": {
|
||||||
"version": "5.10.1",
|
"version": "5.10.1",
|
||||||
"resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-5.10.1.tgz",
|
"resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-5.10.1.tgz",
|
||||||
@@ -2413,7 +2447,6 @@
|
|||||||
"version": "4.4.3",
|
"version": "4.4.3",
|
||||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
|
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
|
||||||
"integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==",
|
"integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==",
|
||||||
"dev": true,
|
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"ms": "^2.1.3"
|
"ms": "^2.1.3"
|
||||||
@@ -2644,6 +2677,49 @@
|
|||||||
"node": ">= 0.8"
|
"node": ">= 0.8"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/engine.io": {
|
||||||
|
"version": "6.6.11",
|
||||||
|
"resolved": "https://registry.npmjs.org/engine.io/-/engine.io-6.6.11.tgz",
|
||||||
|
"integrity": "sha512-z34mGpLoY/82ZbkX1LqGMIaf3wjVMPL6gVC681QDqnmzJ26synI8qAE32P+a7UazmpJZ8UGrFIeAavgRfXzYrg==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"@types/cors": "^2.8.12",
|
||||||
|
"@types/node": ">=10.0.0",
|
||||||
|
"@types/ws": "^8.5.12",
|
||||||
|
"accepts": "~1.3.4",
|
||||||
|
"cookie": "~0.7.2",
|
||||||
|
"cors": "~2.8.5",
|
||||||
|
"debug": "~4.4.1",
|
||||||
|
"engine.io-parser": "~5.2.1",
|
||||||
|
"ws": "~8.21.0"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=10.2.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/engine.io-client": {
|
||||||
|
"version": "6.6.7",
|
||||||
|
"resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-6.6.7.tgz",
|
||||||
|
"integrity": "sha512-GyAvmPA7gnRCpDQbYACAy79Lg4ewevrmaPzySZz0OVvgyZc3gEfLLqPTYk8wVBScjOEpM9ZO4KryyFpVItjGEA==",
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"@socket.io/component-emitter": "~3.1.0",
|
||||||
|
"debug": "~4.4.1",
|
||||||
|
"engine.io-parser": "~5.2.1",
|
||||||
|
"ws": "~8.21.0",
|
||||||
|
"xmlhttprequest-ssl": "~2.1.1"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/engine.io-parser": {
|
||||||
|
"version": "5.2.3",
|
||||||
|
"resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-5.2.3.tgz",
|
||||||
|
"integrity": "sha512-HqD3yTBfnBxIrbnM1DoD6Pcq8NECnh8d4As1Qgh0z5Gg3jRRIqijury0CL3ghu/edArpUYiYqQiDUQBIs4np3Q==",
|
||||||
|
"license": "MIT",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=10.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/es-define-property": {
|
"node_modules/es-define-property": {
|
||||||
"version": "1.0.1",
|
"version": "1.0.1",
|
||||||
"resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
|
"resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
|
||||||
@@ -3705,6 +3781,12 @@
|
|||||||
"node": ">= 0.10"
|
"node": ">= 0.10"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/qrcode-generator": {
|
||||||
|
"version": "2.0.4",
|
||||||
|
"resolved": "https://registry.npmjs.org/qrcode-generator/-/qrcode-generator-2.0.4.tgz",
|
||||||
|
"integrity": "sha512-mZSiP6RnbHl4xL2Ap5HfkjLnmxfKcPWpWe/c+5XxCuetEenqmNFf1FH/ftXPCtFG5/TDobjsjz6sSNL0Sr8Z9g==",
|
||||||
|
"license": "MIT"
|
||||||
|
},
|
||||||
"node_modules/qs": {
|
"node_modules/qs": {
|
||||||
"version": "6.16.0",
|
"version": "6.16.0",
|
||||||
"resolved": "https://registry.npmjs.org/qs/-/qs-6.16.0.tgz",
|
"resolved": "https://registry.npmjs.org/qs/-/qs-6.16.0.tgz",
|
||||||
@@ -4124,6 +4206,82 @@
|
|||||||
"integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==",
|
"integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==",
|
||||||
"license": "ISC"
|
"license": "ISC"
|
||||||
},
|
},
|
||||||
|
"node_modules/simple-icons": {
|
||||||
|
"version": "16.37.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/simple-icons/-/simple-icons-16.37.0.tgz",
|
||||||
|
"integrity": "sha512-8mCJIhpMYOL9g13GW59EiL3sVXJnTEKGCTZFaekl9UQ3Qo0SwVjnLaJ82QiaH8BKBdouEDzrHMXIWh8cBIy48A==",
|
||||||
|
"funding": [
|
||||||
|
{
|
||||||
|
"type": "opencollective",
|
||||||
|
"url": "https://opencollective.com/simple-icons"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "github",
|
||||||
|
"url": "https://github.com/sponsors/simple-icons"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"license": "CC0-1.0",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=0.12.18"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/socket.io": {
|
||||||
|
"version": "4.8.4",
|
||||||
|
"resolved": "https://registry.npmjs.org/socket.io/-/socket.io-4.8.4.tgz",
|
||||||
|
"integrity": "sha512-9ZV9zg8vN8hO3Y25DhR7nUkJ1VDnBhq5O/JrXhFT9s6+F0gqytSASTHVfCFc2mJXaTfARroErplXlzFWwETZqQ==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"accepts": "~1.3.4",
|
||||||
|
"cors": "~2.8.5",
|
||||||
|
"debug": "~4.4.1",
|
||||||
|
"engine.io": "~6.6.0",
|
||||||
|
"socket.io-adapter": "~2.5.2",
|
||||||
|
"socket.io-parser": "~4.2.4"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=10.2.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/socket.io-adapter": {
|
||||||
|
"version": "2.5.8",
|
||||||
|
"resolved": "https://registry.npmjs.org/socket.io-adapter/-/socket.io-adapter-2.5.8.tgz",
|
||||||
|
"integrity": "sha512-6Oy52pbg+kvdCVvjcN+FnY7BvxZ7cIHNScbvztT/It5d0vbwoJoVZmF2gjJmnV0/4WlXRfG15zc45ySk9Ah8bw==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"debug": "~4.4.1",
|
||||||
|
"ws": "~8.21.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/socket.io-client": {
|
||||||
|
"version": "4.8.4",
|
||||||
|
"resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-4.8.4.tgz",
|
||||||
|
"integrity": "sha512-7wjKAvVXYOJsbmUgkWbnhPD4bFpkqrUR0++XOjpGFmqDqsqV1CI93VTraeExLxpjxEqrWgFoSRZu1ihjLWfkAA==",
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"@socket.io/component-emitter": "~3.1.0",
|
||||||
|
"debug": "~4.4.1",
|
||||||
|
"engine.io-client": "~6.6.1",
|
||||||
|
"socket.io-parser": "~4.2.4"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=10.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/socket.io-parser": {
|
||||||
|
"version": "4.2.7",
|
||||||
|
"resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.2.7.tgz",
|
||||||
|
"integrity": "sha512-IH/iSeO9T6gz1KkFleGDWkG9N3dl4jXVYUtMhIqH10Md0ttMer8nUNWiP1DKuNrybD2xBrixLJdCC9J6ECoYkg==",
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"@socket.io/component-emitter": "~3.1.0",
|
||||||
|
"debug": "~4.4.1"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=10.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/sorted-array-functions": {
|
"node_modules/sorted-array-functions": {
|
||||||
"version": "1.3.0",
|
"version": "1.3.0",
|
||||||
"resolved": "https://registry.npmjs.org/sorted-array-functions/-/sorted-array-functions-1.3.0.tgz",
|
"resolved": "https://registry.npmjs.org/sorted-array-functions/-/sorted-array-functions-1.3.0.tgz",
|
||||||
@@ -4978,6 +5136,14 @@
|
|||||||
"node": ">=4.0"
|
"node": ">=4.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/xmlhttprequest-ssl": {
|
||||||
|
"version": "2.1.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-2.1.2.tgz",
|
||||||
|
"integrity": "sha512-TEU+nJVUUnA4CYJFLvK5X9AOeH4KvDvhIfm0vV1GaQRtchnG0hgK5p8hw/xjv8cunWYCsiPCSDzObPyhEwq3KQ==",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=0.4.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/yallist": {
|
"node_modules/yallist": {
|
||||||
"version": "3.1.1",
|
"version": "3.1.1",
|
||||||
"resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
|
"resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
|
||||||
@@ -5002,11 +5168,13 @@
|
|||||||
"drizzle-orm": "^0.45.2",
|
"drizzle-orm": "^0.45.2",
|
||||||
"express": "^4.21.2",
|
"express": "^4.21.2",
|
||||||
"express-session": "^1.18.1",
|
"express-session": "^1.18.1",
|
||||||
|
"jose": "^6.2.12",
|
||||||
"mysql2": "^3.24.5",
|
"mysql2": "^3.24.5",
|
||||||
"node-schedule": "^2.1.1",
|
"node-schedule": "^2.1.1",
|
||||||
"nodemailer": "^6.9.14",
|
"nodemailer": "^6.9.14",
|
||||||
"openid-client": "^6.1.7",
|
"openid-client": "^6.1.7",
|
||||||
"session-file-store": "^1.5.0",
|
"session-file-store": "^1.5.0",
|
||||||
|
"socket.io-client": "^4.8.4",
|
||||||
"xml2js": "^0.6.2",
|
"xml2js": "^0.6.2",
|
||||||
"zod": "^3.24.1"
|
"zod": "^3.24.1"
|
||||||
},
|
},
|
||||||
@@ -5019,6 +5187,7 @@
|
|||||||
"@types/session-file-store": "^1.2.5",
|
"@types/session-file-store": "^1.2.5",
|
||||||
"@types/xml2js": "^0.4.14",
|
"@types/xml2js": "^0.4.14",
|
||||||
"drizzle-kit": "^0.31.10",
|
"drizzle-kit": "^0.31.10",
|
||||||
|
"socket.io": "^4.8.4",
|
||||||
"tsx": "^4.19.2",
|
"tsx": "^4.19.2",
|
||||||
"typescript": "^5.7.3"
|
"typescript": "^5.7.3"
|
||||||
}
|
}
|
||||||
@@ -5030,9 +5199,11 @@
|
|||||||
"@tabler/icons-react": "^3.46.0",
|
"@tabler/icons-react": "^3.46.0",
|
||||||
"cron-parser": "^5.10.0",
|
"cron-parser": "^5.10.0",
|
||||||
"cronstrue": "^2.53.0",
|
"cronstrue": "^2.53.0",
|
||||||
|
"qrcode-generator": "^2.0.4",
|
||||||
"react": "^18.3.1",
|
"react": "^18.3.1",
|
||||||
"react-dom": "^18.3.1",
|
"react-dom": "^18.3.1",
|
||||||
"react-router-dom": "^7.1.1"
|
"react-router-dom": "^7.1.1",
|
||||||
|
"simple-icons": "^16.37.0"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/react": "^18.3.18",
|
"@types/react": "^18.3.18",
|
||||||
|
|||||||
@@ -0,0 +1,18 @@
|
|||||||
|
CREATE TABLE `alert_tickets` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`alert_key` text NOT NULL,
|
||||||
|
`status` text NOT NULL,
|
||||||
|
`seen_count` integer DEFAULT 1 NOT NULL,
|
||||||
|
`integration_id` integer,
|
||||||
|
`ticket_number` text,
|
||||||
|
`grouped` integer DEFAULT false NOT NULL,
|
||||||
|
`severity` text NOT NULL,
|
||||||
|
`message` text NOT NULL,
|
||||||
|
`first_seen_at` text NOT NULL,
|
||||||
|
`last_seen_at` text NOT NULL,
|
||||||
|
`ticketed_at` text,
|
||||||
|
`cleared_at` text,
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE set null
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX `alert_tickets_key_idx` ON `alert_tickets` (`alert_key`);
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
PRAGMA foreign_keys=OFF;--> statement-breakpoint
|
||||||
|
CREATE TABLE `__new_secrets` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`name` text NOT NULL,
|
||||||
|
`type` text DEFAULT 'generic' NOT NULL,
|
||||||
|
`description` text,
|
||||||
|
`expiry_date` text,
|
||||||
|
`warn_days` integer DEFAULT 30 NOT NULL,
|
||||||
|
`notes` text,
|
||||||
|
`check_host` text,
|
||||||
|
`check_port` integer,
|
||||||
|
`last_checked_at` text,
|
||||||
|
`last_check_error` text,
|
||||||
|
`created_at` text DEFAULT (current_timestamp) NOT NULL,
|
||||||
|
`updated_at` text DEFAULT (current_timestamp) NOT NULL
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
INSERT INTO `__new_secrets`("id", "name", "type", "description", "expiry_date", "warn_days", "notes", "check_host", "check_port", "last_checked_at", "last_check_error", "created_at", "updated_at") SELECT "id", "name", "type", "description", "expiry_date", "warn_days", "notes", "check_host", "check_port", "last_checked_at", "last_check_error", "created_at", "updated_at" FROM `secrets`;--> statement-breakpoint
|
||||||
|
DROP TABLE `secrets`;--> statement-breakpoint
|
||||||
|
ALTER TABLE `__new_secrets` RENAME TO `secrets`;--> statement-breakpoint
|
||||||
|
PRAGMA foreign_keys=ON;
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
ALTER TABLE `maintenance_windows` ADD `source` text;
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
CREATE TABLE `maintenance_aliases` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`address` text NOT NULL,
|
||||||
|
`target_type` text NOT NULL,
|
||||||
|
`target_id` integer NOT NULL,
|
||||||
|
`created_at` text DEFAULT (current_timestamp) NOT NULL
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX `maintenance_aliases_unique_idx` ON `maintenance_aliases` (`address`,`target_type`,`target_id`);
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
CREATE TABLE `server_metrics` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`server_id` integer NOT NULL,
|
||||||
|
`metric` text NOT NULL,
|
||||||
|
`value` real NOT NULL,
|
||||||
|
`recorded_at` text NOT NULL,
|
||||||
|
FOREIGN KEY (`server_id`) REFERENCES `servers`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX `server_metrics_lookup_idx` ON `server_metrics` (`server_id`,`metric`,`recorded_at`);
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
ALTER TABLE `users` ADD `favorites` text;
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
ALTER TABLE `users` ADD `last_looked_at` text;
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
ALTER TABLE `users` ADD `dashboard_layout` text;
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
CREATE TABLE `share_links` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`label` text NOT NULL,
|
||||||
|
`token_hash` text NOT NULL,
|
||||||
|
`token_prefix` text NOT NULL,
|
||||||
|
`detail` text DEFAULT 'summary' NOT NULL,
|
||||||
|
`created_by` text,
|
||||||
|
`created_at` text DEFAULT (current_timestamp) NOT NULL,
|
||||||
|
`expires_at` text,
|
||||||
|
`last_used_at` text
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX `share_links_token_hash_unique` ON `share_links` (`token_hash`);
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
ALTER TABLE `servers` ADD `parent_server_id` integer;--> statement-breakpoint
|
||||||
|
ALTER TABLE `servers` ADD `pending_updates` text;--> statement-breakpoint
|
||||||
|
ALTER TABLE `servers` ADD `updates_pending_since` text;
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
CREATE TABLE `maintenance_schedules` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`name` text NOT NULL,
|
||||||
|
`reason` text,
|
||||||
|
`targets` text NOT NULL,
|
||||||
|
`days` text NOT NULL,
|
||||||
|
`start_time` text NOT NULL,
|
||||||
|
`duration_minutes` integer NOT NULL,
|
||||||
|
`timezone` text NOT NULL,
|
||||||
|
`enabled` integer DEFAULT true NOT NULL,
|
||||||
|
`created_by` text,
|
||||||
|
`created_at` text DEFAULT (current_timestamp) NOT NULL
|
||||||
|
);
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
ALTER TABLE `servers` ADD `agent_version` integer;
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
CREATE TABLE `assets` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`name` text NOT NULL,
|
||||||
|
`kind` text DEFAULT 'other' NOT NULL,
|
||||||
|
`manufacturer` text,
|
||||||
|
`model` text,
|
||||||
|
`serial` text,
|
||||||
|
`purchase_date` text,
|
||||||
|
`purchase_price` real,
|
||||||
|
`warranty_until` text,
|
||||||
|
`location` text,
|
||||||
|
`rack_unit` text,
|
||||||
|
`power_watts` integer,
|
||||||
|
`server_id` integer,
|
||||||
|
`notes` text,
|
||||||
|
`created_at` text NOT NULL,
|
||||||
|
`updated_at` text NOT NULL
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE TABLE `event_webhooks` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`label` text NOT NULL,
|
||||||
|
`url` text NOT NULL,
|
||||||
|
`secret` text,
|
||||||
|
`events` text NOT NULL,
|
||||||
|
`enabled` integer DEFAULT true NOT NULL,
|
||||||
|
`created_by` text,
|
||||||
|
`created_at` text DEFAULT (current_timestamp) NOT NULL,
|
||||||
|
`last_attempt_at` text,
|
||||||
|
`last_result` text
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE TABLE `metrics_tokens` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`label` text NOT NULL,
|
||||||
|
`token_hash` text NOT NULL,
|
||||||
|
`token_prefix` text NOT NULL,
|
||||||
|
`created_by` text,
|
||||||
|
`created_at` text DEFAULT (current_timestamp) NOT NULL,
|
||||||
|
`last_used_at` text
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX `metrics_tokens_token_hash_unique` ON `metrics_tokens` (`token_hash`);--> statement-breakpoint
|
||||||
|
CREATE TABLE `runbooks` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`target_type` text NOT NULL,
|
||||||
|
`target_id` integer NOT NULL,
|
||||||
|
`title` text NOT NULL,
|
||||||
|
`body` text NOT NULL,
|
||||||
|
`category` text,
|
||||||
|
`updated_by` text,
|
||||||
|
`created_at` text NOT NULL,
|
||||||
|
`updated_at` text NOT NULL
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX `runbooks_target_idx` ON `runbooks` (`target_type`,`target_id`);--> statement-breakpoint
|
||||||
|
ALTER TABLE `users` ADD `groups` text;
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
ALTER TABLE `servers` ADD `no_parent` integer DEFAULT false NOT NULL;
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
ALTER TABLE `servers` ADD `host_health` text;
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
ALTER TABLE `servers` ADD `thresholds` text;
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
CREATE TABLE `update_runs` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`server_id` integer NOT NULL,
|
||||||
|
`integration_id` integer NOT NULL,
|
||||||
|
`project_id` integer NOT NULL,
|
||||||
|
`template_id` integer NOT NULL,
|
||||||
|
`template_name` text NOT NULL,
|
||||||
|
`host_limit` text NOT NULL,
|
||||||
|
`task_id` integer,
|
||||||
|
`task_status` text,
|
||||||
|
`window_id` integer,
|
||||||
|
`window_created` integer DEFAULT false NOT NULL,
|
||||||
|
`status` text NOT NULL,
|
||||||
|
`message` text,
|
||||||
|
`started_by` text,
|
||||||
|
`started_at` text NOT NULL,
|
||||||
|
`task_ended_at` text,
|
||||||
|
`host_reported_at` text,
|
||||||
|
`finished_at` text,
|
||||||
|
`updates_before` integer,
|
||||||
|
`updates_after` integer,
|
||||||
|
`reboot_required` integer DEFAULT false NOT NULL,
|
||||||
|
`poll_errors` integer DEFAULT 0 NOT NULL,
|
||||||
|
FOREIGN KEY (`server_id`) REFERENCES `servers`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX `update_runs_server_idx` ON `update_runs` (`server_id`,`started_at`);--> statement-breakpoint
|
||||||
|
CREATE INDEX `update_runs_status_idx` ON `update_runs` (`status`);
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
CREATE TABLE `alert_acks` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`alert_key` text NOT NULL,
|
||||||
|
`note` text DEFAULT '' NOT NULL,
|
||||||
|
`assignee_user_id` integer,
|
||||||
|
`assignee_name` text,
|
||||||
|
`acked_by` text NOT NULL,
|
||||||
|
`acked_at` text NOT NULL,
|
||||||
|
`until` text,
|
||||||
|
`last_seen_at` text NOT NULL,
|
||||||
|
FOREIGN KEY (`assignee_user_id`) REFERENCES `users`(`id`) ON UPDATE no action ON DELETE set null
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX `alert_acks_key_idx` ON `alert_acks` (`alert_key`);--> statement-breakpoint
|
||||||
|
CREATE TABLE `alert_incidents` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`alert_key` text NOT NULL,
|
||||||
|
`severity` text NOT NULL,
|
||||||
|
`peak_severity` text NOT NULL,
|
||||||
|
`category` text NOT NULL,
|
||||||
|
`source` text NOT NULL,
|
||||||
|
`message` text NOT NULL,
|
||||||
|
`link` text,
|
||||||
|
`target_type` text,
|
||||||
|
`target_id` integer,
|
||||||
|
`target_name` text,
|
||||||
|
`caused_by` text,
|
||||||
|
`opened_at` text NOT NULL,
|
||||||
|
`resolved_at` text,
|
||||||
|
`last_seen_at` text NOT NULL,
|
||||||
|
`missing_since` text
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX `alert_incidents_key_idx` ON `alert_incidents` (`alert_key`);--> statement-breakpoint
|
||||||
|
CREATE INDEX `alert_incidents_opened_idx` ON `alert_incidents` (`opened_at`);--> statement-breakpoint
|
||||||
|
CREATE INDEX `alert_incidents_target_idx` ON `alert_incidents` (`target_type`,`target_id`);--> statement-breakpoint
|
||||||
|
CREATE TABLE `incident_notes` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`incident_id` integer NOT NULL,
|
||||||
|
`author` text NOT NULL,
|
||||||
|
`body` text NOT NULL,
|
||||||
|
`created_at` text NOT NULL,
|
||||||
|
FOREIGN KEY (`incident_id`) REFERENCES `alert_incidents`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX `incident_notes_incident_idx` ON `incident_notes` (`incident_id`);
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
CREATE TABLE `status_items` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`link_id` integer NOT NULL,
|
||||||
|
`position` integer DEFAULT 0 NOT NULL,
|
||||||
|
`label` text NOT NULL,
|
||||||
|
`kind` text NOT NULL,
|
||||||
|
`target_id` integer NOT NULL,
|
||||||
|
`monitor_id` text,
|
||||||
|
FOREIGN KEY (`link_id`) REFERENCES `share_links`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX `status_items_link_idx` ON `status_items` (`link_id`,`position`);--> statement-breakpoint
|
||||||
|
ALTER TABLE `share_links` ADD `kind` text DEFAULT 'wall' NOT NULL;--> statement-breakpoint
|
||||||
|
ALTER TABLE `share_links` ADD `title` text;--> statement-breakpoint
|
||||||
|
ALTER TABLE `share_links` ADD `message` text;
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
ALTER TABLE `servers` ADD `hw_manufacturer` text;--> statement-breakpoint
|
||||||
|
ALTER TABLE `servers` ADD `hw_model` text;--> statement-breakpoint
|
||||||
|
ALTER TABLE `servers` ADD `hw_serial` text;
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
CREATE TABLE `dependencies` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`from_type` text NOT NULL,
|
||||||
|
`from_id` integer NOT NULL,
|
||||||
|
`to_type` text NOT NULL,
|
||||||
|
`to_id` integer NOT NULL,
|
||||||
|
`created_at` text DEFAULT (current_timestamp) NOT NULL
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX `dependencies_edge_idx` ON `dependencies` (`from_type`,`from_id`,`to_type`,`to_id`);
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
CREATE TABLE `push_subscriptions` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`user_id` integer NOT NULL,
|
||||||
|
`endpoint` text NOT NULL,
|
||||||
|
`p256dh` text NOT NULL,
|
||||||
|
`auth` text NOT NULL,
|
||||||
|
`label` text,
|
||||||
|
`created_at` text DEFAULT (current_timestamp) NOT NULL,
|
||||||
|
`last_success_at` text,
|
||||||
|
FOREIGN KEY (`user_id`) REFERENCES `users`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX `push_subscriptions_endpoint_unique` ON `push_subscriptions` (`endpoint`);
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
ALTER TABLE `alert_incidents` ADD `escalated_step` integer DEFAULT 0 NOT NULL;--> statement-breakpoint
|
||||||
|
ALTER TABLE `alert_incidents` ADD `escalated_at` text;--> statement-breakpoint
|
||||||
|
ALTER TABLE `notification_queue` ADD `audience` text;--> statement-breakpoint
|
||||||
|
ALTER TABLE `push_subscriptions` ADD `min_severity` text;--> statement-breakpoint
|
||||||
|
ALTER TABLE `push_subscriptions` ADD `categories` text;
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
ALTER TABLE `servers` ADD `agent_diagnostics` text;
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
CREATE TABLE `endpoint_checks` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`name` text NOT NULL,
|
||||||
|
`kind` text NOT NULL,
|
||||||
|
`target` text NOT NULL,
|
||||||
|
`port` integer,
|
||||||
|
`expect_status` text,
|
||||||
|
`keyword` text,
|
||||||
|
`insecure` integer DEFAULT false NOT NULL,
|
||||||
|
`interval_minutes` integer DEFAULT 5 NOT NULL,
|
||||||
|
`timeout_seconds` integer DEFAULT 10 NOT NULL,
|
||||||
|
`failures_before_alert` integer DEFAULT 2 NOT NULL,
|
||||||
|
`warn_days` integer DEFAULT 14 NOT NULL,
|
||||||
|
`server_id` integer,
|
||||||
|
`enabled` integer DEFAULT true NOT NULL,
|
||||||
|
`last_checked_at` text,
|
||||||
|
`last_ok` integer,
|
||||||
|
`last_message` text,
|
||||||
|
`last_response_ms` integer,
|
||||||
|
`cert_expires_at` text,
|
||||||
|
`consecutive_failures` integer DEFAULT 0 NOT NULL,
|
||||||
|
`last_change_at` text,
|
||||||
|
`created_at` text DEFAULT (current_timestamp) NOT NULL
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE TABLE `endpoint_results` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`check_id` integer NOT NULL,
|
||||||
|
`at` text NOT NULL,
|
||||||
|
`ok` integer NOT NULL,
|
||||||
|
`ms` integer,
|
||||||
|
`message` text,
|
||||||
|
FOREIGN KEY (`check_id`) REFERENCES `endpoint_checks`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX `endpoint_results_check_idx` ON `endpoint_results` (`check_id`,`at`);--> statement-breakpoint
|
||||||
|
CREATE TABLE `restore_tests` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`subject_key` text NOT NULL,
|
||||||
|
`label` text NOT NULL,
|
||||||
|
`tested_at` text NOT NULL,
|
||||||
|
`result` text NOT NULL,
|
||||||
|
`note` text,
|
||||||
|
`tested_by` text,
|
||||||
|
`created_at` text DEFAULT (current_timestamp) NOT NULL
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX `restore_tests_subject_idx` ON `restore_tests` (`subject_key`,`tested_at`);
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
ALTER TABLE `assets` ADD `end_of_support` text;--> statement-breakpoint
|
||||||
|
ALTER TABLE `assets` ADD `replacement_cost` real;
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
ALTER TABLE `users` ADD `picture_url` text;
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
CREATE TABLE `heartbeat_pings` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`heartbeat_id` integer NOT NULL,
|
||||||
|
`at` text NOT NULL,
|
||||||
|
`kind` text NOT NULL,
|
||||||
|
`duration_ms` integer,
|
||||||
|
`message` text,
|
||||||
|
FOREIGN KEY (`heartbeat_id`) REFERENCES `heartbeats`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX `heartbeat_pings_idx` ON `heartbeat_pings` (`heartbeat_id`,`at`);--> statement-breakpoint
|
||||||
|
CREATE TABLE `heartbeats` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`name` text NOT NULL,
|
||||||
|
`token` text NOT NULL,
|
||||||
|
`period_minutes` integer NOT NULL,
|
||||||
|
`grace_minutes` integer DEFAULT 10 NOT NULL,
|
||||||
|
`server_id` integer,
|
||||||
|
`enabled` integer DEFAULT true NOT NULL,
|
||||||
|
`notes` text,
|
||||||
|
`last_ping_at` text,
|
||||||
|
`last_status` text,
|
||||||
|
`last_message` text,
|
||||||
|
`last_duration_ms` integer,
|
||||||
|
`last_start_at` text,
|
||||||
|
`ping_count` integer DEFAULT 0 NOT NULL,
|
||||||
|
`created_at` text DEFAULT (current_timestamp) NOT NULL
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX `heartbeats_token_unique` ON `heartbeats` (`token`);
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
CREATE TABLE `endpoint_daily` (
|
||||||
|
`check_id` integer NOT NULL,
|
||||||
|
`day` text NOT NULL,
|
||||||
|
`total` integer DEFAULT 0 NOT NULL,
|
||||||
|
`ok` integer DEFAULT 0 NOT NULL,
|
||||||
|
`ms_sum` integer DEFAULT 0 NOT NULL,
|
||||||
|
`ms_count` integer DEFAULT 0 NOT NULL,
|
||||||
|
PRIMARY KEY(`check_id`, `day`),
|
||||||
|
FOREIGN KEY (`check_id`) REFERENCES `endpoint_checks`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE TABLE `patch_plan_runs` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`plan_id` integer NOT NULL,
|
||||||
|
`plan_name` text NOT NULL,
|
||||||
|
`trigger` text NOT NULL,
|
||||||
|
`started_by` text,
|
||||||
|
`status` text NOT NULL,
|
||||||
|
`message` text,
|
||||||
|
`items` text DEFAULT '[]' NOT NULL,
|
||||||
|
`started_at` text NOT NULL,
|
||||||
|
`finished_at` text,
|
||||||
|
FOREIGN KEY (`plan_id`) REFERENCES `patch_plans`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX `patch_plan_runs_plan_idx` ON `patch_plan_runs` (`plan_id`,`started_at`);--> statement-breakpoint
|
||||||
|
CREATE INDEX `patch_plan_runs_status_idx` ON `patch_plan_runs` (`status`);--> statement-breakpoint
|
||||||
|
CREATE TABLE `patch_plans` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`name` text NOT NULL,
|
||||||
|
`enabled` integer DEFAULT true NOT NULL,
|
||||||
|
`days` text NOT NULL,
|
||||||
|
`start_time` text NOT NULL,
|
||||||
|
`timezone` text NOT NULL,
|
||||||
|
`tags` text DEFAULT '[]' NOT NULL,
|
||||||
|
`server_ids` text DEFAULT '[]' NOT NULL,
|
||||||
|
`integration_id` integer NOT NULL,
|
||||||
|
`project_id` integer NOT NULL,
|
||||||
|
`template_id` integer NOT NULL,
|
||||||
|
`template_name` text NOT NULL,
|
||||||
|
`limit_from` text DEFAULT 'name' NOT NULL,
|
||||||
|
`stop_on_failure` integer DEFAULT true NOT NULL,
|
||||||
|
`only_with_updates` integer DEFAULT true NOT NULL,
|
||||||
|
`silence_dependents` integer DEFAULT true NOT NULL,
|
||||||
|
`window_minutes` integer DEFAULT 90 NOT NULL,
|
||||||
|
`last_run_date` text,
|
||||||
|
`created_by` text,
|
||||||
|
`created_at` text DEFAULT (current_timestamp) NOT NULL
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE TABLE `renewals` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`kind` text NOT NULL,
|
||||||
|
`subject` text NOT NULL,
|
||||||
|
`at` text NOT NULL,
|
||||||
|
`from_date` text NOT NULL,
|
||||||
|
`to_date` text NOT NULL
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX `renewals_at_idx` ON `renewals` (`at`);--> statement-breakpoint
|
||||||
|
ALTER TABLE `servers` ADD `port_baseline` text;
|
||||||
|
--> statement-breakpoint
|
||||||
|
-- The daily rollup starts with what the endpoint results already hold (they are kept two weeks), so the first report is not blank.
|
||||||
|
INSERT INTO `endpoint_daily` (`check_id`, `day`, `total`, `ok`, `ms_sum`, `ms_count`)
|
||||||
|
SELECT `check_id`, substr(`at`, 1, 10), count(*), sum(`ok`), sum(CASE WHEN `ok` = 1 AND `ms` IS NOT NULL THEN `ms` ELSE 0 END), sum(CASE WHEN `ok` = 1 AND `ms` IS NOT NULL THEN 1 ELSE 0 END)
|
||||||
|
FROM `endpoint_results` GROUP BY `check_id`, substr(`at`, 1, 10);
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
CREATE TABLE `network_devices` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`integration_id` integer NOT NULL,
|
||||||
|
`external_id` text NOT NULL,
|
||||||
|
`site_name` text,
|
||||||
|
`name` text NOT NULL,
|
||||||
|
`model` text,
|
||||||
|
`kind` text DEFAULT 'other' NOT NULL,
|
||||||
|
`ip` text,
|
||||||
|
`mac` text,
|
||||||
|
`state` text DEFAULT 'other' NOT NULL,
|
||||||
|
`uptime_seconds` integer,
|
||||||
|
`firmware` text,
|
||||||
|
`firmware_updatable` integer DEFAULT false NOT NULL,
|
||||||
|
`uplink_external_id` text,
|
||||||
|
`offline_since` text,
|
||||||
|
`last_online_at` text,
|
||||||
|
`first_seen_at` text NOT NULL,
|
||||||
|
`updated_at` text NOT NULL,
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX `network_devices_unique` ON `network_devices` (`integration_id`,`external_id`);--> statement-breakpoint
|
||||||
|
CREATE TABLE `network_segments` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`integration_id` integer NOT NULL,
|
||||||
|
`kind` text NOT NULL,
|
||||||
|
`external_id` text NOT NULL,
|
||||||
|
`site_name` text,
|
||||||
|
`name` text NOT NULL,
|
||||||
|
`enabled` integer DEFAULT true NOT NULL,
|
||||||
|
`vlan_id` integer,
|
||||||
|
`subnet` text,
|
||||||
|
`purpose` text,
|
||||||
|
`security` text,
|
||||||
|
`network_name` text,
|
||||||
|
`updated_at` text NOT NULL,
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX `network_segments_unique` ON `network_segments` (`integration_id`,`kind`,`external_id`);
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
CREATE TABLE `network_server_links` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`integration_id` integer NOT NULL,
|
||||||
|
`server_id` integer NOT NULL,
|
||||||
|
`device_external_id` text NOT NULL,
|
||||||
|
`client_mac` text NOT NULL,
|
||||||
|
`port` integer,
|
||||||
|
`wired` integer DEFAULT true NOT NULL,
|
||||||
|
`updated_at` text NOT NULL,
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade,
|
||||||
|
FOREIGN KEY (`server_id`) REFERENCES `servers`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX `network_server_links_unique` ON `network_server_links` (`integration_id`,`server_id`,`client_mac`);
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
CREATE TABLE `endpoint_hourly` (
|
||||||
|
`check_id` integer NOT NULL,
|
||||||
|
`hour` text NOT NULL,
|
||||||
|
`total` integer DEFAULT 0 NOT NULL,
|
||||||
|
`ok` integer DEFAULT 0 NOT NULL,
|
||||||
|
`ms_sum` integer DEFAULT 0 NOT NULL,
|
||||||
|
`ms_count` integer DEFAULT 0 NOT NULL,
|
||||||
|
`ms_max` integer DEFAULT 0 NOT NULL,
|
||||||
|
PRIMARY KEY(`check_id`, `hour`),
|
||||||
|
FOREIGN KEY (`check_id`) REFERENCES `endpoint_checks`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE TABLE `server_metrics_rollup` (
|
||||||
|
`server_id` integer NOT NULL,
|
||||||
|
`metric` text NOT NULL,
|
||||||
|
`grain` text NOT NULL,
|
||||||
|
`at` text NOT NULL,
|
||||||
|
`sum` real NOT NULL,
|
||||||
|
`n` integer NOT NULL,
|
||||||
|
`min` real NOT NULL,
|
||||||
|
`max` real NOT NULL,
|
||||||
|
PRIMARY KEY(`server_id`, `metric`, `grain`, `at`),
|
||||||
|
FOREIGN KEY (`server_id`) REFERENCES `servers`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
CREATE TABLE `api_tokens` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`user_id` integer NOT NULL,
|
||||||
|
`label` text NOT NULL,
|
||||||
|
`token_hash` text NOT NULL,
|
||||||
|
`token_prefix` text NOT NULL,
|
||||||
|
`created_at` text DEFAULT (current_timestamp) NOT NULL,
|
||||||
|
`last_used_at` text,
|
||||||
|
`expires_at` text,
|
||||||
|
FOREIGN KEY (`user_id`) REFERENCES `users`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX `api_tokens_token_hash_unique` ON `api_tokens` (`token_hash`);
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
ALTER TABLE `servers` ADD `os_id` text;--> statement-breakpoint
|
||||||
|
ALTER TABLE `servers` ADD `os_name` text;--> statement-breakpoint
|
||||||
|
ALTER TABLE `servers` ADD `os_version` text;
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
CREATE TABLE `notes` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`target_type` text NOT NULL,
|
||||||
|
`target_id` integer NOT NULL,
|
||||||
|
`body` text NOT NULL,
|
||||||
|
`updated_by` text,
|
||||||
|
`updated_at` text NOT NULL
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX `notes_target_idx` ON `notes` (`target_type`,`target_id`);--> statement-breakpoint
|
||||||
|
CREATE TABLE `todos` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`title` text NOT NULL,
|
||||||
|
`detail` text,
|
||||||
|
`due_date` text,
|
||||||
|
`assignee_user_id` integer,
|
||||||
|
`assignee_name` text,
|
||||||
|
`target_type` text,
|
||||||
|
`target_id` integer,
|
||||||
|
`created_by` text,
|
||||||
|
`created_at` text NOT NULL,
|
||||||
|
`done_at` text,
|
||||||
|
`done_by` text,
|
||||||
|
FOREIGN KEY (`assignee_user_id`) REFERENCES `users`(`id`) ON UPDATE no action ON DELETE set null
|
||||||
|
);
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
CREATE TABLE `ha_entities` (
|
||||||
|
`integration_id` integer NOT NULL,
|
||||||
|
`entity_id` text NOT NULL,
|
||||||
|
`friendly_name` text,
|
||||||
|
`state` text NOT NULL,
|
||||||
|
`value` real,
|
||||||
|
`unit` text,
|
||||||
|
`device_class` text,
|
||||||
|
`last_changed` text,
|
||||||
|
`unavailable_since` text,
|
||||||
|
`updated_at` text NOT NULL,
|
||||||
|
PRIMARY KEY(`integration_id`, `entity_id`),
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE TABLE `ha_readings` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`integration_id` integer NOT NULL,
|
||||||
|
`entity_id` text NOT NULL,
|
||||||
|
`at` text NOT NULL,
|
||||||
|
`value` real NOT NULL,
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX `ha_readings_lookup_idx` ON `ha_readings` (`integration_id`,`entity_id`,`at`);--> statement-breakpoint
|
||||||
|
CREATE TABLE `ha_watched` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`integration_id` integer NOT NULL,
|
||||||
|
`entity_id` text NOT NULL,
|
||||||
|
`name` text,
|
||||||
|
`above` real,
|
||||||
|
`below` real,
|
||||||
|
`created_at` text NOT NULL,
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX `ha_watched_unique` ON `ha_watched` (`integration_id`,`entity_id`);
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
CREATE TABLE `alert_rules` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`name` text NOT NULL,
|
||||||
|
`enabled` integer DEFAULT true NOT NULL,
|
||||||
|
`metric` text NOT NULL,
|
||||||
|
`mount` text,
|
||||||
|
`operator` text DEFAULT 'above' NOT NULL,
|
||||||
|
`threshold` real NOT NULL,
|
||||||
|
`for_minutes` integer DEFAULT 0 NOT NULL,
|
||||||
|
`severity` text DEFAULT 'warning' NOT NULL,
|
||||||
|
`server_id` integer,
|
||||||
|
`tag` text,
|
||||||
|
`created_by` text,
|
||||||
|
`created_at` text NOT NULL
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE TABLE `alert_snoozes` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`alert_id` text NOT NULL,
|
||||||
|
`condition_key` text NOT NULL,
|
||||||
|
`message` text NOT NULL,
|
||||||
|
`until` text NOT NULL,
|
||||||
|
`snoozed_by` text NOT NULL,
|
||||||
|
`snoozed_at` text NOT NULL,
|
||||||
|
`last_seen_at` text NOT NULL
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX `alert_snoozes_alert_idx` ON `alert_snoozes` (`alert_id`);--> statement-breakpoint
|
||||||
|
CREATE TABLE `storage_samples` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`series_id` integer NOT NULL,
|
||||||
|
`at` text NOT NULL,
|
||||||
|
`value` real NOT NULL,
|
||||||
|
FOREIGN KEY (`series_id`) REFERENCES `storage_series`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX `storage_samples_series_idx` ON `storage_samples` (`series_id`,`at`);--> statement-breakpoint
|
||||||
|
CREATE TABLE `storage_series` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`integration_id` integer NOT NULL,
|
||||||
|
`key` text NOT NULL,
|
||||||
|
`label` text NOT NULL,
|
||||||
|
`latest_percent` real NOT NULL,
|
||||||
|
`used_bytes` integer,
|
||||||
|
`total_bytes` integer,
|
||||||
|
`updated_at` text NOT NULL,
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX `storage_series_key_idx` ON `storage_series` (`integration_id`,`key`);
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
CREATE TABLE `webhook_deliveries` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`webhook_id` integer NOT NULL,
|
||||||
|
`event` text NOT NULL,
|
||||||
|
`delivery_id` text NOT NULL,
|
||||||
|
`attempts` integer DEFAULT 1 NOT NULL,
|
||||||
|
`ok` integer NOT NULL,
|
||||||
|
`result` text NOT NULL,
|
||||||
|
`body` text NOT NULL,
|
||||||
|
`retry_of` integer,
|
||||||
|
`created_at` text NOT NULL,
|
||||||
|
FOREIGN KEY (`webhook_id`) REFERENCES `event_webhooks`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX `webhook_deliveries_webhook_idx` ON `webhook_deliveries` (`webhook_id`,`created_at`);
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
ALTER TABLE `endpoint_checks` ADD `value_path` text;--> statement-breakpoint
|
||||||
|
ALTER TABLE `endpoint_checks` ADD `value_op` text;--> statement-breakpoint
|
||||||
|
ALTER TABLE `endpoint_checks` ADD `value_expected` text;--> statement-breakpoint
|
||||||
|
ALTER TABLE `endpoint_checks` ADD `value_unit` text;--> statement-breakpoint
|
||||||
|
ALTER TABLE `endpoint_checks` ADD `header_name` text;--> statement-breakpoint
|
||||||
|
ALTER TABLE `endpoint_checks` ADD `header_value` text;--> statement-breakpoint
|
||||||
|
ALTER TABLE `endpoint_checks` ADD `last_value` text;--> statement-breakpoint
|
||||||
|
ALTER TABLE `endpoint_results` ADD `value` real;
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
CREATE TABLE `ups_readings` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`integration_id` integer NOT NULL,
|
||||||
|
`at` text NOT NULL,
|
||||||
|
`charge` real,
|
||||||
|
`runtime_seconds` integer,
|
||||||
|
`load_percent` real,
|
||||||
|
`input_voltage` real,
|
||||||
|
`on_battery` integer DEFAULT false NOT NULL,
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX `ups_readings_lookup_idx` ON `ups_readings` (`integration_id`,`at`);--> statement-breakpoint
|
||||||
|
CREATE TABLE `ups_state` (
|
||||||
|
`integration_id` integer PRIMARY KEY NOT NULL,
|
||||||
|
`model` text,
|
||||||
|
`manufacturer` text,
|
||||||
|
`status_text` text NOT NULL,
|
||||||
|
`on_battery` integer DEFAULT false NOT NULL,
|
||||||
|
`low_battery` integer DEFAULT false NOT NULL,
|
||||||
|
`replace_battery` integer DEFAULT false NOT NULL,
|
||||||
|
`overload` integer DEFAULT false NOT NULL,
|
||||||
|
`charge` real,
|
||||||
|
`runtime_seconds` integer,
|
||||||
|
`load_percent` real,
|
||||||
|
`input_voltage` real,
|
||||||
|
`output_voltage` real,
|
||||||
|
`on_battery_since` text,
|
||||||
|
`updated_at` text NOT NULL,
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
CREATE TABLE `service_levels` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`name` text NOT NULL,
|
||||||
|
`subject_type` text NOT NULL,
|
||||||
|
`subject_id` integer NOT NULL,
|
||||||
|
`target` real NOT NULL,
|
||||||
|
`alert` integer DEFAULT true NOT NULL,
|
||||||
|
`created_by` text,
|
||||||
|
`created_at` text DEFAULT (current_timestamp) NOT NULL
|
||||||
|
);
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
CREATE TABLE `duplicati_backups` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`source_id` integer NOT NULL,
|
||||||
|
`name` text NOT NULL,
|
||||||
|
`last_report_at` text,
|
||||||
|
`last_result` text,
|
||||||
|
`last_begin` text,
|
||||||
|
`last_end` text,
|
||||||
|
`duration_seconds` integer,
|
||||||
|
`examined_files` integer,
|
||||||
|
`size_bytes` integer,
|
||||||
|
`added_files` integer,
|
||||||
|
`modified_files` integer,
|
||||||
|
`deleted_files` integer,
|
||||||
|
`warnings` integer DEFAULT 0 NOT NULL,
|
||||||
|
`errors` integer DEFAULT 0 NOT NULL,
|
||||||
|
`message` text,
|
||||||
|
`last_success_at` text,
|
||||||
|
`consecutive_failures` integer DEFAULT 0 NOT NULL,
|
||||||
|
`expected_hours` integer DEFAULT 48 NOT NULL,
|
||||||
|
`alert` integer DEFAULT true NOT NULL,
|
||||||
|
`created_at` text DEFAULT (current_timestamp) NOT NULL,
|
||||||
|
FOREIGN KEY (`source_id`) REFERENCES `duplicati_sources`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX `duplicati_backups_name_idx` ON `duplicati_backups` (`source_id`,`name`);--> statement-breakpoint
|
||||||
|
CREATE TABLE `duplicati_runs` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`backup_id` integer NOT NULL,
|
||||||
|
`at` text NOT NULL,
|
||||||
|
`result` text NOT NULL,
|
||||||
|
`duration_seconds` integer,
|
||||||
|
`size_bytes` integer,
|
||||||
|
`files_changed` integer,
|
||||||
|
`warnings` integer DEFAULT 0 NOT NULL,
|
||||||
|
`errors` integer DEFAULT 0 NOT NULL,
|
||||||
|
FOREIGN KEY (`backup_id`) REFERENCES `duplicati_backups`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX `duplicati_runs_idx` ON `duplicati_runs` (`backup_id`,`at`);--> statement-breakpoint
|
||||||
|
CREATE TABLE `duplicati_sources` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`name` text NOT NULL,
|
||||||
|
`token_hash` text NOT NULL,
|
||||||
|
`token_prefix` text NOT NULL,
|
||||||
|
`server_id` integer,
|
||||||
|
`created_by` text,
|
||||||
|
`created_at` text DEFAULT (current_timestamp) NOT NULL,
|
||||||
|
`last_report_at` text
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX `duplicati_sources_token_hash_unique` ON `duplicati_sources` (`token_hash`);
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
CREATE TABLE `jellyfin_readings` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`integration_id` integer NOT NULL,
|
||||||
|
`at` text NOT NULL,
|
||||||
|
`streams` integer NOT NULL,
|
||||||
|
`transcodes` integer NOT NULL,
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX `jellyfin_readings_idx` ON `jellyfin_readings` (`integration_id`,`at`);--> statement-breakpoint
|
||||||
|
CREATE TABLE `jellyfin_state` (
|
||||||
|
`integration_id` integer PRIMARY KEY NOT NULL,
|
||||||
|
`server_name` text,
|
||||||
|
`version` text,
|
||||||
|
`os` text,
|
||||||
|
`pending_restart` integer DEFAULT false NOT NULL,
|
||||||
|
`update_available` integer DEFAULT false NOT NULL,
|
||||||
|
`streams` text DEFAULT '[]' NOT NULL,
|
||||||
|
`tasks` text DEFAULT '[]' NOT NULL,
|
||||||
|
`movies` integer,
|
||||||
|
`series` integer,
|
||||||
|
`episodes` integer,
|
||||||
|
`songs` integer,
|
||||||
|
`failed_logins_24h` integer,
|
||||||
|
`updated_at` text NOT NULL,
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
CREATE TABLE `authentik_state` (
|
||||||
|
`integration_id` integer PRIMARY KEY NOT NULL,
|
||||||
|
`version` text,
|
||||||
|
`latest_version` text,
|
||||||
|
`outdated` integer DEFAULT false NOT NULL,
|
||||||
|
`workers` integer,
|
||||||
|
`outposts` text DEFAULT '[]' NOT NULL,
|
||||||
|
`certificates` text DEFAULT '[]' NOT NULL,
|
||||||
|
`tokens` text DEFAULT '[]' NOT NULL,
|
||||||
|
`failed_logins_24h` integer,
|
||||||
|
`config_errors_24h` integer,
|
||||||
|
`last_config_error` text,
|
||||||
|
`updated_at` text NOT NULL,
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
CREATE TABLE `prometheus_state` (
|
||||||
|
`integration_id` integer PRIMARY KEY NOT NULL,
|
||||||
|
`version` text,
|
||||||
|
`jobs` text,
|
||||||
|
`down` text,
|
||||||
|
`alerts` text,
|
||||||
|
`updated_at` text NOT NULL,
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
CREATE TABLE `immich_state` (
|
||||||
|
`integration_id` integer PRIMARY KEY NOT NULL,
|
||||||
|
`version` text,
|
||||||
|
`latest_version` text,
|
||||||
|
`update_available` integer DEFAULT false NOT NULL,
|
||||||
|
`disk_size` integer,
|
||||||
|
`disk_used` integer,
|
||||||
|
`disk_percent` real,
|
||||||
|
`photos` integer,
|
||||||
|
`videos` integer,
|
||||||
|
`usage_bytes` integer,
|
||||||
|
`queues` text,
|
||||||
|
`updated_at` text NOT NULL,
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
CREATE TABLE `paperless_state` (
|
||||||
|
`integration_id` integer PRIMARY KEY NOT NULL,
|
||||||
|
`version` text,
|
||||||
|
`latest_version` text,
|
||||||
|
`update_available` integer DEFAULT false NOT NULL,
|
||||||
|
`storage_total` integer,
|
||||||
|
`storage_available` integer,
|
||||||
|
`components` text,
|
||||||
|
`documents` integer,
|
||||||
|
`inbox` integer,
|
||||||
|
`failed` text,
|
||||||
|
`updated_at` text NOT NULL,
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
CREATE TABLE `note_photos` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`server_id` integer,
|
||||||
|
`device_id` integer,
|
||||||
|
`content_type` text NOT NULL,
|
||||||
|
`data` blob NOT NULL,
|
||||||
|
`size` integer NOT NULL,
|
||||||
|
`caption` text,
|
||||||
|
`created_by` text,
|
||||||
|
`created_at` text NOT NULL,
|
||||||
|
FOREIGN KEY (`server_id`) REFERENCES `servers`(`id`) ON UPDATE no action ON DELETE cascade,
|
||||||
|
FOREIGN KEY (`device_id`) REFERENCES `network_devices`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX `note_photos_server_idx` ON `note_photos` (`server_id`);--> statement-breakpoint
|
||||||
|
CREATE INDEX `note_photos_device_idx` ON `note_photos` (`device_id`);
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
CREATE TABLE `n8n_state` (
|
||||||
|
`integration_id` integer PRIMARY KEY NOT NULL,
|
||||||
|
`workflows` text DEFAULT '[]' NOT NULL,
|
||||||
|
`executions_known` integer DEFAULT false NOT NULL,
|
||||||
|
`updated_at` text NOT NULL,
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE TABLE `netbox_state` (
|
||||||
|
`integration_id` integer PRIMARY KEY NOT NULL,
|
||||||
|
`version` text,
|
||||||
|
`workers_running` integer,
|
||||||
|
`devices` integer,
|
||||||
|
`virtual_machines` integer,
|
||||||
|
`prefixes` integer,
|
||||||
|
`ip_addresses` integer,
|
||||||
|
`ips` text,
|
||||||
|
`updated_at` text NOT NULL,
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE TABLE `nextcloud_state` (
|
||||||
|
`integration_id` integer PRIMARY KEY NOT NULL,
|
||||||
|
`version` text,
|
||||||
|
`maintenance` integer DEFAULT false NOT NULL,
|
||||||
|
`needs_upgrade` integer DEFAULT false NOT NULL,
|
||||||
|
`detail` integer DEFAULT false NOT NULL,
|
||||||
|
`free_bytes` integer,
|
||||||
|
`users` integer,
|
||||||
|
`files` integer,
|
||||||
|
`active_24h` integer,
|
||||||
|
`app_updates` integer,
|
||||||
|
`updated_at` text NOT NULL,
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
ALTER TABLE `alert_snoozes` ADD `kind` text DEFAULT 'snooze' NOT NULL;--> statement-breakpoint
|
||||||
|
ALTER TABLE `alert_snoozes` ADD `severity` text;--> statement-breakpoint
|
||||||
|
ALTER TABLE `alert_snoozes` ADD `note` text;
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
ALTER TABLE `alert_tickets` ADD `ticket_closed_at` text;--> statement-breakpoint
|
||||||
|
ALTER TABLE `alert_tickets` ADD `ticket_closed_by` text;
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
ALTER TABLE `nextcloud_state` ADD `core_update` text;--> statement-breakpoint
|
||||||
|
ALTER TABLE `nextcloud_state` ADD `cron_state` text;--> statement-breakpoint
|
||||||
|
ALTER TABLE `nextcloud_state` ADD `cron_text` text;--> statement-breakpoint
|
||||||
|
ALTER TABLE `nextcloud_state` ADD `setup_issues` text;
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
CREATE TABLE `dockhand_vulnerabilities` (
|
||||||
|
`integration_id` integer NOT NULL,
|
||||||
|
`environment_id` integer NOT NULL,
|
||||||
|
`environment_name` text NOT NULL,
|
||||||
|
`critical` integer,
|
||||||
|
`high` integer,
|
||||||
|
`medium` integer,
|
||||||
|
`low` integer,
|
||||||
|
`images_scanned` integer,
|
||||||
|
`total_images` integer,
|
||||||
|
`findings` text DEFAULT '[]' NOT NULL,
|
||||||
|
`updated_at` text NOT NULL,
|
||||||
|
PRIMARY KEY(`integration_id`, `environment_id`),
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
CREATE TABLE `hetzner_state` (
|
||||||
|
`integration_id` integer PRIMARY KEY NOT NULL,
|
||||||
|
`servers` text DEFAULT '[]' NOT NULL,
|
||||||
|
`volumes` text,
|
||||||
|
`images` text,
|
||||||
|
`cost` text,
|
||||||
|
`updated_at` text NOT NULL,
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
CREATE TABLE `speedtest_results` (
|
||||||
|
`integration_id` integer NOT NULL,
|
||||||
|
`result_id` integer NOT NULL,
|
||||||
|
`at` text NOT NULL,
|
||||||
|
`status` text NOT NULL,
|
||||||
|
`download_bps` real,
|
||||||
|
`upload_bps` real,
|
||||||
|
`ping_ms` real,
|
||||||
|
`healthy` integer,
|
||||||
|
PRIMARY KEY(`integration_id`, `result_id`),
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX `speedtest_results_at_idx` ON `speedtest_results` (`integration_id`,`at`);
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
CREATE TABLE `cost_items` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`name` text NOT NULL,
|
||||||
|
`amount` real NOT NULL,
|
||||||
|
`currency` text NOT NULL,
|
||||||
|
`period` text NOT NULL,
|
||||||
|
`category` text DEFAULT 'Other' NOT NULL,
|
||||||
|
`renews_on` text,
|
||||||
|
`note` text,
|
||||||
|
`created_at` text NOT NULL,
|
||||||
|
`updated_at` text NOT NULL
|
||||||
|
);
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
CREATE TABLE `fragnet_state` (
|
||||||
|
`integration_id` integer PRIMARY KEY NOT NULL,
|
||||||
|
`servers` text DEFAULT '[]' NOT NULL,
|
||||||
|
`updated_at` text NOT NULL,
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
ALTER TABLE `hetzner_state` ADD `load_balancers` text;--> statement-breakpoint
|
||||||
|
ALTER TABLE `hetzner_state` ADD `floating_ips` text;--> statement-breakpoint
|
||||||
|
ALTER TABLE `hetzner_state` ADD `certificates` text;
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
CREATE TABLE `arr_state` (
|
||||||
|
`integration_id` integer PRIMARY KEY NOT NULL,
|
||||||
|
`version` text,
|
||||||
|
`health` text DEFAULT '[]' NOT NULL,
|
||||||
|
`queue` text,
|
||||||
|
`disks` text,
|
||||||
|
`missing` integer,
|
||||||
|
`indexers` text,
|
||||||
|
`updated_at` text NOT NULL,
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
CREATE TABLE `alert_kind_runbooks` (
|
||||||
|
`kind` text PRIMARY KEY NOT NULL,
|
||||||
|
`title` text NOT NULL,
|
||||||
|
`body` text DEFAULT '' NOT NULL,
|
||||||
|
`url` text,
|
||||||
|
`updated_by` text,
|
||||||
|
`updated_at` text NOT NULL
|
||||||
|
);
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
ALTER TABLE `users` ADD `server_page_layout` text;
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
ALTER TABLE `dockhand_vulnerabilities` ADD `images` text;--> statement-breakpoint
|
||||||
|
ALTER TABLE `dockhand_vulnerabilities` ADD `images_at` text;
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
CREATE TABLE `adguard_state` (
|
||||||
|
`integration_id` integer PRIMARY KEY NOT NULL,
|
||||||
|
`version` text,
|
||||||
|
`running` integer DEFAULT true NOT NULL,
|
||||||
|
`protection_enabled` integer DEFAULT true NOT NULL,
|
||||||
|
`protection_off_since` text,
|
||||||
|
`dns_port` integer DEFAULT 53 NOT NULL,
|
||||||
|
`stats` text,
|
||||||
|
`filters` text,
|
||||||
|
`probe` text,
|
||||||
|
`probe_fail_since` text,
|
||||||
|
`updated_at` text NOT NULL,
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE TABLE `grafana_state` (
|
||||||
|
`integration_id` integer PRIMARY KEY NOT NULL,
|
||||||
|
`version` text,
|
||||||
|
`rules` text,
|
||||||
|
`datasources` text,
|
||||||
|
`updated_at` text NOT NULL,
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
CREATE TABLE `release_cache` (
|
||||||
|
`repo` text PRIMARY KEY NOT NULL,
|
||||||
|
`tag` text NOT NULL,
|
||||||
|
`published_at` text,
|
||||||
|
`url` text,
|
||||||
|
`checked_at` text NOT NULL,
|
||||||
|
`error` text
|
||||||
|
);
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
CREATE TABLE `port_presets` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`name` text NOT NULL,
|
||||||
|
`ports` text DEFAULT '[]' NOT NULL,
|
||||||
|
`created_at` text NOT NULL,
|
||||||
|
`updated_at` text NOT NULL
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX `port_presets_name_unique` ON `port_presets` (`name`);
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
CREATE TABLE `server_container_ports` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`server_id` integer NOT NULL,
|
||||||
|
`port` integer NOT NULL,
|
||||||
|
`protocol` text DEFAULT 'tcp' NOT NULL,
|
||||||
|
`container` text NOT NULL,
|
||||||
|
`image` text DEFAULT '' NOT NULL,
|
||||||
|
`updated_at` text NOT NULL,
|
||||||
|
FOREIGN KEY (`server_id`) REFERENCES `servers`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX `server_container_ports_unique` ON `server_container_ports` (`server_id`,`port`,`protocol`);
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
CREATE TABLE `technitium_state` (
|
||||||
|
`integration_id` integer PRIMARY KEY NOT NULL,
|
||||||
|
`version` text,
|
||||||
|
`started_at` text,
|
||||||
|
`dns_port` integer DEFAULT 53 NOT NULL,
|
||||||
|
`blocking_enabled` integer DEFAULT true NOT NULL,
|
||||||
|
`blocking_off_since` text,
|
||||||
|
`settings` text NOT NULL,
|
||||||
|
`stats` text,
|
||||||
|
`zones` text,
|
||||||
|
`dhcp` text,
|
||||||
|
`probe` text,
|
||||||
|
`probe_fail_since` text,
|
||||||
|
`updated_at` text NOT NULL,
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
ALTER TABLE `servers` ADD `hostname_pinned` integer DEFAULT false NOT NULL;--> statement-breakpoint
|
||||||
|
ALTER TABLE `servers` ADD `reported_hostname` text;--> statement-breakpoint
|
||||||
|
UPDATE `servers` SET `reported_hostname` = `hostname` WHERE `last_seen_at` IS NOT NULL AND `hostname` IS NOT NULL;
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
CREATE TABLE `download_state` (
|
||||||
|
`integration_id` integer PRIMARY KEY NOT NULL,
|
||||||
|
`version` text,
|
||||||
|
`speed_down` integer,
|
||||||
|
`speed_up` integer,
|
||||||
|
`paused` integer DEFAULT false NOT NULL,
|
||||||
|
`connection` text,
|
||||||
|
`counts` text NOT NULL,
|
||||||
|
`total` integer DEFAULT 0 NOT NULL,
|
||||||
|
`items` text DEFAULT '[]' NOT NULL,
|
||||||
|
`failed` text,
|
||||||
|
`disks` text,
|
||||||
|
`warnings` text,
|
||||||
|
`since` text DEFAULT '{}' NOT NULL,
|
||||||
|
`updated_at` text NOT NULL,
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
CREATE TABLE `hetzner_robot_state` (
|
||||||
|
`integration_id` integer PRIMARY KEY NOT NULL,
|
||||||
|
`servers` text DEFAULT '[]' NOT NULL,
|
||||||
|
`storage_boxes` text,
|
||||||
|
`skipped_detail` integer DEFAULT 0 NOT NULL,
|
||||||
|
`updated_at` text NOT NULL,
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
CREATE TABLE `cost_item_versions` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`item_id` integer NOT NULL,
|
||||||
|
`name` text NOT NULL,
|
||||||
|
`amount` real NOT NULL,
|
||||||
|
`currency` text NOT NULL,
|
||||||
|
`period` text NOT NULL,
|
||||||
|
`category` text NOT NULL,
|
||||||
|
`valid_from` text NOT NULL,
|
||||||
|
`valid_to` text
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX `cost_item_versions_item_idx` ON `cost_item_versions` (`item_id`);--> statement-breakpoint
|
||||||
|
CREATE TABLE `cost_snapshots` (
|
||||||
|
`day` text NOT NULL,
|
||||||
|
`key` text NOT NULL,
|
||||||
|
`name` text NOT NULL,
|
||||||
|
`currency` text NOT NULL,
|
||||||
|
`monthly` real NOT NULL,
|
||||||
|
PRIMARY KEY(`day`, `key`)
|
||||||
|
);
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
ALTER TABLE `servers` ADD `uptime_seconds` integer;
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
ALTER TABLE `technitium_state` ADD `cluster` text;
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
ALTER TABLE `duplicati_backups` ADD `warning_messages` text;--> statement-breakpoint
|
||||||
|
ALTER TABLE `duplicati_backups` ADD `alert_warnings` integer DEFAULT false NOT NULL;--> statement-breakpoint
|
||||||
|
ALTER TABLE `duplicati_runs` ADD `warning_messages` text;
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
ALTER TABLE `duplicati_sources` ADD `last_report` text;
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
CREATE TABLE `notification_log` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`at` text NOT NULL,
|
||||||
|
`title` text NOT NULL,
|
||||||
|
`message` text NOT NULL,
|
||||||
|
`status` text NOT NULL,
|
||||||
|
`results` text DEFAULT '[]' NOT NULL,
|
||||||
|
`note` text,
|
||||||
|
`resent_of` integer
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX `notification_log_at_idx` ON `notification_log` (`at`);
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
CREATE TABLE `fragnet_power_log` (
|
||||||
|
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
`integration_id` integer NOT NULL,
|
||||||
|
`server_id` text NOT NULL,
|
||||||
|
`server_name` text NOT NULL,
|
||||||
|
`at` text NOT NULL,
|
||||||
|
`kind` text NOT NULL,
|
||||||
|
`detail` text,
|
||||||
|
`by` text,
|
||||||
|
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX `fragnet_power_log_idx` ON `fragnet_power_log` (`integration_id`,`server_id`,`at`);
|
||||||
Loaded 100 of 902 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user