Host health from the Linux agent, agent self-update, and disk forecast in /metrics

- Linux agent v4: failed systemd services, SMART disk health, ZFS pool state, pacman updates;
  SMART/ZFS and self-update run in the ExecStartPre=-+ step outside the sandbox
- Server: hosts' health stored (servers.host_health, migration 0029), alerts under a new
  "Failing services" kind plus disk/ZFS alerts, Host health card, Prometheus series
- Agent self-update (Linux v4, Windows v3): admin switch on the Servers page, checksum-verified
  download, atomic replace; GET /api/agent/report/update
- homelab_server_disk_days_to_full in /metrics
- Tests (server, real PowerShell against a fake server) and docs

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 00:54:24 +02:00
1 parent 540ab704b7
commit 5dc6cea69c
38 files changed
+3884 -73

No files matched your search

+219 -42
View File
@@ -13,8 +13,9 @@ set -euo pipefail
# Which version of this script this is, sent with every report so Homelab Manager can tell which agents are behind. Raise it whenever a
# change here is worth installing the agent again for (and add it to the list at the top of server/src/services/agentVersions.ts).
# 1 = before the number was reported 2 = reports pending updates, and its own version
# 3 = the package manager is asked outside the sandbox (see --prepare-updates), so dnf and zypper can answer there too
AGENT_VERSION=3
# 3 = the package manager is asked outside the sandbox (see --prepare), so dnf and zypper can answer there too
# 4 = also reports failed services, disk health (SMART) and ZFS pools, reads pacman, and can update itself when told to
AGENT_VERSION=4
ENV_FILE="${ENV_FILE:-/etc/homelab-manager-agent.env}"
if [[ -f "$ENV_FILE" ]]; then
@@ -29,16 +30,21 @@ DRY_RUN=0
[[ "${1:-}" == "--dry-run" ]] && DRY_RUN=1
# Two extra ways to run it, neither of which reports anything:
# --prepare-updates asks the package manager what is waiting and leaves the answer in $UPDATES_FILE for the report to pick up. The
# systemd unit runs this step outside its sandbox (ExecStartPre=+), because dnf and zypper write to /var and /run
# even when only asked a question, and the sandbox makes those read-only. A failure here only means no figure.
# --check-updates for a person at a terminal: runs the same check, shows what the package manager said and the result.
# --prepare the things that can't be done inside the systemd sandbox: asks the package manager what is waiting (dnf and zypper
# write to /var and /run even when only asked), reads disk health with smartctl and pool state with zpool (both need the
# devices, which the sandbox closes), and — if an administrator has switched it on — replaces this script with a newer
# one. The answers are left in the unit's private directory for the report to pick up. The unit runs this step outside
# its sandbox (ExecStartPre=-+); a failure here only means those figures are missing. --prepare-updates is the same.
# --check-updates for a person at a terminal: runs the same checks and shows what each said, and whether it would update itself.
MODE=report
case "${1:-}" in
--prepare-updates) MODE=prepare ;;
--check-updates) MODE=check ;;
--prepare|--prepare-updates) MODE=prepare ;;
--check-updates|--check) MODE=check ;;
esac
UPDATES_FILE="${UPDATES_FILE:-/run/homelab-manager-agent/updates.json}"
HEALTH_FILE="${HEALTH_FILE:-$(dirname "$UPDATES_FILE")/health.json}"
# Kept between runs (the unit's private directory is not): a SMART reading is only taken about once an hour, so sleeping disks are left alone.
STATE_DIR="${STATE_DIR:-/var/lib/homelab-manager-agent}"
# Where the package managers' own error output goes: nowhere, unless a person asked to see it.
PM_ERR=/dev/null
[[ "$MODE" == "check" ]] && PM_ERR=/dev/stderr
@@ -228,6 +234,16 @@ collect_updates() {
if command -v needs-restarting >/dev/null 2>&1; then
needs-restarting -r >/dev/null 2>&1 || reboot="true"
fi
elif command -v pacman >/dev/null 2>&1; then
manager="pacman"
# -Qu is what the local copy of the sync databases says could be upgraded (what `checkupdates` lists, without changing anything). It
# exits 1 when there is nothing to upgrade, so an empty answer with status 1 is "none", not an error. Packages held back are left out.
out=$(LC_ALL=C timeout 60 pacman -Qu 2>"$PM_ERR") || status=$?
if [[ "$status" -eq 0 || ( "$status" -eq 1 && -z "$out" ) ]]; then
count=$(grep -v '\[ignored\]' <<<"$out" | grep -c . || true)
fi
# No security figure (pacman doesn't say). A restart is due when the kernel running is no longer the one installed.
[[ -d "/usr/lib/modules/$(uname -r)" ]] || reboot="true"
elif command -v zypper >/dev/null 2>&1; then
manager="zypper"
if out=$(LC_ALL=C timeout 120 zypper -q --no-refresh list-updates 2>"$PM_ERR"); then
@@ -248,7 +264,164 @@ collect_updates() {
fi
}
# The answer --prepare-updates left for this report, if it is there, recent and looks like what it should. Read back through jq rather than
# >>> post_report
# Sends $PAYLOAD to the server and prints the HTTP status code.
#
# Nothing sensitive goes on curl's command line: any user on this machine can read other processes' command lines, and the
# token would be there for the length of every report. So the token is handed to curl on its standard input, as a line of
# curl's own config format, and the payload and the server's reply go through files in a private directory (mktemp -d is
# readable by root only). A fixed name in /tmp is no good either — another user could plant a symlink there and have this
# script, running as root, overwrite whatever it points at.
curl_quote() {
local s=$1
s=${s//\\/\\\\}
s=${s//\"/\\\"}
printf '%s' "$s"
}
post_report() {
case "$API_TOKEN" in
*$'\n'*|*$'\r'*) echo "API_TOKEN must not contain a line break." >&2; return 1 ;;
esac
printf '%s' "$PAYLOAD" > "$HLM_TMPDIR/payload.json"
printf 'header = "Authorization: Bearer %s"\n' "$(curl_quote "$API_TOKEN")" | curl -sS "${CURL_INSECURE_FLAG[@]}" -K - \
-o "$HLM_TMPDIR/response.json" -w "%{http_code}" \
-X POST "$API_URL/api/agent/report" \
-H "Content-Type: application/json" \
--data-binary @"$HLM_TMPDIR/payload.json"
}
# <<< post_report
# A GET with the token handed to curl on its standard input (as the upload does), the reply into the file named; prints the HTTP status.
api_get() {
printf 'header = "Authorization: Bearer %s"\n' "$(curl_quote "$API_TOKEN")" | curl -sS "${CURL_INSECURE_FLAG[@]}" -K - -m 30 -o "$2" -w "%{http_code}" "$API_URL$1"
}
# Services (systemd units) that have failed. systemctl is asked the way the timers are, so this works inside the sandbox.
collect_failed_units() {
FAILED_UNITS_JSON="null"
command -v systemctl >/dev/null 2>&1 || return 0
local out
out=$(timeout 20 systemctl list-units --state=failed --plain --no-legend --no-pager --full 2>"$PM_ERR") || return 0
FAILED_UNITS_JSON=$(awk 'NF > 0 { unit = $1; $1 = $2 = $3 = $4 = ""; sub(/^ +/, ""); print unit "\t" $0 }' <<<"$out" \
| jq -R -s -c 'split("\n") | map(select(length > 0) | split("\t")) | map({unit: .[0], description: (.[1] // "")}) | .[0:100]') || FAILED_UNITS_JSON="null"
}
# Each disk's own SMART verdict, temperature and (where it says) how much of its rated life is used. Needs the devices, so it runs from
# --prepare. Asked at most about once an hour, and with -n standby so a sleeping disk is not woken: what was last known of it is kept.
collect_smart() {
SMART_JSON="null"
command -v smartctl >/dev/null 2>&1 || return 0
local cache="$STATE_DIR/smart.json" now_s previous="[]" items="[]" dev out item scan checked
now_s=$(date +%s)
if [[ -r "$cache" ]]; then
checked=$(jq -r '.checked_at // 0' "$cache" 2>/dev/null || echo 0)
previous=$(jq -c '.devices // []' "$cache" 2>/dev/null || echo "[]")
if [[ "$checked" =~ ^[0-9]+$ ]] && (( now_s - checked < 3300 )) && [[ "$MODE" != "check" ]]; then
SMART_JSON=$previous
return 0
fi
fi
scan=$(timeout 30 smartctl --scan -j 2>"$PM_ERR") || true
while IFS= read -r dev; do
[[ -n "$dev" ]] || continue
# smartctl's exit status is a bit mask that says "something was noted" more often than "it failed", so it is read from the JSON instead.
out=$(timeout 30 smartctl -j -H -A -n standby "$dev" 2>"$PM_ERR") || true
item=$(jq -c --arg dev "$dev" 'select(.smart_status != null) | {
device: $dev,
model: (.model_name // .scsi_model_name // ""),
healthy: .smart_status.passed,
temperature_c: (.temperature.current // null),
wear_percent: (.nvme_smart_health_information_log.percentage_used // ([.ata_smart_attributes.table[]? | select(.id == 177 or .id == 231 or .id == 233) | (100 - .value)] | first) // null)
}' <<<"$out" 2>/dev/null) || item=""
if [[ -z "$item" ]]; then
# Asleep, or no SMART at all (a virtual disk): keep what was known of it, if anything.
item=$(jq -c --arg dev "$dev" '.[] | select(.device == $dev)' <<<"$previous" 2>/dev/null | head -n 1) || item=""
fi
if [[ -n "$item" ]]; then items=$(jq -c --argjson i "$item" '. + [$i]' <<<"$items"); fi
done < <(jq -r '.devices[]?.name' <<<"$scan" 2>/dev/null)
SMART_JSON=$items
if mkdir -p "$STATE_DIR" 2>/dev/null; then
chmod 700 "$STATE_DIR" 2>/dev/null || true
jq -c -n --argjson at "$now_s" --argjson devices "$items" '{checked_at: $at, devices: $devices}' > "$cache.tmp" 2>/dev/null && mv -f "$cache.tmp" "$cache" 2>/dev/null || true
fi
}
# Each ZFS pool and whether it is ONLINE. Needs /dev/zfs, so it runs from --prepare too.
collect_zfs() {
ZFS_JSON="null"
command -v zpool >/dev/null 2>&1 || return 0
local out
out=$(timeout 20 zpool list -H -o name,health,capacity 2>"$PM_ERR") || return 0
ZFS_JSON=$(jq -R -s -c 'split("\n") | map(select(length > 0) | split("\t")) | map({name: .[0], health: (.[1] // "UNKNOWN"), capacity_percent: ((.[2] // "" | rtrimstr("%") | tonumber?) // null)})' <<<"$out") || ZFS_JSON="null"
}
# The disk and pool readings --prepare left for this report, if they are there and recent. Re-shaped by jq, never trusted as they stand.
use_prepared_health() {
[[ -r "$HEALTH_FILE" ]] || return 1
[[ -n "$(find "$HEALTH_FILE" -mmin -10 -print 2>/dev/null)" ]] || return 1
SMART_JSON=$(jq -c 'if (.smart | type) == "array" then [.smart[] | {device: (.device | tostring), model: ((.model // "") | tostring),
healthy: (if (.healthy | type) == "boolean" then .healthy else null end),
temperature_c: (if (.temperature_c | type) == "number" then .temperature_c else null end),
wear_percent: (if (.wear_percent | type) == "number" then .wear_percent else null end)}] else null end' "$HEALTH_FILE" 2>/dev/null) || SMART_JSON="null"
ZFS_JSON=$(jq -c 'if (.zfs | type) == "array" then [.zfs[] | {name: (.name | tostring), health: ((.health // "UNKNOWN") | tostring),
capacity_percent: (if (.capacity_percent | type) == "number" then .capacity_percent else null end)}] else null end' "$HEALTH_FILE" 2>/dev/null) || ZFS_JSON="null"
return 0
}
# Replaces this script with the one this Homelab Manager serves, when an administrator has switched automatic updates on and this agent is
# behind. Nothing is trusted but the checksum: the (authenticated) answer gives the SHA-256 of the script, and a download that doesn't
# match it, doesn't look like this script, doesn't pass bash's syntax check or isn't newer is thrown away. The new file is put beside the
# old and moved over it in one step, and the new version then carries on from here. Runs only from --prepare (outside the sandbox, which
# keeps /usr read-only for the report itself); with --check-updates it says what it would do.
self_update() {
[[ -n "$API_URL" && -n "$API_TOKEN" ]] || return 0
[[ -z "${HLM_JUST_UPDATED:-}" ]] || return 0
local self work code reply_update version sha got
self=$(readlink -f "${BASH_SOURCE[0]}" 2>/dev/null) || return 0
[[ -f "$self" ]] || return 0
work=$(mktemp -d) || return 0
code=$(api_get "/api/agent/report/update?version=$AGENT_VERSION" "$work/reply.json" 2>/dev/null) || code=""
reply_update=$(jq -r '.update // false' "$work/reply.json" 2>/dev/null || echo false)
if [[ "$code" != "200" || "$reply_update" != "true" ]]; then
[[ "$MODE" == "check" ]] && echo "Automatic update: nothing to do (it is off, or this agent is current)."
rm -rf "$work"
return 0
fi
version=$(jq -r '.version // empty' "$work/reply.json" 2>/dev/null || true)
sha=$(jq -r '.sha256 // empty' "$work/reply.json" 2>/dev/null || true)
if [[ ! "$version" =~ ^[0-9]+$ || ! "$sha" =~ ^[0-9a-f]{64}$ ]] || (( version <= AGENT_VERSION )); then
rm -rf "$work"
return 0
fi
if ! curl -fsS "${CURL_INSECURE_FLAG[@]}" -m 60 -o "$work/new.sh" "$API_URL/agent/linux/report-tasks.sh" 2>"$PM_ERR"; then
echo "Note: couldn't download the newer agent (version $version), so it was not installed." >&2
rm -rf "$work"
return 0
fi
got=$(sha256sum "$work/new.sh" 2>/dev/null | awk '{print $1}')
if [[ "$got" != "$sha" ]] || [[ "$(head -n 1 "$work/new.sh")" != "#!/usr/bin/env bash" ]] || ! bash -n "$work/new.sh" 2>/dev/null || ! grep -q "^AGENT_VERSION=$version\$" "$work/new.sh"; then
echo "Note: the newer agent (version $version) didn't check out against what the server said, so it was not installed." >&2
rm -rf "$work"
return 0
fi
if [[ "$MODE" == "check" ]]; then
echo "Automatic update: version $version is available and would be installed on the next run."
rm -rf "$work"
return 0
fi
if install -m 755 "$work/new.sh" "$self.new" && mv -f "$self.new" "$self"; then
echo "Updated the agent from version $AGENT_VERSION to version $version." >&2
rm -rf "$work"
HLM_JUST_UPDATED=1 exec "$self" --prepare
fi
rm -f "$self.new"
echo "Note: couldn't replace the agent with version $version." >&2
rm -rf "$work"
return 0
}
# The answer --prepare left for this report, if it is there, recent and looks like what it should. Read back through jq rather than
# trusted: the report only ever carries these four fields, whatever the file holds.
use_prepared_updates() {
[[ -r "$UPDATES_FILE" ]] || return 1
@@ -325,8 +498,23 @@ collect_system_info() {
UPDATES_JSON="null"
use_prepared_updates || collect_updates || UPDATES_JSON="null"
# Failed services are asked here (systemctl works inside the sandbox); disk and pool health come from --prepare. A person running --dry-run
# isn't in the sandbox, so there they are read directly.
FAILED_UNITS_JSON="null"
collect_failed_units || FAILED_UNITS_JSON="null"
SMART_JSON="null"
ZFS_JSON="null"
if ! use_prepared_health && [[ "$DRY_RUN" == "1" ]]; then
collect_smart || SMART_JSON="null"
collect_zfs || ZFS_JSON="null"
fi
HEALTH_JSON=$(jq -n -c --argjson failed "$FAILED_UNITS_JSON" --argjson smart "$SMART_JSON" --argjson zfs "$ZFS_JSON" \
'{} | (if $failed != null then . + {failed_units: $failed} else . end) | (if $smart != null then . + {smart: $smart} else . end) | (if $zfs != null then . + {zfs: $zfs} else . end)') || HEALTH_JSON="{}"
[[ "$HEALTH_JSON" == "{}" ]] && HEALTH_JSON="null"
SYSTEM_JSON=$(jq -n \
--argjson updates "$UPDATES_JSON" \
--argjson health "$HEALTH_JSON" \
--argjson ip_addresses "$ip_json" \
--argjson listening_ports "$ports_json" \
--arg cpu_model "$cpu_model" \
@@ -341,23 +529,40 @@ collect_system_info() {
memory: { total_bytes: $mem_total_bytes, used_bytes: $mem_used_bytes },
disks: $disks,
listening_ports: $listening_ports,
updates: $updates
updates: $updates,
health: $health
}')
}
if [[ "$MODE" != "report" ]]; then
collect_updates
# prepare: first see whether this script is to be replaced (if so it carries on as the new one, from the top).
[[ "$MODE" == "prepare" ]] && self_update
collect_updates || UPDATES_JSON="null"
collect_smart || SMART_JSON="null"
collect_zfs || ZFS_JSON="null"
if [[ "$MODE" == "check" ]]; then
collect_failed_units || FAILED_UNITS_JSON="null"
echo "Pending updates as they would be reported:"
jq . <<<"$UPDATES_JSON"
echo "Failed services:"
jq . <<<"$FAILED_UNITS_JSON"
echo "Disk health (SMART; null means smartctl isn't installed or this isn't root):"
jq . <<<"$SMART_JSON"
echo "ZFS pools (null means there is no zpool here):"
jq . <<<"$ZFS_JSON"
self_update
exit 0
fi
# prepare: leave the answer (or no file at all) for the report that follows. Written whole and moved into place, so a report never reads half of it.
rm -f "$UPDATES_FILE"
# Leave the answers (or no file at all) for the report that follows. Each is written whole and moved into place, so a report never reads half of one.
rm -f "$UPDATES_FILE" "$HEALTH_FILE"
mkdir -p "$(dirname "$UPDATES_FILE")" 2>/dev/null || exit 0
if [[ "$UPDATES_JSON" != "null" ]]; then
mkdir -p "$(dirname "$UPDATES_FILE")" 2>/dev/null || exit 0
printf '%s\n' "$UPDATES_JSON" > "$UPDATES_FILE.tmp" 2>/dev/null && mv -f "$UPDATES_FILE.tmp" "$UPDATES_FILE" 2>/dev/null || true
fi
HEALTH_FILE_JSON=$(jq -n -c --argjson smart "${SMART_JSON:-null}" --argjson zfs "${ZFS_JSON:-null}" '{} | (if $smart != null then . + {smart: $smart} else . end) | (if $zfs != null then . + {zfs: $zfs} else . end)' 2>/dev/null || echo "{}")
if [[ "$HEALTH_FILE_JSON" != "{}" ]]; then
printf '%s\n' "$HEALTH_FILE_JSON" > "$HEALTH_FILE.tmp" 2>/dev/null && mv -f "$HEALTH_FILE.tmp" "$HEALTH_FILE" 2>/dev/null || true
fi
exit 0
fi
@@ -392,34 +597,6 @@ if [[ "$DRY_RUN" == "1" ]]; then
exit 0
fi
# >>> post_report
# Sends $PAYLOAD to the server and prints the HTTP status code.
#
# Nothing sensitive goes on curl's command line: any user on this machine can read other processes' command lines, and the
# token would be there for the length of every report. So the token is handed to curl on its standard input, as a line of
# curl's own config format, and the payload and the server's reply go through files in a private directory (mktemp -d is
# readable by root only). A fixed name in /tmp is no good either — another user could plant a symlink there and have this
# script, running as root, overwrite whatever it points at.
curl_quote() {
local s=$1
s=${s//\\/\\\\}
s=${s//\"/\\\"}
printf '%s' "$s"
}
post_report() {
case "$API_TOKEN" in
*$'\n'*|*$'\r'*) echo "API_TOKEN must not contain a line break." >&2; return 1 ;;
esac
printf '%s' "$PAYLOAD" > "$HLM_TMPDIR/payload.json"
printf 'header = "Authorization: Bearer %s"\n' "$(curl_quote "$API_TOKEN")" | curl -sS "${CURL_INSECURE_FLAG[@]}" -K - \
-o "$HLM_TMPDIR/response.json" -w "%{http_code}" \
-X POST "$API_URL/api/agent/report" \
-H "Content-Type: application/json" \
--data-binary @"$HLM_TMPDIR/payload.json"
}
# <<< post_report
HLM_TMPDIR=$(mktemp -d)
trap 'rm -rf "$HLM_TMPDIR"' EXIT