Host health from the Linux agent, agent self-update, and disk forecast in /metrics
- Linux agent v4: failed systemd services, SMART disk health, ZFS pool state, pacman updates; SMART/ZFS and self-update run in the ExecStartPre=-+ step outside the sandbox - Server: hosts' health stored (servers.host_health, migration 0029), alerts under a new "Failing services" kind plus disk/ZFS alerts, Host health card, Prometheus series - Agent self-update (Linux v4, Windows v3): admin switch on the Servers page, checksum-verified download, atomic replace; GET /api/agent/report/update - homelab_server_disk_days_to_full in /metrics - Tests (server, real PowerShell against a fake server) and docs Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
540ab704b7
commit
5dc6cea69c
38 files changed
+3884
-73
No files matched your search
+219
-42
@@ -13,8 +13,9 @@ set -euo pipefail
|
||||
# Which version of this script this is, sent with every report so Homelab Manager can tell which agents are behind. Raise it whenever a
|
||||
# change here is worth installing the agent again for (and add it to the list at the top of server/src/services/agentVersions.ts).
|
||||
# 1 = before the number was reported 2 = reports pending updates, and its own version
|
||||
# 3 = the package manager is asked outside the sandbox (see --prepare-updates), so dnf and zypper can answer there too
|
||||
AGENT_VERSION=3
|
||||
# 3 = the package manager is asked outside the sandbox (see --prepare), so dnf and zypper can answer there too
|
||||
# 4 = also reports failed services, disk health (SMART) and ZFS pools, reads pacman, and can update itself when told to
|
||||
AGENT_VERSION=4
|
||||
|
||||
ENV_FILE="${ENV_FILE:-/etc/homelab-manager-agent.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
@@ -29,16 +30,21 @@ DRY_RUN=0
|
||||
[[ "${1:-}" == "--dry-run" ]] && DRY_RUN=1
|
||||
|
||||
# Two extra ways to run it, neither of which reports anything:
|
||||
# --prepare-updates asks the package manager what is waiting and leaves the answer in $UPDATES_FILE for the report to pick up. The
|
||||
# systemd unit runs this step outside its sandbox (ExecStartPre=+), because dnf and zypper write to /var and /run
|
||||
# even when only asked a question, and the sandbox makes those read-only. A failure here only means no figure.
|
||||
# --check-updates for a person at a terminal: runs the same check, shows what the package manager said and the result.
|
||||
# --prepare the things that can't be done inside the systemd sandbox: asks the package manager what is waiting (dnf and zypper
|
||||
# write to /var and /run even when only asked), reads disk health with smartctl and pool state with zpool (both need the
|
||||
# devices, which the sandbox closes), and — if an administrator has switched it on — replaces this script with a newer
|
||||
# one. The answers are left in the unit's private directory for the report to pick up. The unit runs this step outside
|
||||
# its sandbox (ExecStartPre=-+); a failure here only means those figures are missing. --prepare-updates is the same.
|
||||
# --check-updates for a person at a terminal: runs the same checks and shows what each said, and whether it would update itself.
|
||||
MODE=report
|
||||
case "${1:-}" in
|
||||
--prepare-updates) MODE=prepare ;;
|
||||
--check-updates) MODE=check ;;
|
||||
--prepare|--prepare-updates) MODE=prepare ;;
|
||||
--check-updates|--check) MODE=check ;;
|
||||
esac
|
||||
UPDATES_FILE="${UPDATES_FILE:-/run/homelab-manager-agent/updates.json}"
|
||||
HEALTH_FILE="${HEALTH_FILE:-$(dirname "$UPDATES_FILE")/health.json}"
|
||||
# Kept between runs (the unit's private directory is not): a SMART reading is only taken about once an hour, so sleeping disks are left alone.
|
||||
STATE_DIR="${STATE_DIR:-/var/lib/homelab-manager-agent}"
|
||||
# Where the package managers' own error output goes: nowhere, unless a person asked to see it.
|
||||
PM_ERR=/dev/null
|
||||
[[ "$MODE" == "check" ]] && PM_ERR=/dev/stderr
|
||||
@@ -228,6 +234,16 @@ collect_updates() {
|
||||
if command -v needs-restarting >/dev/null 2>&1; then
|
||||
needs-restarting -r >/dev/null 2>&1 || reboot="true"
|
||||
fi
|
||||
elif command -v pacman >/dev/null 2>&1; then
|
||||
manager="pacman"
|
||||
# -Qu is what the local copy of the sync databases says could be upgraded (what `checkupdates` lists, without changing anything). It
|
||||
# exits 1 when there is nothing to upgrade, so an empty answer with status 1 is "none", not an error. Packages held back are left out.
|
||||
out=$(LC_ALL=C timeout 60 pacman -Qu 2>"$PM_ERR") || status=$?
|
||||
if [[ "$status" -eq 0 || ( "$status" -eq 1 && -z "$out" ) ]]; then
|
||||
count=$(grep -v '\[ignored\]' <<<"$out" | grep -c . || true)
|
||||
fi
|
||||
# No security figure (pacman doesn't say). A restart is due when the kernel running is no longer the one installed.
|
||||
[[ -d "/usr/lib/modules/$(uname -r)" ]] || reboot="true"
|
||||
elif command -v zypper >/dev/null 2>&1; then
|
||||
manager="zypper"
|
||||
if out=$(LC_ALL=C timeout 120 zypper -q --no-refresh list-updates 2>"$PM_ERR"); then
|
||||
@@ -248,7 +264,164 @@ collect_updates() {
|
||||
fi
|
||||
}
|
||||
|
||||
# The answer --prepare-updates left for this report, if it is there, recent and looks like what it should. Read back through jq rather than
|
||||
# >>> post_report
|
||||
# Sends $PAYLOAD to the server and prints the HTTP status code.
|
||||
#
|
||||
# Nothing sensitive goes on curl's command line: any user on this machine can read other processes' command lines, and the
|
||||
# token would be there for the length of every report. So the token is handed to curl on its standard input, as a line of
|
||||
# curl's own config format, and the payload and the server's reply go through files in a private directory (mktemp -d is
|
||||
# readable by root only). A fixed name in /tmp is no good either — another user could plant a symlink there and have this
|
||||
# script, running as root, overwrite whatever it points at.
|
||||
curl_quote() {
|
||||
local s=$1
|
||||
s=${s//\\/\\\\}
|
||||
s=${s//\"/\\\"}
|
||||
printf '%s' "$s"
|
||||
}
|
||||
|
||||
post_report() {
|
||||
case "$API_TOKEN" in
|
||||
*$'\n'*|*$'\r'*) echo "API_TOKEN must not contain a line break." >&2; return 1 ;;
|
||||
esac
|
||||
printf '%s' "$PAYLOAD" > "$HLM_TMPDIR/payload.json"
|
||||
printf 'header = "Authorization: Bearer %s"\n' "$(curl_quote "$API_TOKEN")" | curl -sS "${CURL_INSECURE_FLAG[@]}" -K - \
|
||||
-o "$HLM_TMPDIR/response.json" -w "%{http_code}" \
|
||||
-X POST "$API_URL/api/agent/report" \
|
||||
-H "Content-Type: application/json" \
|
||||
--data-binary @"$HLM_TMPDIR/payload.json"
|
||||
}
|
||||
# <<< post_report
|
||||
|
||||
# A GET with the token handed to curl on its standard input (as the upload does), the reply into the file named; prints the HTTP status.
|
||||
api_get() {
|
||||
printf 'header = "Authorization: Bearer %s"\n' "$(curl_quote "$API_TOKEN")" | curl -sS "${CURL_INSECURE_FLAG[@]}" -K - -m 30 -o "$2" -w "%{http_code}" "$API_URL$1"
|
||||
}
|
||||
|
||||
# Services (systemd units) that have failed. systemctl is asked the way the timers are, so this works inside the sandbox.
|
||||
collect_failed_units() {
|
||||
FAILED_UNITS_JSON="null"
|
||||
command -v systemctl >/dev/null 2>&1 || return 0
|
||||
local out
|
||||
out=$(timeout 20 systemctl list-units --state=failed --plain --no-legend --no-pager --full 2>"$PM_ERR") || return 0
|
||||
FAILED_UNITS_JSON=$(awk 'NF > 0 { unit = $1; $1 = $2 = $3 = $4 = ""; sub(/^ +/, ""); print unit "\t" $0 }' <<<"$out" \
|
||||
| jq -R -s -c 'split("\n") | map(select(length > 0) | split("\t")) | map({unit: .[0], description: (.[1] // "")}) | .[0:100]') || FAILED_UNITS_JSON="null"
|
||||
}
|
||||
|
||||
# Each disk's own SMART verdict, temperature and (where it says) how much of its rated life is used. Needs the devices, so it runs from
|
||||
# --prepare. Asked at most about once an hour, and with -n standby so a sleeping disk is not woken: what was last known of it is kept.
|
||||
collect_smart() {
|
||||
SMART_JSON="null"
|
||||
command -v smartctl >/dev/null 2>&1 || return 0
|
||||
local cache="$STATE_DIR/smart.json" now_s previous="[]" items="[]" dev out item scan checked
|
||||
now_s=$(date +%s)
|
||||
if [[ -r "$cache" ]]; then
|
||||
checked=$(jq -r '.checked_at // 0' "$cache" 2>/dev/null || echo 0)
|
||||
previous=$(jq -c '.devices // []' "$cache" 2>/dev/null || echo "[]")
|
||||
if [[ "$checked" =~ ^[0-9]+$ ]] && (( now_s - checked < 3300 )) && [[ "$MODE" != "check" ]]; then
|
||||
SMART_JSON=$previous
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
scan=$(timeout 30 smartctl --scan -j 2>"$PM_ERR") || true
|
||||
while IFS= read -r dev; do
|
||||
[[ -n "$dev" ]] || continue
|
||||
# smartctl's exit status is a bit mask that says "something was noted" more often than "it failed", so it is read from the JSON instead.
|
||||
out=$(timeout 30 smartctl -j -H -A -n standby "$dev" 2>"$PM_ERR") || true
|
||||
item=$(jq -c --arg dev "$dev" 'select(.smart_status != null) | {
|
||||
device: $dev,
|
||||
model: (.model_name // .scsi_model_name // ""),
|
||||
healthy: .smart_status.passed,
|
||||
temperature_c: (.temperature.current // null),
|
||||
wear_percent: (.nvme_smart_health_information_log.percentage_used // ([.ata_smart_attributes.table[]? | select(.id == 177 or .id == 231 or .id == 233) | (100 - .value)] | first) // null)
|
||||
}' <<<"$out" 2>/dev/null) || item=""
|
||||
if [[ -z "$item" ]]; then
|
||||
# Asleep, or no SMART at all (a virtual disk): keep what was known of it, if anything.
|
||||
item=$(jq -c --arg dev "$dev" '.[] | select(.device == $dev)' <<<"$previous" 2>/dev/null | head -n 1) || item=""
|
||||
fi
|
||||
if [[ -n "$item" ]]; then items=$(jq -c --argjson i "$item" '. + [$i]' <<<"$items"); fi
|
||||
done < <(jq -r '.devices[]?.name' <<<"$scan" 2>/dev/null)
|
||||
SMART_JSON=$items
|
||||
if mkdir -p "$STATE_DIR" 2>/dev/null; then
|
||||
chmod 700 "$STATE_DIR" 2>/dev/null || true
|
||||
jq -c -n --argjson at "$now_s" --argjson devices "$items" '{checked_at: $at, devices: $devices}' > "$cache.tmp" 2>/dev/null && mv -f "$cache.tmp" "$cache" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
# Each ZFS pool and whether it is ONLINE. Needs /dev/zfs, so it runs from --prepare too.
|
||||
collect_zfs() {
|
||||
ZFS_JSON="null"
|
||||
command -v zpool >/dev/null 2>&1 || return 0
|
||||
local out
|
||||
out=$(timeout 20 zpool list -H -o name,health,capacity 2>"$PM_ERR") || return 0
|
||||
ZFS_JSON=$(jq -R -s -c 'split("\n") | map(select(length > 0) | split("\t")) | map({name: .[0], health: (.[1] // "UNKNOWN"), capacity_percent: ((.[2] // "" | rtrimstr("%") | tonumber?) // null)})' <<<"$out") || ZFS_JSON="null"
|
||||
}
|
||||
|
||||
# The disk and pool readings --prepare left for this report, if they are there and recent. Re-shaped by jq, never trusted as they stand.
|
||||
use_prepared_health() {
|
||||
[[ -r "$HEALTH_FILE" ]] || return 1
|
||||
[[ -n "$(find "$HEALTH_FILE" -mmin -10 -print 2>/dev/null)" ]] || return 1
|
||||
SMART_JSON=$(jq -c 'if (.smart | type) == "array" then [.smart[] | {device: (.device | tostring), model: ((.model // "") | tostring),
|
||||
healthy: (if (.healthy | type) == "boolean" then .healthy else null end),
|
||||
temperature_c: (if (.temperature_c | type) == "number" then .temperature_c else null end),
|
||||
wear_percent: (if (.wear_percent | type) == "number" then .wear_percent else null end)}] else null end' "$HEALTH_FILE" 2>/dev/null) || SMART_JSON="null"
|
||||
ZFS_JSON=$(jq -c 'if (.zfs | type) == "array" then [.zfs[] | {name: (.name | tostring), health: ((.health // "UNKNOWN") | tostring),
|
||||
capacity_percent: (if (.capacity_percent | type) == "number" then .capacity_percent else null end)}] else null end' "$HEALTH_FILE" 2>/dev/null) || ZFS_JSON="null"
|
||||
return 0
|
||||
}
|
||||
|
||||
# Replaces this script with the one this Homelab Manager serves, when an administrator has switched automatic updates on and this agent is
|
||||
# behind. Nothing is trusted but the checksum: the (authenticated) answer gives the SHA-256 of the script, and a download that doesn't
|
||||
# match it, doesn't look like this script, doesn't pass bash's syntax check or isn't newer is thrown away. The new file is put beside the
|
||||
# old and moved over it in one step, and the new version then carries on from here. Runs only from --prepare (outside the sandbox, which
|
||||
# keeps /usr read-only for the report itself); with --check-updates it says what it would do.
|
||||
self_update() {
|
||||
[[ -n "$API_URL" && -n "$API_TOKEN" ]] || return 0
|
||||
[[ -z "${HLM_JUST_UPDATED:-}" ]] || return 0
|
||||
local self work code reply_update version sha got
|
||||
self=$(readlink -f "${BASH_SOURCE[0]}" 2>/dev/null) || return 0
|
||||
[[ -f "$self" ]] || return 0
|
||||
work=$(mktemp -d) || return 0
|
||||
code=$(api_get "/api/agent/report/update?version=$AGENT_VERSION" "$work/reply.json" 2>/dev/null) || code=""
|
||||
reply_update=$(jq -r '.update // false' "$work/reply.json" 2>/dev/null || echo false)
|
||||
if [[ "$code" != "200" || "$reply_update" != "true" ]]; then
|
||||
[[ "$MODE" == "check" ]] && echo "Automatic update: nothing to do (it is off, or this agent is current)."
|
||||
rm -rf "$work"
|
||||
return 0
|
||||
fi
|
||||
version=$(jq -r '.version // empty' "$work/reply.json" 2>/dev/null || true)
|
||||
sha=$(jq -r '.sha256 // empty' "$work/reply.json" 2>/dev/null || true)
|
||||
if [[ ! "$version" =~ ^[0-9]+$ || ! "$sha" =~ ^[0-9a-f]{64}$ ]] || (( version <= AGENT_VERSION )); then
|
||||
rm -rf "$work"
|
||||
return 0
|
||||
fi
|
||||
if ! curl -fsS "${CURL_INSECURE_FLAG[@]}" -m 60 -o "$work/new.sh" "$API_URL/agent/linux/report-tasks.sh" 2>"$PM_ERR"; then
|
||||
echo "Note: couldn't download the newer agent (version $version), so it was not installed." >&2
|
||||
rm -rf "$work"
|
||||
return 0
|
||||
fi
|
||||
got=$(sha256sum "$work/new.sh" 2>/dev/null | awk '{print $1}')
|
||||
if [[ "$got" != "$sha" ]] || [[ "$(head -n 1 "$work/new.sh")" != "#!/usr/bin/env bash" ]] || ! bash -n "$work/new.sh" 2>/dev/null || ! grep -q "^AGENT_VERSION=$version\$" "$work/new.sh"; then
|
||||
echo "Note: the newer agent (version $version) didn't check out against what the server said, so it was not installed." >&2
|
||||
rm -rf "$work"
|
||||
return 0
|
||||
fi
|
||||
if [[ "$MODE" == "check" ]]; then
|
||||
echo "Automatic update: version $version is available and would be installed on the next run."
|
||||
rm -rf "$work"
|
||||
return 0
|
||||
fi
|
||||
if install -m 755 "$work/new.sh" "$self.new" && mv -f "$self.new" "$self"; then
|
||||
echo "Updated the agent from version $AGENT_VERSION to version $version." >&2
|
||||
rm -rf "$work"
|
||||
HLM_JUST_UPDATED=1 exec "$self" --prepare
|
||||
fi
|
||||
rm -f "$self.new"
|
||||
echo "Note: couldn't replace the agent with version $version." >&2
|
||||
rm -rf "$work"
|
||||
return 0
|
||||
}
|
||||
|
||||
# The answer --prepare left for this report, if it is there, recent and looks like what it should. Read back through jq rather than
|
||||
# trusted: the report only ever carries these four fields, whatever the file holds.
|
||||
use_prepared_updates() {
|
||||
[[ -r "$UPDATES_FILE" ]] || return 1
|
||||
@@ -325,8 +498,23 @@ collect_system_info() {
|
||||
UPDATES_JSON="null"
|
||||
use_prepared_updates || collect_updates || UPDATES_JSON="null"
|
||||
|
||||
# Failed services are asked here (systemctl works inside the sandbox); disk and pool health come from --prepare. A person running --dry-run
|
||||
# isn't in the sandbox, so there they are read directly.
|
||||
FAILED_UNITS_JSON="null"
|
||||
collect_failed_units || FAILED_UNITS_JSON="null"
|
||||
SMART_JSON="null"
|
||||
ZFS_JSON="null"
|
||||
if ! use_prepared_health && [[ "$DRY_RUN" == "1" ]]; then
|
||||
collect_smart || SMART_JSON="null"
|
||||
collect_zfs || ZFS_JSON="null"
|
||||
fi
|
||||
HEALTH_JSON=$(jq -n -c --argjson failed "$FAILED_UNITS_JSON" --argjson smart "$SMART_JSON" --argjson zfs "$ZFS_JSON" \
|
||||
'{} | (if $failed != null then . + {failed_units: $failed} else . end) | (if $smart != null then . + {smart: $smart} else . end) | (if $zfs != null then . + {zfs: $zfs} else . end)') || HEALTH_JSON="{}"
|
||||
[[ "$HEALTH_JSON" == "{}" ]] && HEALTH_JSON="null"
|
||||
|
||||
SYSTEM_JSON=$(jq -n \
|
||||
--argjson updates "$UPDATES_JSON" \
|
||||
--argjson health "$HEALTH_JSON" \
|
||||
--argjson ip_addresses "$ip_json" \
|
||||
--argjson listening_ports "$ports_json" \
|
||||
--arg cpu_model "$cpu_model" \
|
||||
@@ -341,23 +529,40 @@ collect_system_info() {
|
||||
memory: { total_bytes: $mem_total_bytes, used_bytes: $mem_used_bytes },
|
||||
disks: $disks,
|
||||
listening_ports: $listening_ports,
|
||||
updates: $updates
|
||||
updates: $updates,
|
||||
health: $health
|
||||
}')
|
||||
}
|
||||
|
||||
if [[ "$MODE" != "report" ]]; then
|
||||
collect_updates
|
||||
# prepare: first see whether this script is to be replaced (if so it carries on as the new one, from the top).
|
||||
[[ "$MODE" == "prepare" ]] && self_update
|
||||
collect_updates || UPDATES_JSON="null"
|
||||
collect_smart || SMART_JSON="null"
|
||||
collect_zfs || ZFS_JSON="null"
|
||||
if [[ "$MODE" == "check" ]]; then
|
||||
collect_failed_units || FAILED_UNITS_JSON="null"
|
||||
echo "Pending updates as they would be reported:"
|
||||
jq . <<<"$UPDATES_JSON"
|
||||
echo "Failed services:"
|
||||
jq . <<<"$FAILED_UNITS_JSON"
|
||||
echo "Disk health (SMART; null means smartctl isn't installed or this isn't root):"
|
||||
jq . <<<"$SMART_JSON"
|
||||
echo "ZFS pools (null means there is no zpool here):"
|
||||
jq . <<<"$ZFS_JSON"
|
||||
self_update
|
||||
exit 0
|
||||
fi
|
||||
# prepare: leave the answer (or no file at all) for the report that follows. Written whole and moved into place, so a report never reads half of it.
|
||||
rm -f "$UPDATES_FILE"
|
||||
# Leave the answers (or no file at all) for the report that follows. Each is written whole and moved into place, so a report never reads half of one.
|
||||
rm -f "$UPDATES_FILE" "$HEALTH_FILE"
|
||||
mkdir -p "$(dirname "$UPDATES_FILE")" 2>/dev/null || exit 0
|
||||
if [[ "$UPDATES_JSON" != "null" ]]; then
|
||||
mkdir -p "$(dirname "$UPDATES_FILE")" 2>/dev/null || exit 0
|
||||
printf '%s\n' "$UPDATES_JSON" > "$UPDATES_FILE.tmp" 2>/dev/null && mv -f "$UPDATES_FILE.tmp" "$UPDATES_FILE" 2>/dev/null || true
|
||||
fi
|
||||
HEALTH_FILE_JSON=$(jq -n -c --argjson smart "${SMART_JSON:-null}" --argjson zfs "${ZFS_JSON:-null}" '{} | (if $smart != null then . + {smart: $smart} else . end) | (if $zfs != null then . + {zfs: $zfs} else . end)' 2>/dev/null || echo "{}")
|
||||
if [[ "$HEALTH_FILE_JSON" != "{}" ]]; then
|
||||
printf '%s\n' "$HEALTH_FILE_JSON" > "$HEALTH_FILE.tmp" 2>/dev/null && mv -f "$HEALTH_FILE.tmp" "$HEALTH_FILE" 2>/dev/null || true
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
@@ -392,34 +597,6 @@ if [[ "$DRY_RUN" == "1" ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# >>> post_report
|
||||
# Sends $PAYLOAD to the server and prints the HTTP status code.
|
||||
#
|
||||
# Nothing sensitive goes on curl's command line: any user on this machine can read other processes' command lines, and the
|
||||
# token would be there for the length of every report. So the token is handed to curl on its standard input, as a line of
|
||||
# curl's own config format, and the payload and the server's reply go through files in a private directory (mktemp -d is
|
||||
# readable by root only). A fixed name in /tmp is no good either — another user could plant a symlink there and have this
|
||||
# script, running as root, overwrite whatever it points at.
|
||||
curl_quote() {
|
||||
local s=$1
|
||||
s=${s//\\/\\\\}
|
||||
s=${s//\"/\\\"}
|
||||
printf '%s' "$s"
|
||||
}
|
||||
|
||||
post_report() {
|
||||
case "$API_TOKEN" in
|
||||
*$'\n'*|*$'\r'*) echo "API_TOKEN must not contain a line break." >&2; return 1 ;;
|
||||
esac
|
||||
printf '%s' "$PAYLOAD" > "$HLM_TMPDIR/payload.json"
|
||||
printf 'header = "Authorization: Bearer %s"\n' "$(curl_quote "$API_TOKEN")" | curl -sS "${CURL_INSECURE_FLAG[@]}" -K - \
|
||||
-o "$HLM_TMPDIR/response.json" -w "%{http_code}" \
|
||||
-X POST "$API_URL/api/agent/report" \
|
||||
-H "Content-Type: application/json" \
|
||||
--data-binary @"$HLM_TMPDIR/payload.json"
|
||||
}
|
||||
# <<< post_report
|
||||
|
||||
HLM_TMPDIR=$(mktemp -d)
|
||||
trap 'rm -rf "$HLM_TMPDIR"' EXIT
|
||||
|
||||
|
||||
Reference in new issue
Block a user