Agent diagnostics ("what the agent couldn't collect") and services to ignore per server

- Agents (Linux 6, Windows 5) note each check they couldn't make and why; the server page turns that into tips and fixes, including when the agent is behind
- Linux --prepare leaves its notes for the report; a missing prepare step is itself noted
- Services to ignore are set per server from the Host health card (names or * patterns), applied on the server so alerts and /metrics drop them at once; saving alert limits keeps the list

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 22:16:31 +02:00
1 parent b5384c1e03
commit c3bc6bbf72
26 files changed
+4118 -45

No files matched your search

+79 -10
View File
@@ -16,7 +16,8 @@ set -euo pipefail
# 3 = the package manager is asked outside the sandbox (see --prepare), so dnf and zypper can answer there too
# 4 = also reports failed services, disk health (SMART) and ZFS pools, reads pacman, and can update itself when told to
# 5 = also says what the machine is (maker, model, serial number), from the firmware, for the inventory
AGENT_VERSION=5
# 6 = also says what it could not collect and why (no smartctl, no systemd, a package manager that failed ...), so the server can say what to fix
AGENT_VERSION=6
ENV_FILE="${ENV_FILE:-/etc/homelab-manager-agent.env}"
if [[ -f "$ENV_FILE" ]]; then
@@ -44,6 +45,8 @@ case "${1:-}" in
esac
UPDATES_FILE="${UPDATES_FILE:-/run/homelab-manager-agent/updates.json}"
HEALTH_FILE="${HEALTH_FILE:-$(dirname "$UPDATES_FILE")/health.json}"
# What --prepare could not collect and why, left for the report to carry along.
DIAG_FILE="${DIAG_FILE:-$(dirname "$UPDATES_FILE")/diagnostics.json}"
# Kept between runs (the unit's private directory is not): a SMART reading is only taken about once an hour, so sleeping disks are left alone.
STATE_DIR="${STATE_DIR:-/var/lib/homelab-manager-agent}"
# Where the package managers' own error output goes: nowhere, unless a person asked to see it.
@@ -55,6 +58,23 @@ if [[ "$MODE" == "report" && ( -z "$API_URL" || -z "$API_TOKEN" ) ]]; then
exit 1
fi
# Things that could not be collected, each a small JSON object: which check, whether it is "unavailable" (nothing to read it with, or nothing
# there to read) or "failed" (it tried and could not), why in a word, and at most a line of what the tool said. Only problems are noted: a
# check that is not mentioned went fine. The server turns these into what to do about them.
DIAG_ITEMS=()
diag() {
local check="$1" status="$2" reason="${3:-}" detail="${4:-}"
detail=$(printf '%s' "$detail" | tr -d '\000-\037' | head -c 160)
DIAG_ITEMS+=("$(jq -n -c --arg c "$check" --arg s "$status" --arg r "$reason" --arg d "$detail" \
'{check: $c, status: $s} + (if $r != "" then {reason: $r} else {} end) + (if $d != "" then {detail: $d} else {} end)')")
}
# The notes made so far as one JSON array ([] when there are none).
diag_json() {
if (( ${#DIAG_ITEMS[@]} == 0 )); then echo "[]"; return 0; fi
printf '%s\n' "${DIAG_ITEMS[@]}" | jq -s -c '.'
}
CURL_INSECURE_FLAG=()
case "${API_INSECURE,,}" in
1|true|yes) CURL_INSECURE_FLAG=(-k) ;;
@@ -253,9 +273,12 @@ collect_updates() {
[[ -f /run/reboot-needed ]] && reboot="true"
fi
if [[ -n "$manager" && ! "$count" =~ ^[0-9]+$ ]]; then
if [[ -z "$manager" ]]; then
diag updates unavailable no_package_manager
elif [[ ! "$count" =~ ^[0-9]+$ ]]; then
# In the journal (systemctl status / journalctl -u homelab-manager-agent) so "not reported" has a reason to look for.
echo "Note: couldn't read the pending updates from $manager - the report goes without them." >&2
diag updates failed error "$manager did not answer"
fi
if [[ -n "$manager" && "$count" =~ ^[0-9]+$ ]]; then
@@ -325,7 +348,10 @@ collect_hardware() {
maker=$(dmi_value sys_vendor board_vendor)
model=$(dmi_value product_name board_name)
serial=$(dmi_value product_serial board_serial)
[[ -n "$maker$model$serial" ]] || return 0
if [[ -z "$maker$model$serial" ]]; then
diag hardware unavailable not_found
return 0
fi
HARDWARE_JSON=$(jq -n -c --arg maker "$maker" --arg model "$model" --arg serial "$serial" \
'{} | (if $maker != "" then . + {manufacturer: $maker} else . end) | (if $model != "" then . + {model: $model} else . end) | (if $serial != "" then . + {serial: $serial} else . end)') || HARDWARE_JSON="null"
}
@@ -333,9 +359,15 @@ collect_hardware() {
# Services (systemd units) that have failed. systemctl is asked the way the timers are, so this works inside the sandbox.
collect_failed_units() {
FAILED_UNITS_JSON="null"
command -v systemctl >/dev/null 2>&1 || return 0
if ! command -v systemctl >/dev/null 2>&1; then
diag failed_units unavailable no_systemd
return 0
fi
local out
out=$(timeout 20 systemctl list-units --state=failed --plain --no-legend --no-pager --full 2>"$PM_ERR") || return 0
if ! out=$(timeout 20 systemctl list-units --state=failed --plain --no-legend --no-pager --full 2>"$PM_ERR"); then
diag failed_units failed error "systemctl list-units did not answer"
return 0
fi
FAILED_UNITS_JSON=$(awk 'NF > 0 { unit = $1; $1 = $2 = $3 = $4 = ""; sub(/^ +/, ""); print unit "\t" $0 }' <<<"$out" \
| jq -R -s -c 'split("\n") | map(select(length > 0) | split("\t")) | map({unit: .[0], description: (.[1] // "")}) | .[0:100]') || FAILED_UNITS_JSON="null"
}
@@ -344,7 +376,14 @@ collect_failed_units() {
# --prepare. Asked at most about once an hour, and with -n standby so a sleeping disk is not woken: what was last known of it is kept.
collect_smart() {
SMART_JSON="null"
command -v smartctl >/dev/null 2>&1 || return 0
if ! command -v smartctl >/dev/null 2>&1; then
diag smart unavailable tool_missing smartctl
return 0
fi
if [[ "$(id -u)" != "0" ]]; then
diag smart unavailable not_root
return 0
fi
local cache="$STATE_DIR/smart.json" now_s previous="[]" items="[]" dev out item scan checked
now_s=$(date +%s)
if [[ -r "$cache" ]]; then
@@ -383,9 +422,15 @@ collect_smart() {
# Each ZFS pool and whether it is ONLINE. Needs /dev/zfs, so it runs from --prepare too.
collect_zfs() {
ZFS_JSON="null"
command -v zpool >/dev/null 2>&1 || return 0
if ! command -v zpool >/dev/null 2>&1; then
diag zfs unavailable tool_missing zpool
return 0
fi
local out
out=$(timeout 20 zpool list -H -o name,health,capacity 2>"$PM_ERR") || return 0
if ! out=$(timeout 20 zpool list -H -o name,health,capacity 2>"$PM_ERR"); then
diag zfs failed error "zpool list did not answer"
return 0
fi
ZFS_JSON=$(jq -R -s -c 'split("\n") | map(select(length > 0) | split("\t")) | map({name: .[0], health: (.[1] // "UNKNOWN"), capacity_percent: ((.[2] // "" | rtrimstr("%") | tonumber?) // null)})' <<<"$out") || ZFS_JSON="null"
}
@@ -466,6 +511,22 @@ use_prepared_updates() {
UPDATES_JSON=$json
}
# What this report could not collect: the notes made here, then those --prepare left (it does the outside-the-sandbox part), one for each check.
# With nothing left by --prepare — on a real run, not a dry one — that step did not run, which is worth saying: disk health and ZFS are only read there.
collect_diagnostics() {
local from_prepare="[]" shaped
if [[ -r "$DIAG_FILE" && -n "$(find "$DIAG_FILE" -mmin -10 -print 2>/dev/null)" ]]; then
from_prepare=$(jq -c 'if type == "array" then [.[] | select(type == "object" and (.check | type) == "string" and (.status | type) == "string")
| {check: (.check | tostring), status: (.status | tostring)}
+ (if ((.reason // "") | tostring) != "" then {reason: (.reason | tostring)} else {} end)
+ (if ((.detail // "") | tostring) != "" then {detail: (.detail | tostring | .[0:160])} else {} end)] | .[0:20] else [] end' "$DIAG_FILE" 2>/dev/null) || from_prepare="[]"
elif [[ "$DRY_RUN" != "1" ]]; then
diag prepare unavailable step_missing
fi
shaped=$(jq -c -n --argjson here "$(diag_json)" --argjson there "$from_prepare" '($here + $there) | unique_by(.check) | .[0:20]') || shaped="[]"
DIAGNOSTICS_JSON=$shaped
}
collect_system_info() {
local ip_json="[]"
if command -v ip >/dev/null 2>&1; then
@@ -543,6 +604,7 @@ collect_system_info() {
fi
HARDWARE_JSON="null"
collect_hardware || HARDWARE_JSON="null"
collect_diagnostics
HEALTH_JSON=$(jq -n -c --argjson failed "$FAILED_UNITS_JSON" --argjson smart "$SMART_JSON" --argjson zfs "$ZFS_JSON" \
'{} | (if $failed != null then . + {failed_units: $failed} else . end) | (if $smart != null then . + {smart: $smart} else . end) | (if $zfs != null then . + {zfs: $zfs} else . end)') || HEALTH_JSON="{}"
[[ "$HEALTH_JSON" == "{}" ]] && HEALTH_JSON="null"
@@ -551,6 +613,7 @@ collect_system_info() {
--argjson updates "$UPDATES_JSON" \
--argjson health "$HEALTH_JSON" \
--argjson hardware "$HARDWARE_JSON" \
--argjson diagnostics "$DIAGNOSTICS_JSON" \
--argjson ip_addresses "$ip_json" \
--argjson listening_ports "$ports_json" \
--arg cpu_model "$cpu_model" \
@@ -567,7 +630,8 @@ collect_system_info() {
listening_ports: $listening_ports,
updates: $updates,
health: $health,
hardware: $hardware
hardware: $hardware,
diagnostics: $diagnostics
}')
}
@@ -587,12 +651,17 @@ if [[ "$MODE" != "report" ]]; then
jq . <<<"$SMART_JSON"
echo "ZFS pools (null means there is no zpool here):"
jq . <<<"$ZFS_JSON"
collect_hardware || true
echo "What could not be collected (empty means nothing was missed):"
diag_json | jq .
self_update
exit 0
fi
# Leave the answers (or no file at all) for the report that follows. Each is written whole and moved into place, so a report never reads half of one.
rm -f "$UPDATES_FILE" "$HEALTH_FILE"
rm -f "$UPDATES_FILE" "$HEALTH_FILE" "$DIAG_FILE"
mkdir -p "$(dirname "$UPDATES_FILE")" 2>/dev/null || exit 0
# The notes always go, even when there are none: a missing file is how the report knows this step didn't run.
diag_json > "$DIAG_FILE.tmp" 2>/dev/null && mv -f "$DIAG_FILE.tmp" "$DIAG_FILE" 2>/dev/null || true
if [[ "$UPDATES_JSON" != "null" ]]; then
printf '%s\n' "$UPDATES_JSON" > "$UPDATES_FILE.tmp" 2>/dev/null && mv -f "$UPDATES_FILE.tmp" "$UPDATES_FILE" 2>/dev/null || true
fi
+29 -8
View File
@@ -35,10 +35,23 @@ $ErrorActionPreference = 'Stop'
# change here is worth installing the agent again for. 1 = before the number was reported 2 = reports pending updates, and its own version
# 3 = can replace itself with the newer script when an administrator has switched that on in Homelab Manager
# 4 = also reports services that have failed, disk health, Storage Spaces pools and what the machine is (maker, model, serial)
$script:AgentVersion = 4
# 5 = also says what it could not collect and why, so Homelab Manager can say what to fix
$script:AgentVersion = 5
$script:DefaultConfigPath = Join-Path $env:ProgramData 'HomelabManager\agent.json'
# Things that could not be collected: which check, whether it is 'unavailable' (nothing there to read) or 'failed' (it tried and could not), why in a
# word, and at most a line of what Windows said. Only problems are noted; a check that is not mentioned went fine.
$script:Diag = New-Object System.Collections.ArrayList
function Add-Diag([string]$Check, [string]$Status, [string]$Reason = '', [string]$Detail = '') {
$item = [ordered]@{ check = $Check; status = $Status }
if ($Reason) { $item['reason'] = $Reason }
$clean = ($Detail -replace '[\x00-\x1f]', ' ').Trim()
if ($clean.Length -gt 160) { $clean = $clean.Substring(0, 160) }
if ($clean) { $item['detail'] = $clean }
[void]$script:Diag.Add([pscustomobject]$item)
}
# ---- configuration ------------------------------------------------------------
function Get-AgentConfig {
@@ -287,7 +300,10 @@ function Get-PendingUpdates {
}
# No answer is no answer: the report then leaves updates out, and the server keeps what it was last told.
if ($cached.failed) { return $null }
if ($cached.failed) {
Add-Diag 'updates' 'failed' 'error' 'Windows Update did not answer; it is asked again in an hour'
return $null
}
return [ordered]@{ manager = 'windows-update'; count = [int]$cached.count; security = [int]$cached.security; reboot_required = (Test-RebootRequired) }
}
@@ -355,7 +371,7 @@ function Get-HostHealth($Config = $null) {
$health = [ordered]@{}
$ignore = @()
if ($Config -and $Config.IgnoreServices) { $ignore = @($Config.IgnoreServices) }
try { $health['failed_units'] = @(ConvertTo-FailedUnits @(Get-CimInstance -ClassName Win32_Service -ErrorAction Stop) $ignore) } catch { Write-Warning "Reading services failed. ($($_.Exception.Message))" }
try { $health['failed_units'] = @(ConvertTo-FailedUnits @(Get-CimInstance -ClassName Win32_Service -ErrorAction Stop) $ignore) } catch { Write-Warning "Reading services failed. ($($_.Exception.Message))"; Add-Diag 'failed_units' 'failed' 'error' $_.Exception.Message }
try {
$disks = @(Get-PhysicalDisk -ErrorAction Stop)
$counters = @{}
@@ -363,12 +379,12 @@ function Get-HostHealth($Config = $null) {
try { $counters[[string]$d.DeviceId] = Get-StorageReliabilityCounter -PhysicalDisk $d -ErrorAction Stop } catch { }
}
$health['smart'] = @(ConvertTo-DiskHealth $disks $counters)
} catch { Write-Warning "Reading disk health failed. ($($_.Exception.Message))" }
} catch { Write-Warning "Reading disk health failed. ($($_.Exception.Message))"; Add-Diag 'smart' 'failed' 'error' $_.Exception.Message }
try {
$health['zfs'] = @(ConvertTo-PoolHealth @(Get-StoragePool -ErrorAction Stop | Where-Object { -not $_.IsPrimordial }))
} catch {
# A machine with no Storage Spaces pools gets an error, not an empty list: that is an answer ("no pools"), not a failure to read.
if ($_.FullyQualifiedErrorId -like '*NotFound*') { $health['zfs'] = @() } else { Write-Warning "Reading storage pools failed. ($($_.Exception.Message))" }
if ($_.FullyQualifiedErrorId -like '*NotFound*') { $health['zfs'] = @() } else { Write-Warning "Reading storage pools failed. ($($_.Exception.Message))"; Add-Diag 'zfs' 'failed' 'error' $_.Exception.Message }
}
if ($health.Count -eq 0) { return $null }
return $health
@@ -402,6 +418,7 @@ function Get-HardwareInfo {
}
function Get-SystemInfo($Config = $null) {
$script:Diag.Clear()
$ips = @(Get-NetIPAddress -AddressFamily IPv4 -ErrorAction SilentlyContinue |
Where-Object { $_.IPAddress -notlike '127.*' -and $_.IPAddress -notlike '169.254.*' -and $_.AddressState -eq 'Preferred' } |
ForEach-Object { $_.IPAddress } | Select-Object -Unique)
@@ -423,18 +440,22 @@ function Get-SystemInfo($Config = $null) {
# Best-effort: Windows Update being unreachable must not cost the rest of the report.
$updates = $null
try { $updates = Get-PendingUpdates } catch { Write-Warning "Checking for updates failed. ($($_.Exception.Message))" }
try { $updates = Get-PendingUpdates } catch { Write-Warning "Checking for updates failed. ($($_.Exception.Message))"; Add-Diag 'updates' 'failed' 'error' $_.Exception.Message }
# The same for what is wrong with the machine, and what it is: neither may cost the rest of the report.
$health = $null
try { $health = Get-HostHealth $Config } catch { Write-Warning "Reading host health failed. ($($_.Exception.Message))" }
try { $health = Get-HostHealth $Config } catch { Write-Warning "Reading host health failed. ($($_.Exception.Message))"; Add-Diag 'host_health' 'failed' 'error' $_.Exception.Message }
$hardware = $null
try { $hardware = Get-HardwareInfo } catch { Write-Warning "Reading the hardware identity failed. ($($_.Exception.Message))" }
try {
$hardware = Get-HardwareInfo
if (-not $hardware) { Add-Diag 'hardware' 'unavailable' 'not_found' }
} catch { Write-Warning "Reading the hardware identity failed. ($($_.Exception.Message))"; Add-Diag 'hardware' 'failed' 'error' $_.Exception.Message }
return [ordered]@{
updates = $updates
health = $health
hardware = $hardware
diagnostics = @($script:Diag)
ip_addresses = @($ips)
cpu = [ordered]@{ model = [string]($cpus[0].Name).Trim(); cores = [int]$cores; load_percent = $(if ($null -ne $load) { [math]::Round([double]$load, 1) } else { $null }) }
memory = [ordered]@{ total_bytes = $totalBytes; used_bytes = ($totalBytes - $freeBytes) }