Agent diagnostics ("what the agent couldn't collect") and services to ignore per server
- Agents (Linux 6, Windows 5) note each check they couldn't make and why; the server page turns that into tips and fixes, including when the agent is behind - Linux --prepare leaves its notes for the report; a missing prepare step is itself noted - Services to ignore are set per server from the Host health card (names or * patterns), applied on the server so alerts and /metrics drop them at once; saving alert limits keeps the list Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b5384c1e03
commit
c3bc6bbf72
26 files changed
+4118
-45
No files matched your search
+79
-10
@@ -16,7 +16,8 @@ set -euo pipefail
|
||||
# 3 = the package manager is asked outside the sandbox (see --prepare), so dnf and zypper can answer there too
|
||||
# 4 = also reports failed services, disk health (SMART) and ZFS pools, reads pacman, and can update itself when told to
|
||||
# 5 = also says what the machine is (maker, model, serial number), from the firmware, for the inventory
|
||||
AGENT_VERSION=5
|
||||
# 6 = also says what it could not collect and why (no smartctl, no systemd, a package manager that failed ...), so the server can say what to fix
|
||||
AGENT_VERSION=6
|
||||
|
||||
ENV_FILE="${ENV_FILE:-/etc/homelab-manager-agent.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
@@ -44,6 +45,8 @@ case "${1:-}" in
|
||||
esac
|
||||
UPDATES_FILE="${UPDATES_FILE:-/run/homelab-manager-agent/updates.json}"
|
||||
HEALTH_FILE="${HEALTH_FILE:-$(dirname "$UPDATES_FILE")/health.json}"
|
||||
# What --prepare could not collect and why, left for the report to carry along.
|
||||
DIAG_FILE="${DIAG_FILE:-$(dirname "$UPDATES_FILE")/diagnostics.json}"
|
||||
# Kept between runs (the unit's private directory is not): a SMART reading is only taken about once an hour, so sleeping disks are left alone.
|
||||
STATE_DIR="${STATE_DIR:-/var/lib/homelab-manager-agent}"
|
||||
# Where the package managers' own error output goes: nowhere, unless a person asked to see it.
|
||||
@@ -55,6 +58,23 @@ if [[ "$MODE" == "report" && ( -z "$API_URL" || -z "$API_TOKEN" ) ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Things that could not be collected, each a small JSON object: which check, whether it is "unavailable" (nothing to read it with, or nothing
|
||||
# there to read) or "failed" (it tried and could not), why in a word, and at most a line of what the tool said. Only problems are noted: a
|
||||
# check that is not mentioned went fine. The server turns these into what to do about them.
|
||||
DIAG_ITEMS=()
|
||||
diag() {
|
||||
local check="$1" status="$2" reason="${3:-}" detail="${4:-}"
|
||||
detail=$(printf '%s' "$detail" | tr -d '\000-\037' | head -c 160)
|
||||
DIAG_ITEMS+=("$(jq -n -c --arg c "$check" --arg s "$status" --arg r "$reason" --arg d "$detail" \
|
||||
'{check: $c, status: $s} + (if $r != "" then {reason: $r} else {} end) + (if $d != "" then {detail: $d} else {} end)')")
|
||||
}
|
||||
|
||||
# The notes made so far as one JSON array ([] when there are none).
|
||||
diag_json() {
|
||||
if (( ${#DIAG_ITEMS[@]} == 0 )); then echo "[]"; return 0; fi
|
||||
printf '%s\n' "${DIAG_ITEMS[@]}" | jq -s -c '.'
|
||||
}
|
||||
|
||||
CURL_INSECURE_FLAG=()
|
||||
case "${API_INSECURE,,}" in
|
||||
1|true|yes) CURL_INSECURE_FLAG=(-k) ;;
|
||||
@@ -253,9 +273,12 @@ collect_updates() {
|
||||
[[ -f /run/reboot-needed ]] && reboot="true"
|
||||
fi
|
||||
|
||||
if [[ -n "$manager" && ! "$count" =~ ^[0-9]+$ ]]; then
|
||||
if [[ -z "$manager" ]]; then
|
||||
diag updates unavailable no_package_manager
|
||||
elif [[ ! "$count" =~ ^[0-9]+$ ]]; then
|
||||
# In the journal (systemctl status / journalctl -u homelab-manager-agent) so "not reported" has a reason to look for.
|
||||
echo "Note: couldn't read the pending updates from $manager - the report goes without them." >&2
|
||||
diag updates failed error "$manager did not answer"
|
||||
fi
|
||||
|
||||
if [[ -n "$manager" && "$count" =~ ^[0-9]+$ ]]; then
|
||||
@@ -325,7 +348,10 @@ collect_hardware() {
|
||||
maker=$(dmi_value sys_vendor board_vendor)
|
||||
model=$(dmi_value product_name board_name)
|
||||
serial=$(dmi_value product_serial board_serial)
|
||||
[[ -n "$maker$model$serial" ]] || return 0
|
||||
if [[ -z "$maker$model$serial" ]]; then
|
||||
diag hardware unavailable not_found
|
||||
return 0
|
||||
fi
|
||||
HARDWARE_JSON=$(jq -n -c --arg maker "$maker" --arg model "$model" --arg serial "$serial" \
|
||||
'{} | (if $maker != "" then . + {manufacturer: $maker} else . end) | (if $model != "" then . + {model: $model} else . end) | (if $serial != "" then . + {serial: $serial} else . end)') || HARDWARE_JSON="null"
|
||||
}
|
||||
@@ -333,9 +359,15 @@ collect_hardware() {
|
||||
# Services (systemd units) that have failed. systemctl is asked the way the timers are, so this works inside the sandbox.
|
||||
collect_failed_units() {
|
||||
FAILED_UNITS_JSON="null"
|
||||
command -v systemctl >/dev/null 2>&1 || return 0
|
||||
if ! command -v systemctl >/dev/null 2>&1; then
|
||||
diag failed_units unavailable no_systemd
|
||||
return 0
|
||||
fi
|
||||
local out
|
||||
out=$(timeout 20 systemctl list-units --state=failed --plain --no-legend --no-pager --full 2>"$PM_ERR") || return 0
|
||||
if ! out=$(timeout 20 systemctl list-units --state=failed --plain --no-legend --no-pager --full 2>"$PM_ERR"); then
|
||||
diag failed_units failed error "systemctl list-units did not answer"
|
||||
return 0
|
||||
fi
|
||||
FAILED_UNITS_JSON=$(awk 'NF > 0 { unit = $1; $1 = $2 = $3 = $4 = ""; sub(/^ +/, ""); print unit "\t" $0 }' <<<"$out" \
|
||||
| jq -R -s -c 'split("\n") | map(select(length > 0) | split("\t")) | map({unit: .[0], description: (.[1] // "")}) | .[0:100]') || FAILED_UNITS_JSON="null"
|
||||
}
|
||||
@@ -344,7 +376,14 @@ collect_failed_units() {
|
||||
# --prepare. Asked at most about once an hour, and with -n standby so a sleeping disk is not woken: what was last known of it is kept.
|
||||
collect_smart() {
|
||||
SMART_JSON="null"
|
||||
command -v smartctl >/dev/null 2>&1 || return 0
|
||||
if ! command -v smartctl >/dev/null 2>&1; then
|
||||
diag smart unavailable tool_missing smartctl
|
||||
return 0
|
||||
fi
|
||||
if [[ "$(id -u)" != "0" ]]; then
|
||||
diag smart unavailable not_root
|
||||
return 0
|
||||
fi
|
||||
local cache="$STATE_DIR/smart.json" now_s previous="[]" items="[]" dev out item scan checked
|
||||
now_s=$(date +%s)
|
||||
if [[ -r "$cache" ]]; then
|
||||
@@ -383,9 +422,15 @@ collect_smart() {
|
||||
# Each ZFS pool and whether it is ONLINE. Needs /dev/zfs, so it runs from --prepare too.
|
||||
collect_zfs() {
|
||||
ZFS_JSON="null"
|
||||
command -v zpool >/dev/null 2>&1 || return 0
|
||||
if ! command -v zpool >/dev/null 2>&1; then
|
||||
diag zfs unavailable tool_missing zpool
|
||||
return 0
|
||||
fi
|
||||
local out
|
||||
out=$(timeout 20 zpool list -H -o name,health,capacity 2>"$PM_ERR") || return 0
|
||||
if ! out=$(timeout 20 zpool list -H -o name,health,capacity 2>"$PM_ERR"); then
|
||||
diag zfs failed error "zpool list did not answer"
|
||||
return 0
|
||||
fi
|
||||
ZFS_JSON=$(jq -R -s -c 'split("\n") | map(select(length > 0) | split("\t")) | map({name: .[0], health: (.[1] // "UNKNOWN"), capacity_percent: ((.[2] // "" | rtrimstr("%") | tonumber?) // null)})' <<<"$out") || ZFS_JSON="null"
|
||||
}
|
||||
|
||||
@@ -466,6 +511,22 @@ use_prepared_updates() {
|
||||
UPDATES_JSON=$json
|
||||
}
|
||||
|
||||
# What this report could not collect: the notes made here, then those --prepare left (it does the outside-the-sandbox part), one for each check.
|
||||
# With nothing left by --prepare — on a real run, not a dry one — that step did not run, which is worth saying: disk health and ZFS are only read there.
|
||||
collect_diagnostics() {
|
||||
local from_prepare="[]" shaped
|
||||
if [[ -r "$DIAG_FILE" && -n "$(find "$DIAG_FILE" -mmin -10 -print 2>/dev/null)" ]]; then
|
||||
from_prepare=$(jq -c 'if type == "array" then [.[] | select(type == "object" and (.check | type) == "string" and (.status | type) == "string")
|
||||
| {check: (.check | tostring), status: (.status | tostring)}
|
||||
+ (if ((.reason // "") | tostring) != "" then {reason: (.reason | tostring)} else {} end)
|
||||
+ (if ((.detail // "") | tostring) != "" then {detail: (.detail | tostring | .[0:160])} else {} end)] | .[0:20] else [] end' "$DIAG_FILE" 2>/dev/null) || from_prepare="[]"
|
||||
elif [[ "$DRY_RUN" != "1" ]]; then
|
||||
diag prepare unavailable step_missing
|
||||
fi
|
||||
shaped=$(jq -c -n --argjson here "$(diag_json)" --argjson there "$from_prepare" '($here + $there) | unique_by(.check) | .[0:20]') || shaped="[]"
|
||||
DIAGNOSTICS_JSON=$shaped
|
||||
}
|
||||
|
||||
collect_system_info() {
|
||||
local ip_json="[]"
|
||||
if command -v ip >/dev/null 2>&1; then
|
||||
@@ -543,6 +604,7 @@ collect_system_info() {
|
||||
fi
|
||||
HARDWARE_JSON="null"
|
||||
collect_hardware || HARDWARE_JSON="null"
|
||||
collect_diagnostics
|
||||
HEALTH_JSON=$(jq -n -c --argjson failed "$FAILED_UNITS_JSON" --argjson smart "$SMART_JSON" --argjson zfs "$ZFS_JSON" \
|
||||
'{} | (if $failed != null then . + {failed_units: $failed} else . end) | (if $smart != null then . + {smart: $smart} else . end) | (if $zfs != null then . + {zfs: $zfs} else . end)') || HEALTH_JSON="{}"
|
||||
[[ "$HEALTH_JSON" == "{}" ]] && HEALTH_JSON="null"
|
||||
@@ -551,6 +613,7 @@ collect_system_info() {
|
||||
--argjson updates "$UPDATES_JSON" \
|
||||
--argjson health "$HEALTH_JSON" \
|
||||
--argjson hardware "$HARDWARE_JSON" \
|
||||
--argjson diagnostics "$DIAGNOSTICS_JSON" \
|
||||
--argjson ip_addresses "$ip_json" \
|
||||
--argjson listening_ports "$ports_json" \
|
||||
--arg cpu_model "$cpu_model" \
|
||||
@@ -567,7 +630,8 @@ collect_system_info() {
|
||||
listening_ports: $listening_ports,
|
||||
updates: $updates,
|
||||
health: $health,
|
||||
hardware: $hardware
|
||||
hardware: $hardware,
|
||||
diagnostics: $diagnostics
|
||||
}')
|
||||
}
|
||||
|
||||
@@ -587,12 +651,17 @@ if [[ "$MODE" != "report" ]]; then
|
||||
jq . <<<"$SMART_JSON"
|
||||
echo "ZFS pools (null means there is no zpool here):"
|
||||
jq . <<<"$ZFS_JSON"
|
||||
collect_hardware || true
|
||||
echo "What could not be collected (empty means nothing was missed):"
|
||||
diag_json | jq .
|
||||
self_update
|
||||
exit 0
|
||||
fi
|
||||
# Leave the answers (or no file at all) for the report that follows. Each is written whole and moved into place, so a report never reads half of one.
|
||||
rm -f "$UPDATES_FILE" "$HEALTH_FILE"
|
||||
rm -f "$UPDATES_FILE" "$HEALTH_FILE" "$DIAG_FILE"
|
||||
mkdir -p "$(dirname "$UPDATES_FILE")" 2>/dev/null || exit 0
|
||||
# The notes always go, even when there are none: a missing file is how the report knows this step didn't run.
|
||||
diag_json > "$DIAG_FILE.tmp" 2>/dev/null && mv -f "$DIAG_FILE.tmp" "$DIAG_FILE" 2>/dev/null || true
|
||||
if [[ "$UPDATES_JSON" != "null" ]]; then
|
||||
printf '%s\n' "$UPDATES_JSON" > "$UPDATES_FILE.tmp" 2>/dev/null && mv -f "$UPDATES_FILE.tmp" "$UPDATES_FILE" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
@@ -35,10 +35,23 @@ $ErrorActionPreference = 'Stop'
|
||||
# change here is worth installing the agent again for. 1 = before the number was reported 2 = reports pending updates, and its own version
|
||||
# 3 = can replace itself with the newer script when an administrator has switched that on in Homelab Manager
|
||||
# 4 = also reports services that have failed, disk health, Storage Spaces pools and what the machine is (maker, model, serial)
|
||||
$script:AgentVersion = 4
|
||||
# 5 = also says what it could not collect and why, so Homelab Manager can say what to fix
|
||||
$script:AgentVersion = 5
|
||||
|
||||
$script:DefaultConfigPath = Join-Path $env:ProgramData 'HomelabManager\agent.json'
|
||||
|
||||
# Things that could not be collected: which check, whether it is 'unavailable' (nothing there to read) or 'failed' (it tried and could not), why in a
|
||||
# word, and at most a line of what Windows said. Only problems are noted; a check that is not mentioned went fine.
|
||||
$script:Diag = New-Object System.Collections.ArrayList
|
||||
function Add-Diag([string]$Check, [string]$Status, [string]$Reason = '', [string]$Detail = '') {
|
||||
$item = [ordered]@{ check = $Check; status = $Status }
|
||||
if ($Reason) { $item['reason'] = $Reason }
|
||||
$clean = ($Detail -replace '[\x00-\x1f]', ' ').Trim()
|
||||
if ($clean.Length -gt 160) { $clean = $clean.Substring(0, 160) }
|
||||
if ($clean) { $item['detail'] = $clean }
|
||||
[void]$script:Diag.Add([pscustomobject]$item)
|
||||
}
|
||||
|
||||
# ---- configuration ------------------------------------------------------------
|
||||
|
||||
function Get-AgentConfig {
|
||||
@@ -287,7 +300,10 @@ function Get-PendingUpdates {
|
||||
}
|
||||
|
||||
# No answer is no answer: the report then leaves updates out, and the server keeps what it was last told.
|
||||
if ($cached.failed) { return $null }
|
||||
if ($cached.failed) {
|
||||
Add-Diag 'updates' 'failed' 'error' 'Windows Update did not answer; it is asked again in an hour'
|
||||
return $null
|
||||
}
|
||||
return [ordered]@{ manager = 'windows-update'; count = [int]$cached.count; security = [int]$cached.security; reboot_required = (Test-RebootRequired) }
|
||||
}
|
||||
|
||||
@@ -355,7 +371,7 @@ function Get-HostHealth($Config = $null) {
|
||||
$health = [ordered]@{}
|
||||
$ignore = @()
|
||||
if ($Config -and $Config.IgnoreServices) { $ignore = @($Config.IgnoreServices) }
|
||||
try { $health['failed_units'] = @(ConvertTo-FailedUnits @(Get-CimInstance -ClassName Win32_Service -ErrorAction Stop) $ignore) } catch { Write-Warning "Reading services failed. ($($_.Exception.Message))" }
|
||||
try { $health['failed_units'] = @(ConvertTo-FailedUnits @(Get-CimInstance -ClassName Win32_Service -ErrorAction Stop) $ignore) } catch { Write-Warning "Reading services failed. ($($_.Exception.Message))"; Add-Diag 'failed_units' 'failed' 'error' $_.Exception.Message }
|
||||
try {
|
||||
$disks = @(Get-PhysicalDisk -ErrorAction Stop)
|
||||
$counters = @{}
|
||||
@@ -363,12 +379,12 @@ function Get-HostHealth($Config = $null) {
|
||||
try { $counters[[string]$d.DeviceId] = Get-StorageReliabilityCounter -PhysicalDisk $d -ErrorAction Stop } catch { }
|
||||
}
|
||||
$health['smart'] = @(ConvertTo-DiskHealth $disks $counters)
|
||||
} catch { Write-Warning "Reading disk health failed. ($($_.Exception.Message))" }
|
||||
} catch { Write-Warning "Reading disk health failed. ($($_.Exception.Message))"; Add-Diag 'smart' 'failed' 'error' $_.Exception.Message }
|
||||
try {
|
||||
$health['zfs'] = @(ConvertTo-PoolHealth @(Get-StoragePool -ErrorAction Stop | Where-Object { -not $_.IsPrimordial }))
|
||||
} catch {
|
||||
# A machine with no Storage Spaces pools gets an error, not an empty list: that is an answer ("no pools"), not a failure to read.
|
||||
if ($_.FullyQualifiedErrorId -like '*NotFound*') { $health['zfs'] = @() } else { Write-Warning "Reading storage pools failed. ($($_.Exception.Message))" }
|
||||
if ($_.FullyQualifiedErrorId -like '*NotFound*') { $health['zfs'] = @() } else { Write-Warning "Reading storage pools failed. ($($_.Exception.Message))"; Add-Diag 'zfs' 'failed' 'error' $_.Exception.Message }
|
||||
}
|
||||
if ($health.Count -eq 0) { return $null }
|
||||
return $health
|
||||
@@ -402,6 +418,7 @@ function Get-HardwareInfo {
|
||||
}
|
||||
|
||||
function Get-SystemInfo($Config = $null) {
|
||||
$script:Diag.Clear()
|
||||
$ips = @(Get-NetIPAddress -AddressFamily IPv4 -ErrorAction SilentlyContinue |
|
||||
Where-Object { $_.IPAddress -notlike '127.*' -and $_.IPAddress -notlike '169.254.*' -and $_.AddressState -eq 'Preferred' } |
|
||||
ForEach-Object { $_.IPAddress } | Select-Object -Unique)
|
||||
@@ -423,18 +440,22 @@ function Get-SystemInfo($Config = $null) {
|
||||
|
||||
# Best-effort: Windows Update being unreachable must not cost the rest of the report.
|
||||
$updates = $null
|
||||
try { $updates = Get-PendingUpdates } catch { Write-Warning "Checking for updates failed. ($($_.Exception.Message))" }
|
||||
try { $updates = Get-PendingUpdates } catch { Write-Warning "Checking for updates failed. ($($_.Exception.Message))"; Add-Diag 'updates' 'failed' 'error' $_.Exception.Message }
|
||||
|
||||
# The same for what is wrong with the machine, and what it is: neither may cost the rest of the report.
|
||||
$health = $null
|
||||
try { $health = Get-HostHealth $Config } catch { Write-Warning "Reading host health failed. ($($_.Exception.Message))" }
|
||||
try { $health = Get-HostHealth $Config } catch { Write-Warning "Reading host health failed. ($($_.Exception.Message))"; Add-Diag 'host_health' 'failed' 'error' $_.Exception.Message }
|
||||
$hardware = $null
|
||||
try { $hardware = Get-HardwareInfo } catch { Write-Warning "Reading the hardware identity failed. ($($_.Exception.Message))" }
|
||||
try {
|
||||
$hardware = Get-HardwareInfo
|
||||
if (-not $hardware) { Add-Diag 'hardware' 'unavailable' 'not_found' }
|
||||
} catch { Write-Warning "Reading the hardware identity failed. ($($_.Exception.Message))"; Add-Diag 'hardware' 'failed' 'error' $_.Exception.Message }
|
||||
|
||||
return [ordered]@{
|
||||
updates = $updates
|
||||
health = $health
|
||||
hardware = $hardware
|
||||
diagnostics = @($script:Diag)
|
||||
ip_addresses = @($ips)
|
||||
cpu = [ordered]@{ model = [string]($cpus[0].Name).Trim(); cores = [int]$cores; load_percent = $(if ($null -ne $load) { [math]::Round([double]$load, 1) } else { $null }) }
|
||||
memory = [ordered]@{ total_bytes = $totalBytes; used_bytes = ($totalBytes - $freeBytes) }
|
||||
|
||||
Reference in new issue
Block a user