Files
Homelab-manager/agent/linux/report-tasks.sh
T
bobbanandClaude Sonnet 5.5 41fbbf3467 Report uptime from the agents, and stop the Integrations page overflowing a phone
Linux agent 8 and Windows agent 7 say how long the machine has been up; the
server stores it (migration 0086) and shows it as Up for in the status strip
(left out while offline) and Uptime on the server page.

The Integrations header wraps and its button no longer squeezes, and
.table-responsive is positioned so hidden text in scrolled-off columns cannot
widen the page.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-11-08 21:50:44 +01:00

758 lines
36 KiB
Bash

#!/usr/bin/env bash
# Collects cron jobs and systemd timers on this host and POSTs them to the
# Homelab Manager API. Intended to run as root via a periodic systemd timer
# (see install.sh) but can be run manually for testing:
#
# API_URL=https://homelab.example.lan API_TOKEN=hlm_xxx ./report-tasks.sh --dry-run
#
# Set API_INSECURE=true (also written to the env file by install.sh when
# passed there) if Homelab Manager uses a self-signed certificate.
#
set -euo pipefail
# Which version of this script this is, sent with every report so Homelab Manager can tell which agents are behind. Raise it whenever a
# change here is worth installing the agent again for (and add it to the list at the top of server/src/services/agentVersions.ts).
# 1 = before the number was reported 2 = reports pending updates, and its own version
# 3 = the package manager is asked outside the sandbox (see --prepare), so dnf and zypper can answer there too
# 4 = also reports failed services, disk health (SMART) and ZFS pools, reads pacman, and can update itself when told to
# 5 = also says what the machine is (maker, model, serial number), from the firmware, for the inventory
# 6 = also says what it could not collect and why (no smartctl, no systemd, a package manager that failed ...), so the server can say what to fix
AGENT_VERSION=8
ENV_FILE="${ENV_FILE:-/etc/homelab-manager-agent.env}"
if [[ -f "$ENV_FILE" ]]; then
# shellcheck disable=SC1090
source "$ENV_FILE"
fi
API_URL="${API_URL:-}"
API_TOKEN="${API_TOKEN:-}"
API_INSECURE="${API_INSECURE:-false}"
DRY_RUN=0
[[ "${1:-}" == "--dry-run" ]] && DRY_RUN=1
# Two extra ways to run it, neither of which reports anything:
# --prepare the things that can't be done inside the systemd sandbox: asks the package manager what is waiting (dnf and zypper
# write to /var and /run even when only asked), reads disk health with smartctl and pool state with zpool (both need the
# devices, which the sandbox closes), and — if an administrator has switched it on — replaces this script with a newer
# one. The answers are left in the unit's private directory for the report to pick up. The unit runs this step outside
# its sandbox (ExecStartPre=-+); a failure here only means those figures are missing. --prepare-updates is the same.
# --check-updates for a person at a terminal: runs the same checks and shows what each said, and whether it would update itself.
MODE=report
case "${1:-}" in
--prepare|--prepare-updates) MODE=prepare ;;
--check-updates|--check) MODE=check ;;
esac
UPDATES_FILE="${UPDATES_FILE:-/run/homelab-manager-agent/updates.json}"
HEALTH_FILE="${HEALTH_FILE:-$(dirname "$UPDATES_FILE")/health.json}"
# What --prepare could not collect and why, left for the report to carry along.
DIAG_FILE="${DIAG_FILE:-$(dirname "$UPDATES_FILE")/diagnostics.json}"
# Kept between runs (the unit's private directory is not): a SMART reading is only taken about once an hour, so sleeping disks are left alone.
STATE_DIR="${STATE_DIR:-/var/lib/homelab-manager-agent}"
# Where the package managers' own error output goes: nowhere, unless a person asked to see it.
PM_ERR=/dev/null
[[ "$MODE" == "check" ]] && PM_ERR=/dev/stderr
if [[ "$MODE" == "report" && ( -z "$API_URL" || -z "$API_TOKEN" ) ]]; then
echo "API_URL and API_TOKEN must be set (env vars or $ENV_FILE)" >&2
exit 1
fi
# Things that could not be collected, each a small JSON object: which check, whether it is "unavailable" (nothing to read it with, or nothing
# there to read) or "failed" (it tried and could not), why in a word, and at most a line of what the tool said. Only problems are noted: a
# check that is not mentioned went fine. The server turns these into what to do about them.
DIAG_ITEMS=()
diag() {
local check="$1" status="$2" reason="${3:-}" detail="${4:-}"
detail=$(printf '%s' "$detail" | tr -d '\000-\037' | head -c 160)
DIAG_ITEMS+=("$(jq -n -c --arg c "$check" --arg s "$status" --arg r "$reason" --arg d "$detail" \
'{check: $c, status: $s} + (if $r != "" then {reason: $r} else {} end) + (if $d != "" then {detail: $d} else {} end)')")
}
# The notes made so far as one JSON array ([] when there are none).
diag_json() {
if (( ${#DIAG_ITEMS[@]} == 0 )); then echo "[]"; return 0; fi
printf '%s\n' "${DIAG_ITEMS[@]}" | jq -s -c '.'
}
CURL_INSECURE_FLAG=()
case "${API_INSECURE,,}" in
1|true|yes) CURL_INSECURE_FLAG=(-k) ;;
esac
suggest_package_install() {
local pkg="$1"
if command -v apt-get >/dev/null 2>&1; then
echo " sudo apt-get update && sudo apt-get install -y $pkg"
elif command -v dnf >/dev/null 2>&1; then
echo " sudo dnf install -y $pkg"
elif command -v yum >/dev/null 2>&1; then
echo " sudo yum install -y $pkg"
elif command -v apk >/dev/null 2>&1; then
echo " sudo apk add $pkg"
elif command -v pacman >/dev/null 2>&1; then
echo " sudo pacman -S $pkg"
elif command -v zypper >/dev/null 2>&1; then
echo " sudo zypper install -y $pkg"
fi
}
for bin in curl jq; do
if ! command -v "$bin" >/dev/null 2>&1; then
echo "Required dependency '$bin' is not installed. Try:" >&2
suggest_package_install "$bin" >&2
exit 1
fi
done
TASKS_JSON="[]"
add_task() {
local schedule_type="$1" name="$2" command="$3" schedule_expression="$4" source="$5" enabled="$6"
TASKS_JSON=$(jq -c \
--arg schedule_type "$schedule_type" \
--arg name "$name" \
--arg command "$command" \
--arg schedule_expression "$schedule_expression" \
--arg source "$source" \
--argjson enabled "$enabled" \
'. + [{
schedule_type: $schedule_type,
name: $name,
command: $command,
schedule_expression: $schedule_expression,
source: $source,
enabled: $enabled
}]' <<<"$TASKS_JSON")
}
add_task_with_next_run() {
local schedule_type="$1" name="$2" command="$3" schedule_expression="$4" source="$5" enabled="$6" next_run_at="$7"
TASKS_JSON=$(jq -c \
--arg schedule_type "$schedule_type" \
--arg name "$name" \
--arg command "$command" \
--arg schedule_expression "$schedule_expression" \
--arg source "$source" \
--argjson enabled "$enabled" \
--arg next_run_at "$next_run_at" \
'. + [{
schedule_type: $schedule_type,
name: $name,
command: $command,
schedule_expression: $schedule_expression,
source: $source,
enabled: $enabled,
next_run_at: $next_run_at
}]' <<<"$TASKS_JSON")
}
parse_crontab_lines() {
# Reads 5-field-schedule + command cron lines from stdin.
# $1 = source label, $2 = "system" (7-field, includes a user column) or "user" (6-field)
local source="$1" mode="$2"
while IFS= read -r line; do
line="${line%%$'\r'}"
[[ -z "$line" ]] && continue
[[ "$line" =~ ^[[:space:]]*# ]] && continue
[[ "$line" =~ ^[[:space:]]*[A-Za-z_][A-Za-z0-9_]*= ]] && continue
if [[ "$mode" == "system" ]]; then
# min hour dom mon dow user command...
if [[ "$line" =~ ^[[:space:]]*([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+[^[:space:]]+[[:space:]]+(.+)$ ]]; then
local sched="${BASH_REMATCH[1]} ${BASH_REMATCH[2]} ${BASH_REMATCH[3]} ${BASH_REMATCH[4]} ${BASH_REMATCH[5]}"
local cmd="${BASH_REMATCH[6]}"
add_task "cron" "$cmd" "$cmd" "$sched" "$source" true
fi
else
# min hour dom mon dow command...
if [[ "$line" =~ ^[[:space:]]*([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+(.+)$ ]]; then
local sched="${BASH_REMATCH[1]} ${BASH_REMATCH[2]} ${BASH_REMATCH[3]} ${BASH_REMATCH[4]} ${BASH_REMATCH[5]}"
local cmd="${BASH_REMATCH[6]}"
add_task "cron" "$cmd" "$cmd" "$sched" "$source" true
fi
fi
done
}
collect_cron() {
[[ -r /etc/crontab ]] && parse_crontab_lines "/etc/crontab" "system" < /etc/crontab
if [[ -d /etc/cron.d ]]; then
for f in /etc/cron.d/*; do
[[ -f "$f" && -r "$f" ]] || continue
parse_crontab_lines "$f" "system" < "$f"
done
fi
if command -v crontab >/dev/null 2>&1 && [[ -r /etc/passwd ]]; then
while IFS=: read -r user _ uid _ _ _ shell; do
case "$shell" in
*/nologin|*/false|"") continue ;;
esac
[[ "$uid" -lt 1000 && "$uid" != "0" ]] && continue
local_crontab=$(crontab -l -u "$user" 2>/dev/null || true)
[[ -z "$local_crontab" ]] && continue
parse_crontab_lines "crontab:$user" "user" <<<"$local_crontab"
done < /etc/passwd
fi
}
collect_systemd_timers() {
command -v systemctl >/dev/null 2>&1 || return 0
local timers_json
timers_json=$(systemctl list-timers --all --output=json 2>/dev/null || echo "[]")
while IFS= read -r entry; do
local unit activates next_usec enabled_state schedule_expr next_run_at
unit=$(jq -r '.unit' <<<"$entry")
activates=$(jq -r '.activates // ""' <<<"$entry")
next_usec=$(jq -r '.next // 0' <<<"$entry")
enabled_state=$(systemctl is-enabled "$unit" 2>/dev/null || echo "unknown")
local enabled_bool="false"
[[ "$enabled_state" == "enabled" || "$enabled_state" == "static" ]] && enabled_bool="true"
schedule_expr=$(systemctl show "$unit" -p TimersCalendar --value 2>/dev/null | sed -n 's/.*OnCalendar=\([^;]*\);.*/\1/p' | sed 's/[[:space:]]*$//')
[[ -z "$schedule_expr" ]] && schedule_expr="(see: systemctl status $unit)"
next_run_at=""
if [[ "$next_usec" =~ ^[0-9]+$ && "$next_usec" -gt 0 ]]; then
next_run_at=$(date -u -d "@$((next_usec / 1000000))" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || echo "")
fi
if [[ -n "$next_run_at" ]]; then
add_task_with_next_run "systemd_timer" "$unit" "$activates" "$schedule_expr" "$unit" "$enabled_bool" "$next_run_at"
else
add_task "systemd_timer" "$unit" "$activates" "$schedule_expr" "$unit" "$enabled_bool"
fi
done < <(jq -c '.[]' <<<"$timers_json")
}
# Updates waiting to be installed, as JSON in UPDATES_JSON ("null" when this host's package manager isn't one we know, or couldn't answer).
# Nothing is installed or refreshed: this reads what the package manager already knows about, so it is quick and changes nothing.
collect_updates() {
UPDATES_JSON="null"
local manager="" count="" security="null" reboot="false" out="" status=0
if command -v apt-get >/dev/null 2>&1; then
manager="apt"
# -s simulates: "Inst" lines are what an upgrade would install. Security ones come from a "-security" or "Security" archive.
if out=$(LC_ALL=C timeout 60 apt-get -s -o Debug::NoLocking=true upgrade 2>"$PM_ERR"); then
count=$(grep -c '^Inst ' <<<"$out" || true)
security=$(grep '^Inst ' <<<"$out" | grep -ci 'security' || true)
fi
[[ -f /var/run/reboot-required ]] && reboot="true"
elif command -v dnf >/dev/null 2>&1; then
manager="dnf"
# check-update exits 100 when there are updates, 0 when none, 1 on an error. --cacheonly: never goes to the network from here.
out=$(LC_ALL=C timeout 120 dnf -q check-update --cacheonly 2>"$PM_ERR") || status=$?
if [[ "$status" -eq 0 || "$status" -eq 100 ]]; then
count=$(awk 'NF >= 3 && $1 ~ /^[A-Za-z0-9_.+-]+\.[A-Za-z0-9_]+$/ && /Obsoleting/ == 0 { n++ } /^Obsoleting/ { exit } END { print n + 0 }' <<<"$out")
security=$(LC_ALL=C timeout 120 dnf -q updateinfo list --security --cacheonly 2>"$PM_ERR" | grep -c '/Sec\.' || true)
fi
if command -v needs-restarting >/dev/null 2>&1; then
needs-restarting -r >/dev/null 2>&1 || reboot="true"
fi
elif command -v pacman >/dev/null 2>&1; then
manager="pacman"
# -Qu is what the local copy of the sync databases says could be upgraded (what `checkupdates` lists, without changing anything). It
# exits 1 when there is nothing to upgrade, so an empty answer with status 1 is "none", not an error. Packages held back are left out.
out=$(LC_ALL=C timeout 60 pacman -Qu 2>"$PM_ERR") || status=$?
if [[ "$status" -eq 0 || ( "$status" -eq 1 && -z "$out" ) ]]; then
count=$(grep -v '\[ignored\]' <<<"$out" | grep -c . || true)
fi
# No security figure (pacman doesn't say). A restart is due when the kernel running is no longer the one installed.
[[ -d "/usr/lib/modules/$(uname -r)" ]] || reboot="true"
elif command -v zypper >/dev/null 2>&1; then
manager="zypper"
if out=$(LC_ALL=C timeout 120 zypper -q --no-refresh list-updates 2>"$PM_ERR"); then
count=$(grep -c '^v |' <<<"$out" || true)
fi
[[ -f /run/reboot-needed ]] && reboot="true"
fi
if [[ -z "$manager" ]]; then
diag updates unavailable no_package_manager
elif [[ ! "$count" =~ ^[0-9]+$ ]]; then
# In the journal (systemctl status / journalctl -u homelab-manager-agent) so "not reported" has a reason to look for.
echo "Note: couldn't read the pending updates from $manager - the report goes without them." >&2
diag updates failed error "$manager did not answer"
fi
if [[ -n "$manager" && "$count" =~ ^[0-9]+$ ]]; then
[[ "$security" =~ ^[0-9]+$ ]] || security="null"
UPDATES_JSON=$(jq -n --arg manager "$manager" --argjson count "$count" --argjson security "$security" --argjson reboot "$reboot" \
'{manager: $manager, count: $count, security: $security, reboot_required: $reboot}')
fi
}
# >>> post_report
# Sends $PAYLOAD to the server and prints the HTTP status code.
#
# Nothing sensitive goes on curl's command line: any user on this machine can read other processes' command lines, and the
# token would be there for the length of every report. So the token is handed to curl on its standard input, as a line of
# curl's own config format, and the payload and the server's reply go through files in a private directory (mktemp -d is
# readable by root only). A fixed name in /tmp is no good either — another user could plant a symlink there and have this
# script, running as root, overwrite whatever it points at.
curl_quote() {
local s=$1
s=${s//\\/\\\\}
s=${s//\"/\\\"}
printf '%s' "$s"
}
post_report() {
case "$API_TOKEN" in
*$'\n'*|*$'\r'*) echo "API_TOKEN must not contain a line break." >&2; return 1 ;;
esac
printf '%s' "$PAYLOAD" > "$HLM_TMPDIR/payload.json"
printf 'header = "Authorization: Bearer %s"\n' "$(curl_quote "$API_TOKEN")" | curl -sS "${CURL_INSECURE_FLAG[@]}" -K - \
-o "$HLM_TMPDIR/response.json" -w "%{http_code}" \
-X POST "$API_URL/api/agent/report" \
-H "Content-Type: application/json" \
--data-binary @"$HLM_TMPDIR/payload.json"
}
# <<< post_report
# A GET with the token handed to curl on its standard input (as the upload does), the reply into the file named; prints the HTTP status.
api_get() {
printf 'header = "Authorization: Bearer %s"\n' "$(curl_quote "$API_TOKEN")" | curl -sS "${CURL_INSECURE_FLAG[@]}" -K - -m 30 -o "$2" -w "%{http_code}" "$API_URL$1"
}
# One value from the firmware's table (the kernel shows it under /sys/class/dmi/id), trying the named files in turn and leaving out what a
# maker writes when it has nothing to say ("To Be Filled By O.E.M.", "Default string", all zeros ...). DMI_DIR is for the tests.
dmi_value() {
local name value
for name in "$@"; do
# (the file is checked first: a failing redirection prints its complaint before any 2>/dev/null applies)
[[ -r "${DMI_DIR:-/sys/class/dmi/id}/$name" ]] || continue
value=$(tr -d '\0' < "${DMI_DIR:-/sys/class/dmi/id}/$name" 2>/dev/null | head -c 200 | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') || value=""
[[ -n "$value" ]] || continue
case "${value,,}" in
"to be filled by o.e.m."|"default string"|"system serial number"|"system product name"|"system manufacturer"|"not specified"|"not applicable"|"none"|"n/a"|"unknown"|"o.e.m."|"oem") continue ;;
esac
[[ "$value" =~ ^0+$ ]] && continue
printf '%s' "$value"
return 0
done
return 0
}
# What the machine is: maker, model and serial number (a serial is only readable by root). "null" when the firmware has nothing to say, as on
# many virtual machines and single-board computers.
collect_hardware() {
HARDWARE_JSON="null"
local maker model serial
maker=$(dmi_value sys_vendor board_vendor)
model=$(dmi_value product_name board_name)
serial=$(dmi_value product_serial board_serial)
if [[ -z "$maker$model$serial" ]]; then
diag hardware unavailable not_found
return 0
fi
HARDWARE_JSON=$(jq -n -c --arg maker "$maker" --arg model "$model" --arg serial "$serial" \
'{} | (if $maker != "" then . + {manufacturer: $maker} else . end) | (if $model != "" then . + {model: $model} else . end) | (if $serial != "" then . + {serial: $serial} else . end)') || HARDWARE_JSON="null"
}
# What the machine runs, as /etc/os-release says (OS_RELEASE_FILE is for the tests): the distribution's id and version, so the app can say when that release stops
# being supported. Proxmox VE is Debian underneath, so it is asked by its own command. "null" when there is nothing to say.
collect_os() {
OS_JSON="null"
local file="${OS_RELEASE_FILE:-/etc/os-release}" id="" version="" pretty=""
if [[ -r "$file" ]]; then
id=$(sed -n 's/^ID=//p' "$file" | head -n 1 | tr -d '"' | tr 'A-Z' 'a-z' | head -c 40)
version=$(sed -n 's/^VERSION_ID=//p' "$file" | head -n 1 | tr -d '"' | head -c 40)
pretty=$(sed -n 's/^PRETTY_NAME=//p' "$file" | head -n 1 | tr -d '"' | head -c 80)
fi
if command -v pveversion >/dev/null 2>&1; then
local pve
pve=$(pveversion 2>/dev/null | sed -n 's#^pve-manager/\([0-9]*\.[0-9]*\).*#\1#p' | head -n 1)
if [[ -n "$pve" ]]; then
id="proxmox-ve"
version="$pve"
pretty="Proxmox VE $pve"
fi
fi
[[ -n "$id" ]] || return 0
OS_JSON=$(jq -n -c --arg id "$id" --arg version "$version" --arg name "$pretty" \
'{id: $id} | (if $version != "" then . + {version: $version} else . end) | (if $name != "" then . + {name: $name} else . end)') || OS_JSON="null"
}
# Services (systemd units) that have failed. systemctl is asked the way the timers are, so this works inside the sandbox.
collect_failed_units() {
FAILED_UNITS_JSON="null"
if ! command -v systemctl >/dev/null 2>&1; then
diag failed_units unavailable no_systemd
return 0
fi
local out
if ! out=$(timeout 20 systemctl list-units --state=failed --plain --no-legend --no-pager --full 2>"$PM_ERR"); then
diag failed_units failed error "systemctl list-units did not answer"
return 0
fi
FAILED_UNITS_JSON=$(awk 'NF > 0 { unit = $1; $1 = $2 = $3 = $4 = ""; sub(/^ +/, ""); print unit "\t" $0 }' <<<"$out" \
| jq -R -s -c 'split("\n") | map(select(length > 0) | split("\t")) | map({unit: .[0], description: (.[1] // "")}) | .[0:100]') || FAILED_UNITS_JSON="null"
}
# Each disk's own SMART verdict, temperature and (where it says) how much of its rated life is used. Needs the devices, so it runs from
# --prepare. Asked at most about once an hour, and with -n standby so a sleeping disk is not woken: what was last known of it is kept.
collect_smart() {
SMART_JSON="null"
if ! command -v smartctl >/dev/null 2>&1; then
diag smart unavailable tool_missing smartctl
return 0
fi
if [[ "$(id -u)" != "0" ]]; then
diag smart unavailable not_root
return 0
fi
local cache="$STATE_DIR/smart.json" now_s previous="[]" items="[]" dev out item scan checked
now_s=$(date +%s)
if [[ -r "$cache" ]]; then
checked=$(jq -r '.checked_at // 0' "$cache" 2>/dev/null || echo 0)
previous=$(jq -c '.devices // []' "$cache" 2>/dev/null || echo "[]")
if [[ "$checked" =~ ^[0-9]+$ ]] && (( now_s - checked < 3300 )) && [[ "$MODE" != "check" ]]; then
SMART_JSON=$previous
return 0
fi
fi
scan=$(timeout 30 smartctl --scan -j 2>"$PM_ERR") || true
while IFS= read -r dev; do
[[ -n "$dev" ]] || continue
# smartctl's exit status is a bit mask that says "something was noted" more often than "it failed", so it is read from the JSON instead.
out=$(timeout 30 smartctl -j -H -A -n standby "$dev" 2>"$PM_ERR") || true
item=$(jq -c --arg dev "$dev" 'select(.smart_status != null) | {
device: $dev,
model: (.model_name // .scsi_model_name // ""),
healthy: .smart_status.passed,
temperature_c: (.temperature.current // null),
wear_percent: (.nvme_smart_health_information_log.percentage_used // ([.ata_smart_attributes.table[]? | select(.id == 177 or .id == 231 or .id == 233) | (100 - .value)] | first) // null)
}' <<<"$out" 2>/dev/null) || item=""
if [[ -z "$item" ]]; then
# Asleep, or no SMART at all (a virtual disk): keep what was known of it, if anything.
item=$(jq -c --arg dev "$dev" '.[] | select(.device == $dev)' <<<"$previous" 2>/dev/null | head -n 1) || item=""
fi
if [[ -n "$item" ]]; then items=$(jq -c --argjson i "$item" '. + [$i]' <<<"$items"); fi
done < <(jq -r '.devices[]?.name' <<<"$scan" 2>/dev/null)
SMART_JSON=$items
if mkdir -p "$STATE_DIR" 2>/dev/null; then
chmod 700 "$STATE_DIR" 2>/dev/null || true
jq -c -n --argjson at "$now_s" --argjson devices "$items" '{checked_at: $at, devices: $devices}' > "$cache.tmp" 2>/dev/null && mv -f "$cache.tmp" "$cache" 2>/dev/null || true
fi
}
# Each ZFS pool and whether it is ONLINE. Needs /dev/zfs, so it runs from --prepare too.
collect_zfs() {
ZFS_JSON="null"
if ! command -v zpool >/dev/null 2>&1; then
diag zfs unavailable tool_missing zpool
return 0
fi
local out
if ! out=$(timeout 20 zpool list -H -o name,health,capacity 2>"$PM_ERR"); then
diag zfs failed error "zpool list did not answer"
return 0
fi
ZFS_JSON=$(jq -R -s -c 'split("\n") | map(select(length > 0) | split("\t")) | map({name: .[0], health: (.[1] // "UNKNOWN"), capacity_percent: ((.[2] // "" | rtrimstr("%") | tonumber?) // null)})' <<<"$out") || ZFS_JSON="null"
}
# The disk and pool readings --prepare left for this report, if they are there and recent. Re-shaped by jq, never trusted as they stand.
use_prepared_health() {
[[ -r "$HEALTH_FILE" ]] || return 1
[[ -n "$(find "$HEALTH_FILE" -mmin -10 -print 2>/dev/null)" ]] || return 1
SMART_JSON=$(jq -c 'if (.smart | type) == "array" then [.smart[] | {device: (.device | tostring), model: ((.model // "") | tostring),
healthy: (if (.healthy | type) == "boolean" then .healthy else null end),
temperature_c: (if (.temperature_c | type) == "number" then .temperature_c else null end),
wear_percent: (if (.wear_percent | type) == "number" then .wear_percent else null end)}] else null end' "$HEALTH_FILE" 2>/dev/null) || SMART_JSON="null"
ZFS_JSON=$(jq -c 'if (.zfs | type) == "array" then [.zfs[] | {name: (.name | tostring), health: ((.health // "UNKNOWN") | tostring),
capacity_percent: (if (.capacity_percent | type) == "number" then .capacity_percent else null end)}] else null end' "$HEALTH_FILE" 2>/dev/null) || ZFS_JSON="null"
return 0
}
# Replaces this script with the one this Homelab Manager serves, when an administrator has switched automatic updates on and this agent is
# behind. Nothing is trusted but the checksum: the (authenticated) answer gives the SHA-256 of the script, and a download that doesn't
# match it, doesn't look like this script, doesn't pass bash's syntax check or isn't newer is thrown away. The new file is put beside the
# old and moved over it in one step, and the new version then carries on from here. Runs only from --prepare (outside the sandbox, which
# keeps /usr read-only for the report itself); with --check-updates it says what it would do.
self_update() {
[[ -n "$API_URL" && -n "$API_TOKEN" ]] || return 0
[[ -z "${HLM_JUST_UPDATED:-}" ]] || return 0
local self work code reply_update version sha got
self=$(readlink -f "${BASH_SOURCE[0]}" 2>/dev/null) || return 0
[[ -f "$self" ]] || return 0
work=$(mktemp -d) || return 0
code=$(api_get "/api/agent/report/update?version=$AGENT_VERSION" "$work/reply.json" 2>/dev/null) || code=""
reply_update=$(jq -r '.update // false' "$work/reply.json" 2>/dev/null || echo false)
if [[ "$code" != "200" || "$reply_update" != "true" ]]; then
[[ "$MODE" == "check" ]] && echo "Automatic update: nothing to do (it is off, or this agent is current)."
rm -rf "$work"
return 0
fi
version=$(jq -r '.version // empty' "$work/reply.json" 2>/dev/null || true)
sha=$(jq -r '.sha256 // empty' "$work/reply.json" 2>/dev/null || true)
if [[ ! "$version" =~ ^[0-9]+$ || ! "$sha" =~ ^[0-9a-f]{64}$ ]] || (( version <= AGENT_VERSION )); then
rm -rf "$work"
return 0
fi
if ! curl -fsS "${CURL_INSECURE_FLAG[@]}" -m 60 -o "$work/new.sh" "$API_URL/agent/linux/report-tasks.sh" 2>"$PM_ERR"; then
echo "Note: couldn't download the newer agent (version $version), so it was not installed." >&2
rm -rf "$work"
return 0
fi
got=$(sha256sum "$work/new.sh" 2>/dev/null | awk '{print $1}')
if [[ "$got" != "$sha" ]] || [[ "$(head -n 1 "$work/new.sh")" != "#!/usr/bin/env bash" ]] || ! bash -n "$work/new.sh" 2>/dev/null || ! grep -q "^AGENT_VERSION=$version\$" "$work/new.sh"; then
echo "Note: the newer agent (version $version) didn't check out against what the server said, so it was not installed." >&2
rm -rf "$work"
return 0
fi
if [[ "$MODE" == "check" ]]; then
echo "Automatic update: version $version is available and would be installed on the next run."
rm -rf "$work"
return 0
fi
if install -m 755 "$work/new.sh" "$self.new" && mv -f "$self.new" "$self"; then
echo "Updated the agent from version $AGENT_VERSION to version $version." >&2
rm -rf "$work"
HLM_JUST_UPDATED=1 exec "$self" --prepare
fi
rm -f "$self.new"
echo "Note: couldn't replace the agent with version $version." >&2
rm -rf "$work"
return 0
}
# The answer --prepare left for this report, if it is there, recent and looks like what it should. Read back through jq rather than
# trusted: the report only ever carries these four fields, whatever the file holds.
use_prepared_updates() {
[[ -r "$UPDATES_FILE" ]] || return 1
[[ -n "$(find "$UPDATES_FILE" -mmin -10 -print 2>/dev/null)" ]] || return 1
local json
json=$(jq -ce 'select(type == "object" and (.manager | type) == "string" and (.count | type) == "number")
| {manager: .manager, count: .count, security: (if (.security | type) == "number" then .security else null end), reboot_required: (.reboot_required == true)}' "$UPDATES_FILE" 2>/dev/null) || return 1
[[ -n "$json" ]] || return 1
UPDATES_JSON=$json
}
# What this report could not collect: the notes made here, then those --prepare left (it does the outside-the-sandbox part), one for each check.
# With nothing left by --prepare — on a real run, not a dry one — that step did not run, which is worth saying: disk health and ZFS are only read there.
collect_diagnostics() {
local from_prepare="[]" shaped
if [[ -r "$DIAG_FILE" && -n "$(find "$DIAG_FILE" -mmin -10 -print 2>/dev/null)" ]]; then
from_prepare=$(jq -c 'if type == "array" then [.[] | select(type == "object" and (.check | type) == "string" and (.status | type) == "string")
| {check: (.check | tostring), status: (.status | tostring)}
+ (if ((.reason // "") | tostring) != "" then {reason: (.reason | tostring)} else {} end)
+ (if ((.detail // "") | tostring) != "" then {detail: (.detail | tostring | .[0:160])} else {} end)] | .[0:20] else [] end' "$DIAG_FILE" 2>/dev/null) || from_prepare="[]"
elif [[ "$DRY_RUN" != "1" ]]; then
diag prepare unavailable step_missing
fi
shaped=$(jq -c -n --argjson here "$(diag_json)" --argjson there "$from_prepare" '($here + $there) | unique_by(.check) | .[0:20]') || shaped="[]"
DIAGNOSTICS_JSON=$shaped
}
collect_system_info() {
local ip_json="[]"
if command -v ip >/dev/null 2>&1; then
ip_json=$(ip -4 -o addr show scope global 2>/dev/null \
| awk '{print $4}' | cut -d/ -f1 \
| jq -R -s -c 'split("\n") | map(select(length > 0))')
fi
local cpu_model="" cpu_cores=0 cpu_load_percent="null"
# x86 /proc/cpuinfo has a per-core "model name" line. Most 64-bit ARM
# kernels (Raspberry Pi included) don't — they instead have a single
# "Model" line at the very end, or nothing at all, in which case
# /proc/device-tree/model (present on any device-tree/SBC board) has the
# board's friendly name. Try each in turn; leave blank if none exist.
cpu_model=$(grep -m1 "^model name" /proc/cpuinfo 2>/dev/null | cut -d: -f2- | sed 's/^ *//' || true)
if [[ -z "$cpu_model" ]]; then
cpu_model=$(grep -m1 "^Model" /proc/cpuinfo 2>/dev/null | cut -d: -f2- | sed 's/^ *//' || true)
fi
if [[ -z "$cpu_model" && -r /proc/device-tree/model ]]; then
cpu_model=$(tr -d '\0' < /proc/device-tree/model 2>/dev/null || true)
fi
cpu_cores=$(nproc 2>/dev/null || echo 0)
if [[ -r /proc/loadavg && "$cpu_cores" -gt 0 ]]; then
local load1
load1=$(awk '{print $1}' /proc/loadavg)
cpu_load_percent=$(awk -v l="$load1" -v c="$cpu_cores" 'BEGIN { printf "%.1f", (l/c)*100 }')
fi
# How long the host has been up: the first figure in /proc/uptime, in seconds. null if it cannot be read.
local uptime_seconds="null"
if [[ -r /proc/uptime ]]; then
uptime_seconds=$(awk '{ printf "%d", $1 }' /proc/uptime 2>/dev/null || true)
[[ "$uptime_seconds" =~ ^[0-9]+$ ]] || uptime_seconds="null"
fi
local mem_total=0 mem_used=0
if command -v free >/dev/null 2>&1; then
read -r mem_total mem_used < <(free -b | awk '/^Mem:/ {print $2, $3}')
fi
local disks_json="[]"
if command -v df >/dev/null 2>&1; then
disks_json=$(df -B1 --output=target,size,used -x tmpfs -x devtmpfs -x squashfs -x overlay 2>/dev/null \
| tail -n +2 \
| awk '{print $1"\t"$2"\t"$3}' \
| jq -R -s -c '
split("\n") | map(select(length > 0) | split("\t")) |
map({mount: .[0], size_bytes: (.[1]|tonumber), used_bytes: (.[2]|tonumber)})
')
fi
# Everything bound to a port on this host, including services listening on localhost only (which a network
# scan from elsewhere can't see). `ss -p` needs root to name the process; without it the process is blank.
# One row per socket — the server groups them per port.
local ports_json="[]"
if command -v ss >/dev/null 2>&1; then
ports_json=$(ss -H -tulnp 2>/dev/null \
| awk '{
local = $5; port = local; sub(/.*:/, "", port); addr = local; sub(/:[0-9]+$/, "", addr);
proc = ""; if (match($0, /users:\(\("[^"]+"/)) { proc = substr($0, RSTART + 9, RLENGTH - 10) }
if (port ~ /^[0-9]+$/) print $1 "\t" port "\t" addr "\t" proc
}' \
| jq -R -s -c '
split("\n") | map(select(length > 0) | split("\t")) |
map({protocol: .[0], port: (.[1]|tonumber), address: .[2], process: (.[3] // "")})
')
fi
# Best-effort like the rest: a package manager that misbehaves must not cost the report.
UPDATES_JSON="null"
use_prepared_updates || collect_updates || UPDATES_JSON="null"
# Failed services are asked here (systemctl works inside the sandbox); disk and pool health come from --prepare. A person running --dry-run
# isn't in the sandbox, so there they are read directly.
FAILED_UNITS_JSON="null"
collect_failed_units || FAILED_UNITS_JSON="null"
SMART_JSON="null"
ZFS_JSON="null"
if ! use_prepared_health && [[ "$DRY_RUN" == "1" ]]; then
collect_smart || SMART_JSON="null"
collect_zfs || ZFS_JSON="null"
fi
HARDWARE_JSON="null"
collect_hardware || HARDWARE_JSON="null"
OS_JSON="null"
collect_os || OS_JSON="null"
collect_diagnostics
HEALTH_JSON=$(jq -n -c --argjson failed "$FAILED_UNITS_JSON" --argjson smart "$SMART_JSON" --argjson zfs "$ZFS_JSON" \
'{} | (if $failed != null then . + {failed_units: $failed} else . end) | (if $smart != null then . + {smart: $smart} else . end) | (if $zfs != null then . + {zfs: $zfs} else . end)') || HEALTH_JSON="{}"
[[ "$HEALTH_JSON" == "{}" ]] && HEALTH_JSON="null"
SYSTEM_JSON=$(jq -n \
--argjson updates "$UPDATES_JSON" \
--argjson health "$HEALTH_JSON" \
--argjson hardware "$HARDWARE_JSON" \
--argjson os "$OS_JSON" \
--argjson diagnostics "$DIAGNOSTICS_JSON" \
--argjson ip_addresses "$ip_json" \
--argjson listening_ports "$ports_json" \
--arg cpu_model "$cpu_model" \
--argjson cpu_cores "${cpu_cores:-0}" \
--argjson cpu_load_percent "$cpu_load_percent" \
--argjson mem_total_bytes "${mem_total:-0}" \
--argjson mem_used_bytes "${mem_used:-0}" \
--argjson uptime_seconds "$uptime_seconds" \
--argjson disks "$disks_json" \
'{
ip_addresses: $ip_addresses,
cpu: { model: $cpu_model, cores: $cpu_cores, load_percent: $cpu_load_percent },
memory: { total_bytes: $mem_total_bytes, used_bytes: $mem_used_bytes },
uptime_seconds: $uptime_seconds,
disks: $disks,
listening_ports: $listening_ports,
updates: $updates,
health: $health,
hardware: $hardware,
os: $os,
diagnostics: $diagnostics
}')
}
if [[ "$MODE" != "report" ]]; then
# prepare: first see whether this script is to be replaced (if so it carries on as the new one, from the top).
[[ "$MODE" == "prepare" ]] && self_update
collect_updates || UPDATES_JSON="null"
collect_smart || SMART_JSON="null"
collect_zfs || ZFS_JSON="null"
if [[ "$MODE" == "check" ]]; then
collect_failed_units || FAILED_UNITS_JSON="null"
echo "Pending updates as they would be reported:"
jq . <<<"$UPDATES_JSON"
echo "Failed services:"
jq . <<<"$FAILED_UNITS_JSON"
echo "Disk health (SMART; null means smartctl isn't installed or this isn't root):"
jq . <<<"$SMART_JSON"
echo "ZFS pools (null means there is no zpool here):"
jq . <<<"$ZFS_JSON"
collect_hardware || true
collect_os || true
echo "What it runs (null means there is no os-release to read):"
jq . <<<"$OS_JSON"
echo "What could not be collected (empty means nothing was missed):"
diag_json | jq .
self_update
exit 0
fi
# Leave the answers (or no file at all) for the report that follows. Each is written whole and moved into place, so a report never reads half of one.
rm -f "$UPDATES_FILE" "$HEALTH_FILE" "$DIAG_FILE"
mkdir -p "$(dirname "$UPDATES_FILE")" 2>/dev/null || exit 0
# The notes always go, even when there are none: a missing file is how the report knows this step didn't run.
diag_json > "$DIAG_FILE.tmp" 2>/dev/null && mv -f "$DIAG_FILE.tmp" "$DIAG_FILE" 2>/dev/null || true
if [[ "$UPDATES_JSON" != "null" ]]; then
printf '%s\n' "$UPDATES_JSON" > "$UPDATES_FILE.tmp" 2>/dev/null && mv -f "$UPDATES_FILE.tmp" "$UPDATES_FILE" 2>/dev/null || true
fi
HEALTH_FILE_JSON=$(jq -n -c --argjson smart "${SMART_JSON:-null}" --argjson zfs "${ZFS_JSON:-null}" '{} | (if $smart != null then . + {smart: $smart} else . end) | (if $zfs != null then . + {zfs: $zfs} else . end)' 2>/dev/null || echo "{}")
if [[ "$HEALTH_FILE_JSON" != "{}" ]]; then
printf '%s\n' "$HEALTH_FILE_JSON" > "$HEALTH_FILE.tmp" 2>/dev/null && mv -f "$HEALTH_FILE.tmp" "$HEALTH_FILE" 2>/dev/null || true
fi
exit 0
fi
collect_cron
collect_systemd_timers
# Hardware/network facts are best-effort: a quirk on any given host (e.g. no
# "model name" line in /proc/cpuinfo on some ARM boards) must never abort the
# whole report over it, so errexit is relaxed just for this call.
SYSTEM_JSON="null"
set +e
collect_system_info
system_info_status=$?
set -e
if [[ "$system_info_status" -ne 0 ]]; then
echo "Warning: collecting hardware/network info failed (exit $system_info_status) — reporting tasks without it." >&2
SYSTEM_JSON="null"
fi
HOSTNAME_VALUE=$(hostname -f 2>/dev/null || hostname)
PAYLOAD=$(jq -n \
--arg hostname "$HOSTNAME_VALUE" \
--arg os_type "linux" \
--arg reported_at "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
--argjson agent_version "$AGENT_VERSION" \
--argjson system "$SYSTEM_JSON" \
--argjson tasks "$TASKS_JSON" \
'{hostname: $hostname, os_type: $os_type, reported_at: $reported_at, agent_version: $agent_version, system: $system, tasks: $tasks}')
if [[ "$DRY_RUN" == "1" ]]; then
echo "$PAYLOAD" | jq .
exit 0
fi
HLM_TMPDIR=$(mktemp -d)
trap 'rm -rf "$HLM_TMPDIR"' EXIT
response=$(post_report)
if [[ "$response" -lt 200 || "$response" -ge 300 ]]; then
echo "Report failed with HTTP $response:" >&2
cat "$HLM_TMPDIR/response.json" >&2
exit 1
fi
echo "Reported $(jq 'length' <<<"$TASKS_JSON") task(s) successfully."