Files
Homelab-manager/agent/linux/report-tasks.sh
T
bobbanandClaude Sonnet 5.5 c3bc6bbf72 Agent diagnostics ("what the agent couldn't collect") and services to ignore per server
- Agents (Linux 6, Windows 5) note each check they couldn't make and why; the server page turns that into tips and fixes, including when the agent is behind
- Linux --prepare leaves its notes for the report; a missing prepare step is itself noted
- Services to ignore are set per server from the Host health card (names or * patterns), applied on the server so alerts and /metrics drop them at once; saving alert limits keeps the list

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-18 22:16:31 +02:00

718 lines
34 KiB
Bash

#!/usr/bin/env bash
# Collects cron jobs and systemd timers on this host and POSTs them to the
# Homelab Manager API. Intended to run as root via a periodic systemd timer
# (see install.sh) but can be run manually for testing:
#
# API_URL=https://homelab.example.lan API_TOKEN=hlm_xxx ./report-tasks.sh --dry-run
#
# Set API_INSECURE=true (also written to the env file by install.sh when
# passed there) if Homelab Manager uses a self-signed certificate.
#
set -euo pipefail
# Which version of this script this is, sent with every report so Homelab Manager can tell which agents are behind. Raise it whenever a
# change here is worth installing the agent again for (and add it to the list at the top of server/src/services/agentVersions.ts).
# 1 = before the number was reported 2 = reports pending updates, and its own version
# 3 = the package manager is asked outside the sandbox (see --prepare), so dnf and zypper can answer there too
# 4 = also reports failed services, disk health (SMART) and ZFS pools, reads pacman, and can update itself when told to
# 5 = also says what the machine is (maker, model, serial number), from the firmware, for the inventory
# 6 = also says what it could not collect and why (no smartctl, no systemd, a package manager that failed ...), so the server can say what to fix
AGENT_VERSION=6
ENV_FILE="${ENV_FILE:-/etc/homelab-manager-agent.env}"
if [[ -f "$ENV_FILE" ]]; then
# shellcheck disable=SC1090
source "$ENV_FILE"
fi
API_URL="${API_URL:-}"
API_TOKEN="${API_TOKEN:-}"
API_INSECURE="${API_INSECURE:-false}"
DRY_RUN=0
[[ "${1:-}" == "--dry-run" ]] && DRY_RUN=1
# Two extra ways to run it, neither of which reports anything:
# --prepare the things that can't be done inside the systemd sandbox: asks the package manager what is waiting (dnf and zypper
# write to /var and /run even when only asked), reads disk health with smartctl and pool state with zpool (both need the
# devices, which the sandbox closes), and — if an administrator has switched it on — replaces this script with a newer
# one. The answers are left in the unit's private directory for the report to pick up. The unit runs this step outside
# its sandbox (ExecStartPre=-+); a failure here only means those figures are missing. --prepare-updates is the same.
# --check-updates for a person at a terminal: runs the same checks and shows what each said, and whether it would update itself.
MODE=report
case "${1:-}" in
--prepare|--prepare-updates) MODE=prepare ;;
--check-updates|--check) MODE=check ;;
esac
UPDATES_FILE="${UPDATES_FILE:-/run/homelab-manager-agent/updates.json}"
HEALTH_FILE="${HEALTH_FILE:-$(dirname "$UPDATES_FILE")/health.json}"
# What --prepare could not collect and why, left for the report to carry along.
DIAG_FILE="${DIAG_FILE:-$(dirname "$UPDATES_FILE")/diagnostics.json}"
# Kept between runs (the unit's private directory is not): a SMART reading is only taken about once an hour, so sleeping disks are left alone.
STATE_DIR="${STATE_DIR:-/var/lib/homelab-manager-agent}"
# Where the package managers' own error output goes: nowhere, unless a person asked to see it.
PM_ERR=/dev/null
[[ "$MODE" == "check" ]] && PM_ERR=/dev/stderr
if [[ "$MODE" == "report" && ( -z "$API_URL" || -z "$API_TOKEN" ) ]]; then
echo "API_URL and API_TOKEN must be set (env vars or $ENV_FILE)" >&2
exit 1
fi
# Things that could not be collected, each a small JSON object: which check, whether it is "unavailable" (nothing to read it with, or nothing
# there to read) or "failed" (it tried and could not), why in a word, and at most a line of what the tool said. Only problems are noted: a
# check that is not mentioned went fine. The server turns these into what to do about them.
DIAG_ITEMS=()
diag() {
local check="$1" status="$2" reason="${3:-}" detail="${4:-}"
detail=$(printf '%s' "$detail" | tr -d '\000-\037' | head -c 160)
DIAG_ITEMS+=("$(jq -n -c --arg c "$check" --arg s "$status" --arg r "$reason" --arg d "$detail" \
'{check: $c, status: $s} + (if $r != "" then {reason: $r} else {} end) + (if $d != "" then {detail: $d} else {} end)')")
}
# The notes made so far as one JSON array ([] when there are none).
diag_json() {
if (( ${#DIAG_ITEMS[@]} == 0 )); then echo "[]"; return 0; fi
printf '%s\n' "${DIAG_ITEMS[@]}" | jq -s -c '.'
}
CURL_INSECURE_FLAG=()
case "${API_INSECURE,,}" in
1|true|yes) CURL_INSECURE_FLAG=(-k) ;;
esac
suggest_package_install() {
local pkg="$1"
if command -v apt-get >/dev/null 2>&1; then
echo " sudo apt-get update && sudo apt-get install -y $pkg"
elif command -v dnf >/dev/null 2>&1; then
echo " sudo dnf install -y $pkg"
elif command -v yum >/dev/null 2>&1; then
echo " sudo yum install -y $pkg"
elif command -v apk >/dev/null 2>&1; then
echo " sudo apk add $pkg"
elif command -v pacman >/dev/null 2>&1; then
echo " sudo pacman -S $pkg"
elif command -v zypper >/dev/null 2>&1; then
echo " sudo zypper install -y $pkg"
fi
}
for bin in curl jq; do
if ! command -v "$bin" >/dev/null 2>&1; then
echo "Required dependency '$bin' is not installed. Try:" >&2
suggest_package_install "$bin" >&2
exit 1
fi
done
TASKS_JSON="[]"
add_task() {
local schedule_type="$1" name="$2" command="$3" schedule_expression="$4" source="$5" enabled="$6"
TASKS_JSON=$(jq -c \
--arg schedule_type "$schedule_type" \
--arg name "$name" \
--arg command "$command" \
--arg schedule_expression "$schedule_expression" \
--arg source "$source" \
--argjson enabled "$enabled" \
'. + [{
schedule_type: $schedule_type,
name: $name,
command: $command,
schedule_expression: $schedule_expression,
source: $source,
enabled: $enabled
}]' <<<"$TASKS_JSON")
}
add_task_with_next_run() {
local schedule_type="$1" name="$2" command="$3" schedule_expression="$4" source="$5" enabled="$6" next_run_at="$7"
TASKS_JSON=$(jq -c \
--arg schedule_type "$schedule_type" \
--arg name "$name" \
--arg command "$command" \
--arg schedule_expression "$schedule_expression" \
--arg source "$source" \
--argjson enabled "$enabled" \
--arg next_run_at "$next_run_at" \
'. + [{
schedule_type: $schedule_type,
name: $name,
command: $command,
schedule_expression: $schedule_expression,
source: $source,
enabled: $enabled,
next_run_at: $next_run_at
}]' <<<"$TASKS_JSON")
}
parse_crontab_lines() {
# Reads 5-field-schedule + command cron lines from stdin.
# $1 = source label, $2 = "system" (7-field, includes a user column) or "user" (6-field)
local source="$1" mode="$2"
while IFS= read -r line; do
line="${line%%$'\r'}"
[[ -z "$line" ]] && continue
[[ "$line" =~ ^[[:space:]]*# ]] && continue
[[ "$line" =~ ^[[:space:]]*[A-Za-z_][A-Za-z0-9_]*= ]] && continue
if [[ "$mode" == "system" ]]; then
# min hour dom mon dow user command...
if [[ "$line" =~ ^[[:space:]]*([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+[^[:space:]]+[[:space:]]+(.+)$ ]]; then
local sched="${BASH_REMATCH[1]} ${BASH_REMATCH[2]} ${BASH_REMATCH[3]} ${BASH_REMATCH[4]} ${BASH_REMATCH[5]}"
local cmd="${BASH_REMATCH[6]}"
add_task "cron" "$cmd" "$cmd" "$sched" "$source" true
fi
else
# min hour dom mon dow command...
if [[ "$line" =~ ^[[:space:]]*([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+(.+)$ ]]; then
local sched="${BASH_REMATCH[1]} ${BASH_REMATCH[2]} ${BASH_REMATCH[3]} ${BASH_REMATCH[4]} ${BASH_REMATCH[5]}"
local cmd="${BASH_REMATCH[6]}"
add_task "cron" "$cmd" "$cmd" "$sched" "$source" true
fi
fi
done
}
collect_cron() {
[[ -r /etc/crontab ]] && parse_crontab_lines "/etc/crontab" "system" < /etc/crontab
if [[ -d /etc/cron.d ]]; then
for f in /etc/cron.d/*; do
[[ -f "$f" && -r "$f" ]] || continue
parse_crontab_lines "$f" "system" < "$f"
done
fi
if command -v crontab >/dev/null 2>&1 && [[ -r /etc/passwd ]]; then
while IFS=: read -r user _ uid _ _ _ shell; do
case "$shell" in
*/nologin|*/false|"") continue ;;
esac
[[ "$uid" -lt 1000 && "$uid" != "0" ]] && continue
local_crontab=$(crontab -l -u "$user" 2>/dev/null || true)
[[ -z "$local_crontab" ]] && continue
parse_crontab_lines "crontab:$user" "user" <<<"$local_crontab"
done < /etc/passwd
fi
}
collect_systemd_timers() {
command -v systemctl >/dev/null 2>&1 || return 0
local timers_json
timers_json=$(systemctl list-timers --all --output=json 2>/dev/null || echo "[]")
while IFS= read -r entry; do
local unit activates next_usec enabled_state schedule_expr next_run_at
unit=$(jq -r '.unit' <<<"$entry")
activates=$(jq -r '.activates // ""' <<<"$entry")
next_usec=$(jq -r '.next // 0' <<<"$entry")
enabled_state=$(systemctl is-enabled "$unit" 2>/dev/null || echo "unknown")
local enabled_bool="false"
[[ "$enabled_state" == "enabled" || "$enabled_state" == "static" ]] && enabled_bool="true"
schedule_expr=$(systemctl show "$unit" -p TimersCalendar --value 2>/dev/null | sed -n 's/.*OnCalendar=\([^;]*\);.*/\1/p' | sed 's/[[:space:]]*$//')
[[ -z "$schedule_expr" ]] && schedule_expr="(see: systemctl status $unit)"
next_run_at=""
if [[ "$next_usec" =~ ^[0-9]+$ && "$next_usec" -gt 0 ]]; then
next_run_at=$(date -u -d "@$((next_usec / 1000000))" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || echo "")
fi
if [[ -n "$next_run_at" ]]; then
add_task_with_next_run "systemd_timer" "$unit" "$activates" "$schedule_expr" "$unit" "$enabled_bool" "$next_run_at"
else
add_task "systemd_timer" "$unit" "$activates" "$schedule_expr" "$unit" "$enabled_bool"
fi
done < <(jq -c '.[]' <<<"$timers_json")
}
# Updates waiting to be installed, as JSON in UPDATES_JSON ("null" when this host's package manager isn't one we know, or couldn't answer).
# Nothing is installed or refreshed: this reads what the package manager already knows about, so it is quick and changes nothing.
collect_updates() {
UPDATES_JSON="null"
local manager="" count="" security="null" reboot="false" out="" status=0
if command -v apt-get >/dev/null 2>&1; then
manager="apt"
# -s simulates: "Inst" lines are what an upgrade would install. Security ones come from a "-security" or "Security" archive.
if out=$(LC_ALL=C timeout 60 apt-get -s -o Debug::NoLocking=true upgrade 2>"$PM_ERR"); then
count=$(grep -c '^Inst ' <<<"$out" || true)
security=$(grep '^Inst ' <<<"$out" | grep -ci 'security' || true)
fi
[[ -f /var/run/reboot-required ]] && reboot="true"
elif command -v dnf >/dev/null 2>&1; then
manager="dnf"
# check-update exits 100 when there are updates, 0 when none, 1 on an error. --cacheonly: never goes to the network from here.
out=$(LC_ALL=C timeout 120 dnf -q check-update --cacheonly 2>"$PM_ERR") || status=$?
if [[ "$status" -eq 0 || "$status" -eq 100 ]]; then
count=$(awk 'NF >= 3 && $1 ~ /^[A-Za-z0-9_.+-]+\.[A-Za-z0-9_]+$/ && /Obsoleting/ == 0 { n++ } /^Obsoleting/ { exit } END { print n + 0 }' <<<"$out")
security=$(LC_ALL=C timeout 120 dnf -q updateinfo list --security --cacheonly 2>"$PM_ERR" | grep -c '/Sec\.' || true)
fi
if command -v needs-restarting >/dev/null 2>&1; then
needs-restarting -r >/dev/null 2>&1 || reboot="true"
fi
elif command -v pacman >/dev/null 2>&1; then
manager="pacman"
# -Qu is what the local copy of the sync databases says could be upgraded (what `checkupdates` lists, without changing anything). It
# exits 1 when there is nothing to upgrade, so an empty answer with status 1 is "none", not an error. Packages held back are left out.
out=$(LC_ALL=C timeout 60 pacman -Qu 2>"$PM_ERR") || status=$?
if [[ "$status" -eq 0 || ( "$status" -eq 1 && -z "$out" ) ]]; then
count=$(grep -v '\[ignored\]' <<<"$out" | grep -c . || true)
fi
# No security figure (pacman doesn't say). A restart is due when the kernel running is no longer the one installed.
[[ -d "/usr/lib/modules/$(uname -r)" ]] || reboot="true"
elif command -v zypper >/dev/null 2>&1; then
manager="zypper"
if out=$(LC_ALL=C timeout 120 zypper -q --no-refresh list-updates 2>"$PM_ERR"); then
count=$(grep -c '^v |' <<<"$out" || true)
fi
[[ -f /run/reboot-needed ]] && reboot="true"
fi
if [[ -z "$manager" ]]; then
diag updates unavailable no_package_manager
elif [[ ! "$count" =~ ^[0-9]+$ ]]; then
# In the journal (systemctl status / journalctl -u homelab-manager-agent) so "not reported" has a reason to look for.
echo "Note: couldn't read the pending updates from $manager - the report goes without them." >&2
diag updates failed error "$manager did not answer"
fi
if [[ -n "$manager" && "$count" =~ ^[0-9]+$ ]]; then
[[ "$security" =~ ^[0-9]+$ ]] || security="null"
UPDATES_JSON=$(jq -n --arg manager "$manager" --argjson count "$count" --argjson security "$security" --argjson reboot "$reboot" \
'{manager: $manager, count: $count, security: $security, reboot_required: $reboot}')
fi
}
# >>> post_report
# Sends $PAYLOAD to the server and prints the HTTP status code.
#
# Nothing sensitive goes on curl's command line: any user on this machine can read other processes' command lines, and the
# token would be there for the length of every report. So the token is handed to curl on its standard input, as a line of
# curl's own config format, and the payload and the server's reply go through files in a private directory (mktemp -d is
# readable by root only). A fixed name in /tmp is no good either — another user could plant a symlink there and have this
# script, running as root, overwrite whatever it points at.
curl_quote() {
local s=$1
s=${s//\\/\\\\}
s=${s//\"/\\\"}
printf '%s' "$s"
}
post_report() {
case "$API_TOKEN" in
*$'\n'*|*$'\r'*) echo "API_TOKEN must not contain a line break." >&2; return 1 ;;
esac
printf '%s' "$PAYLOAD" > "$HLM_TMPDIR/payload.json"
printf 'header = "Authorization: Bearer %s"\n' "$(curl_quote "$API_TOKEN")" | curl -sS "${CURL_INSECURE_FLAG[@]}" -K - \
-o "$HLM_TMPDIR/response.json" -w "%{http_code}" \
-X POST "$API_URL/api/agent/report" \
-H "Content-Type: application/json" \
--data-binary @"$HLM_TMPDIR/payload.json"
}
# <<< post_report
# A GET with the token handed to curl on its standard input (as the upload does), the reply into the file named; prints the HTTP status.
api_get() {
printf 'header = "Authorization: Bearer %s"\n' "$(curl_quote "$API_TOKEN")" | curl -sS "${CURL_INSECURE_FLAG[@]}" -K - -m 30 -o "$2" -w "%{http_code}" "$API_URL$1"
}
# One value from the firmware's table (the kernel shows it under /sys/class/dmi/id), trying the named files in turn and leaving out what a
# maker writes when it has nothing to say ("To Be Filled By O.E.M.", "Default string", all zeros ...). DMI_DIR is for the tests.
dmi_value() {
local name value
for name in "$@"; do
# (the file is checked first: a failing redirection prints its complaint before any 2>/dev/null applies)
[[ -r "${DMI_DIR:-/sys/class/dmi/id}/$name" ]] || continue
value=$(tr -d '\0' < "${DMI_DIR:-/sys/class/dmi/id}/$name" 2>/dev/null | head -c 200 | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') || value=""
[[ -n "$value" ]] || continue
case "${value,,}" in
"to be filled by o.e.m."|"default string"|"system serial number"|"system product name"|"system manufacturer"|"not specified"|"not applicable"|"none"|"n/a"|"unknown"|"o.e.m."|"oem") continue ;;
esac
[[ "$value" =~ ^0+$ ]] && continue
printf '%s' "$value"
return 0
done
return 0
}
# What the machine is: maker, model and serial number (a serial is only readable by root). "null" when the firmware has nothing to say, as on
# many virtual machines and single-board computers.
collect_hardware() {
HARDWARE_JSON="null"
local maker model serial
maker=$(dmi_value sys_vendor board_vendor)
model=$(dmi_value product_name board_name)
serial=$(dmi_value product_serial board_serial)
if [[ -z "$maker$model$serial" ]]; then
diag hardware unavailable not_found
return 0
fi
HARDWARE_JSON=$(jq -n -c --arg maker "$maker" --arg model "$model" --arg serial "$serial" \
'{} | (if $maker != "" then . + {manufacturer: $maker} else . end) | (if $model != "" then . + {model: $model} else . end) | (if $serial != "" then . + {serial: $serial} else . end)') || HARDWARE_JSON="null"
}
# Services (systemd units) that have failed. systemctl is asked the way the timers are, so this works inside the sandbox.
collect_failed_units() {
FAILED_UNITS_JSON="null"
if ! command -v systemctl >/dev/null 2>&1; then
diag failed_units unavailable no_systemd
return 0
fi
local out
if ! out=$(timeout 20 systemctl list-units --state=failed --plain --no-legend --no-pager --full 2>"$PM_ERR"); then
diag failed_units failed error "systemctl list-units did not answer"
return 0
fi
FAILED_UNITS_JSON=$(awk 'NF > 0 { unit = $1; $1 = $2 = $3 = $4 = ""; sub(/^ +/, ""); print unit "\t" $0 }' <<<"$out" \
| jq -R -s -c 'split("\n") | map(select(length > 0) | split("\t")) | map({unit: .[0], description: (.[1] // "")}) | .[0:100]') || FAILED_UNITS_JSON="null"
}
# Each disk's own SMART verdict, temperature and (where it says) how much of its rated life is used. Needs the devices, so it runs from
# --prepare. Asked at most about once an hour, and with -n standby so a sleeping disk is not woken: what was last known of it is kept.
collect_smart() {
SMART_JSON="null"
if ! command -v smartctl >/dev/null 2>&1; then
diag smart unavailable tool_missing smartctl
return 0
fi
if [[ "$(id -u)" != "0" ]]; then
diag smart unavailable not_root
return 0
fi
local cache="$STATE_DIR/smart.json" now_s previous="[]" items="[]" dev out item scan checked
now_s=$(date +%s)
if [[ -r "$cache" ]]; then
checked=$(jq -r '.checked_at // 0' "$cache" 2>/dev/null || echo 0)
previous=$(jq -c '.devices // []' "$cache" 2>/dev/null || echo "[]")
if [[ "$checked" =~ ^[0-9]+$ ]] && (( now_s - checked < 3300 )) && [[ "$MODE" != "check" ]]; then
SMART_JSON=$previous
return 0
fi
fi
scan=$(timeout 30 smartctl --scan -j 2>"$PM_ERR") || true
while IFS= read -r dev; do
[[ -n "$dev" ]] || continue
# smartctl's exit status is a bit mask that says "something was noted" more often than "it failed", so it is read from the JSON instead.
out=$(timeout 30 smartctl -j -H -A -n standby "$dev" 2>"$PM_ERR") || true
item=$(jq -c --arg dev "$dev" 'select(.smart_status != null) | {
device: $dev,
model: (.model_name // .scsi_model_name // ""),
healthy: .smart_status.passed,
temperature_c: (.temperature.current // null),
wear_percent: (.nvme_smart_health_information_log.percentage_used // ([.ata_smart_attributes.table[]? | select(.id == 177 or .id == 231 or .id == 233) | (100 - .value)] | first) // null)
}' <<<"$out" 2>/dev/null) || item=""
if [[ -z "$item" ]]; then
# Asleep, or no SMART at all (a virtual disk): keep what was known of it, if anything.
item=$(jq -c --arg dev "$dev" '.[] | select(.device == $dev)' <<<"$previous" 2>/dev/null | head -n 1) || item=""
fi
if [[ -n "$item" ]]; then items=$(jq -c --argjson i "$item" '. + [$i]' <<<"$items"); fi
done < <(jq -r '.devices[]?.name' <<<"$scan" 2>/dev/null)
SMART_JSON=$items
if mkdir -p "$STATE_DIR" 2>/dev/null; then
chmod 700 "$STATE_DIR" 2>/dev/null || true
jq -c -n --argjson at "$now_s" --argjson devices "$items" '{checked_at: $at, devices: $devices}' > "$cache.tmp" 2>/dev/null && mv -f "$cache.tmp" "$cache" 2>/dev/null || true
fi
}
# Each ZFS pool and whether it is ONLINE. Needs /dev/zfs, so it runs from --prepare too.
collect_zfs() {
ZFS_JSON="null"
if ! command -v zpool >/dev/null 2>&1; then
diag zfs unavailable tool_missing zpool
return 0
fi
local out
if ! out=$(timeout 20 zpool list -H -o name,health,capacity 2>"$PM_ERR"); then
diag zfs failed error "zpool list did not answer"
return 0
fi
ZFS_JSON=$(jq -R -s -c 'split("\n") | map(select(length > 0) | split("\t")) | map({name: .[0], health: (.[1] // "UNKNOWN"), capacity_percent: ((.[2] // "" | rtrimstr("%") | tonumber?) // null)})' <<<"$out") || ZFS_JSON="null"
}
# The disk and pool readings --prepare left for this report, if they are there and recent. Re-shaped by jq, never trusted as they stand.
use_prepared_health() {
[[ -r "$HEALTH_FILE" ]] || return 1
[[ -n "$(find "$HEALTH_FILE" -mmin -10 -print 2>/dev/null)" ]] || return 1
SMART_JSON=$(jq -c 'if (.smart | type) == "array" then [.smart[] | {device: (.device | tostring), model: ((.model // "") | tostring),
healthy: (if (.healthy | type) == "boolean" then .healthy else null end),
temperature_c: (if (.temperature_c | type) == "number" then .temperature_c else null end),
wear_percent: (if (.wear_percent | type) == "number" then .wear_percent else null end)}] else null end' "$HEALTH_FILE" 2>/dev/null) || SMART_JSON="null"
ZFS_JSON=$(jq -c 'if (.zfs | type) == "array" then [.zfs[] | {name: (.name | tostring), health: ((.health // "UNKNOWN") | tostring),
capacity_percent: (if (.capacity_percent | type) == "number" then .capacity_percent else null end)}] else null end' "$HEALTH_FILE" 2>/dev/null) || ZFS_JSON="null"
return 0
}
# Replaces this script with the one this Homelab Manager serves, when an administrator has switched automatic updates on and this agent is
# behind. Nothing is trusted but the checksum: the (authenticated) answer gives the SHA-256 of the script, and a download that doesn't
# match it, doesn't look like this script, doesn't pass bash's syntax check or isn't newer is thrown away. The new file is put beside the
# old and moved over it in one step, and the new version then carries on from here. Runs only from --prepare (outside the sandbox, which
# keeps /usr read-only for the report itself); with --check-updates it says what it would do.
self_update() {
[[ -n "$API_URL" && -n "$API_TOKEN" ]] || return 0
[[ -z "${HLM_JUST_UPDATED:-}" ]] || return 0
local self work code reply_update version sha got
self=$(readlink -f "${BASH_SOURCE[0]}" 2>/dev/null) || return 0
[[ -f "$self" ]] || return 0
work=$(mktemp -d) || return 0
code=$(api_get "/api/agent/report/update?version=$AGENT_VERSION" "$work/reply.json" 2>/dev/null) || code=""
reply_update=$(jq -r '.update // false' "$work/reply.json" 2>/dev/null || echo false)
if [[ "$code" != "200" || "$reply_update" != "true" ]]; then
[[ "$MODE" == "check" ]] && echo "Automatic update: nothing to do (it is off, or this agent is current)."
rm -rf "$work"
return 0
fi
version=$(jq -r '.version // empty' "$work/reply.json" 2>/dev/null || true)
sha=$(jq -r '.sha256 // empty' "$work/reply.json" 2>/dev/null || true)
if [[ ! "$version" =~ ^[0-9]+$ || ! "$sha" =~ ^[0-9a-f]{64}$ ]] || (( version <= AGENT_VERSION )); then
rm -rf "$work"
return 0
fi
if ! curl -fsS "${CURL_INSECURE_FLAG[@]}" -m 60 -o "$work/new.sh" "$API_URL/agent/linux/report-tasks.sh" 2>"$PM_ERR"; then
echo "Note: couldn't download the newer agent (version $version), so it was not installed." >&2
rm -rf "$work"
return 0
fi
got=$(sha256sum "$work/new.sh" 2>/dev/null | awk '{print $1}')
if [[ "$got" != "$sha" ]] || [[ "$(head -n 1 "$work/new.sh")" != "#!/usr/bin/env bash" ]] || ! bash -n "$work/new.sh" 2>/dev/null || ! grep -q "^AGENT_VERSION=$version\$" "$work/new.sh"; then
echo "Note: the newer agent (version $version) didn't check out against what the server said, so it was not installed." >&2
rm -rf "$work"
return 0
fi
if [[ "$MODE" == "check" ]]; then
echo "Automatic update: version $version is available and would be installed on the next run."
rm -rf "$work"
return 0
fi
if install -m 755 "$work/new.sh" "$self.new" && mv -f "$self.new" "$self"; then
echo "Updated the agent from version $AGENT_VERSION to version $version." >&2
rm -rf "$work"
HLM_JUST_UPDATED=1 exec "$self" --prepare
fi
rm -f "$self.new"
echo "Note: couldn't replace the agent with version $version." >&2
rm -rf "$work"
return 0
}
# The answer --prepare left for this report, if it is there, recent and looks like what it should. Read back through jq rather than
# trusted: the report only ever carries these four fields, whatever the file holds.
use_prepared_updates() {
[[ -r "$UPDATES_FILE" ]] || return 1
[[ -n "$(find "$UPDATES_FILE" -mmin -10 -print 2>/dev/null)" ]] || return 1
local json
json=$(jq -ce 'select(type == "object" and (.manager | type) == "string" and (.count | type) == "number")
| {manager: .manager, count: .count, security: (if (.security | type) == "number" then .security else null end), reboot_required: (.reboot_required == true)}' "$UPDATES_FILE" 2>/dev/null) || return 1
[[ -n "$json" ]] || return 1
UPDATES_JSON=$json
}
# What this report could not collect: the notes made here, then those --prepare left (it does the outside-the-sandbox part), one for each check.
# With nothing left by --prepare — on a real run, not a dry one — that step did not run, which is worth saying: disk health and ZFS are only read there.
collect_diagnostics() {
local from_prepare="[]" shaped
if [[ -r "$DIAG_FILE" && -n "$(find "$DIAG_FILE" -mmin -10 -print 2>/dev/null)" ]]; then
from_prepare=$(jq -c 'if type == "array" then [.[] | select(type == "object" and (.check | type) == "string" and (.status | type) == "string")
| {check: (.check | tostring), status: (.status | tostring)}
+ (if ((.reason // "") | tostring) != "" then {reason: (.reason | tostring)} else {} end)
+ (if ((.detail // "") | tostring) != "" then {detail: (.detail | tostring | .[0:160])} else {} end)] | .[0:20] else [] end' "$DIAG_FILE" 2>/dev/null) || from_prepare="[]"
elif [[ "$DRY_RUN" != "1" ]]; then
diag prepare unavailable step_missing
fi
shaped=$(jq -c -n --argjson here "$(diag_json)" --argjson there "$from_prepare" '($here + $there) | unique_by(.check) | .[0:20]') || shaped="[]"
DIAGNOSTICS_JSON=$shaped
}
collect_system_info() {
local ip_json="[]"
if command -v ip >/dev/null 2>&1; then
ip_json=$(ip -4 -o addr show scope global 2>/dev/null \
| awk '{print $4}' | cut -d/ -f1 \
| jq -R -s -c 'split("\n") | map(select(length > 0))')
fi
local cpu_model="" cpu_cores=0 cpu_load_percent="null"
# x86 /proc/cpuinfo has a per-core "model name" line. Most 64-bit ARM
# kernels (Raspberry Pi included) don't — they instead have a single
# "Model" line at the very end, or nothing at all, in which case
# /proc/device-tree/model (present on any device-tree/SBC board) has the
# board's friendly name. Try each in turn; leave blank if none exist.
cpu_model=$(grep -m1 "^model name" /proc/cpuinfo 2>/dev/null | cut -d: -f2- | sed 's/^ *//' || true)
if [[ -z "$cpu_model" ]]; then
cpu_model=$(grep -m1 "^Model" /proc/cpuinfo 2>/dev/null | cut -d: -f2- | sed 's/^ *//' || true)
fi
if [[ -z "$cpu_model" && -r /proc/device-tree/model ]]; then
cpu_model=$(tr -d '\0' < /proc/device-tree/model 2>/dev/null || true)
fi
cpu_cores=$(nproc 2>/dev/null || echo 0)
if [[ -r /proc/loadavg && "$cpu_cores" -gt 0 ]]; then
local load1
load1=$(awk '{print $1}' /proc/loadavg)
cpu_load_percent=$(awk -v l="$load1" -v c="$cpu_cores" 'BEGIN { printf "%.1f", (l/c)*100 }')
fi
local mem_total=0 mem_used=0
if command -v free >/dev/null 2>&1; then
read -r mem_total mem_used < <(free -b | awk '/^Mem:/ {print $2, $3}')
fi
local disks_json="[]"
if command -v df >/dev/null 2>&1; then
disks_json=$(df -B1 --output=target,size,used -x tmpfs -x devtmpfs -x squashfs -x overlay 2>/dev/null \
| tail -n +2 \
| awk '{print $1"\t"$2"\t"$3}' \
| jq -R -s -c '
split("\n") | map(select(length > 0) | split("\t")) |
map({mount: .[0], size_bytes: (.[1]|tonumber), used_bytes: (.[2]|tonumber)})
')
fi
# Everything bound to a port on this host, including services listening on localhost only (which a network
# scan from elsewhere can't see). `ss -p` needs root to name the process; without it the process is blank.
# One row per socket — the server groups them per port.
local ports_json="[]"
if command -v ss >/dev/null 2>&1; then
ports_json=$(ss -H -tulnp 2>/dev/null \
| awk '{
local = $5; port = local; sub(/.*:/, "", port); addr = local; sub(/:[0-9]+$/, "", addr);
proc = ""; if (match($0, /users:\(\("[^"]+"/)) { proc = substr($0, RSTART + 9, RLENGTH - 10) }
if (port ~ /^[0-9]+$/) print $1 "\t" port "\t" addr "\t" proc
}' \
| jq -R -s -c '
split("\n") | map(select(length > 0) | split("\t")) |
map({protocol: .[0], port: (.[1]|tonumber), address: .[2], process: (.[3] // "")})
')
fi
# Best-effort like the rest: a package manager that misbehaves must not cost the report.
UPDATES_JSON="null"
use_prepared_updates || collect_updates || UPDATES_JSON="null"
# Failed services are asked here (systemctl works inside the sandbox); disk and pool health come from --prepare. A person running --dry-run
# isn't in the sandbox, so there they are read directly.
FAILED_UNITS_JSON="null"
collect_failed_units || FAILED_UNITS_JSON="null"
SMART_JSON="null"
ZFS_JSON="null"
if ! use_prepared_health && [[ "$DRY_RUN" == "1" ]]; then
collect_smart || SMART_JSON="null"
collect_zfs || ZFS_JSON="null"
fi
HARDWARE_JSON="null"
collect_hardware || HARDWARE_JSON="null"
collect_diagnostics
HEALTH_JSON=$(jq -n -c --argjson failed "$FAILED_UNITS_JSON" --argjson smart "$SMART_JSON" --argjson zfs "$ZFS_JSON" \
'{} | (if $failed != null then . + {failed_units: $failed} else . end) | (if $smart != null then . + {smart: $smart} else . end) | (if $zfs != null then . + {zfs: $zfs} else . end)') || HEALTH_JSON="{}"
[[ "$HEALTH_JSON" == "{}" ]] && HEALTH_JSON="null"
SYSTEM_JSON=$(jq -n \
--argjson updates "$UPDATES_JSON" \
--argjson health "$HEALTH_JSON" \
--argjson hardware "$HARDWARE_JSON" \
--argjson diagnostics "$DIAGNOSTICS_JSON" \
--argjson ip_addresses "$ip_json" \
--argjson listening_ports "$ports_json" \
--arg cpu_model "$cpu_model" \
--argjson cpu_cores "${cpu_cores:-0}" \
--argjson cpu_load_percent "$cpu_load_percent" \
--argjson mem_total_bytes "${mem_total:-0}" \
--argjson mem_used_bytes "${mem_used:-0}" \
--argjson disks "$disks_json" \
'{
ip_addresses: $ip_addresses,
cpu: { model: $cpu_model, cores: $cpu_cores, load_percent: $cpu_load_percent },
memory: { total_bytes: $mem_total_bytes, used_bytes: $mem_used_bytes },
disks: $disks,
listening_ports: $listening_ports,
updates: $updates,
health: $health,
hardware: $hardware,
diagnostics: $diagnostics
}')
}
if [[ "$MODE" != "report" ]]; then
# prepare: first see whether this script is to be replaced (if so it carries on as the new one, from the top).
[[ "$MODE" == "prepare" ]] && self_update
collect_updates || UPDATES_JSON="null"
collect_smart || SMART_JSON="null"
collect_zfs || ZFS_JSON="null"
if [[ "$MODE" == "check" ]]; then
collect_failed_units || FAILED_UNITS_JSON="null"
echo "Pending updates as they would be reported:"
jq . <<<"$UPDATES_JSON"
echo "Failed services:"
jq . <<<"$FAILED_UNITS_JSON"
echo "Disk health (SMART; null means smartctl isn't installed or this isn't root):"
jq . <<<"$SMART_JSON"
echo "ZFS pools (null means there is no zpool here):"
jq . <<<"$ZFS_JSON"
collect_hardware || true
echo "What could not be collected (empty means nothing was missed):"
diag_json | jq .
self_update
exit 0
fi
# Leave the answers (or no file at all) for the report that follows. Each is written whole and moved into place, so a report never reads half of one.
rm -f "$UPDATES_FILE" "$HEALTH_FILE" "$DIAG_FILE"
mkdir -p "$(dirname "$UPDATES_FILE")" 2>/dev/null || exit 0
# The notes always go, even when there are none: a missing file is how the report knows this step didn't run.
diag_json > "$DIAG_FILE.tmp" 2>/dev/null && mv -f "$DIAG_FILE.tmp" "$DIAG_FILE" 2>/dev/null || true
if [[ "$UPDATES_JSON" != "null" ]]; then
printf '%s\n' "$UPDATES_JSON" > "$UPDATES_FILE.tmp" 2>/dev/null && mv -f "$UPDATES_FILE.tmp" "$UPDATES_FILE" 2>/dev/null || true
fi
HEALTH_FILE_JSON=$(jq -n -c --argjson smart "${SMART_JSON:-null}" --argjson zfs "${ZFS_JSON:-null}" '{} | (if $smart != null then . + {smart: $smart} else . end) | (if $zfs != null then . + {zfs: $zfs} else . end)' 2>/dev/null || echo "{}")
if [[ "$HEALTH_FILE_JSON" != "{}" ]]; then
printf '%s\n' "$HEALTH_FILE_JSON" > "$HEALTH_FILE.tmp" 2>/dev/null && mv -f "$HEALTH_FILE.tmp" "$HEALTH_FILE" 2>/dev/null || true
fi
exit 0
fi
collect_cron
collect_systemd_timers
# Hardware/network facts are best-effort: a quirk on any given host (e.g. no
# "model name" line in /proc/cpuinfo on some ARM boards) must never abort the
# whole report over it, so errexit is relaxed just for this call.
SYSTEM_JSON="null"
set +e
collect_system_info
system_info_status=$?
set -e
if [[ "$system_info_status" -ne 0 ]]; then
echo "Warning: collecting hardware/network info failed (exit $system_info_status) — reporting tasks without it." >&2
SYSTEM_JSON="null"
fi
HOSTNAME_VALUE=$(hostname -f 2>/dev/null || hostname)
PAYLOAD=$(jq -n \
--arg hostname "$HOSTNAME_VALUE" \
--arg os_type "linux" \
--arg reported_at "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
--argjson agent_version "$AGENT_VERSION" \
--argjson system "$SYSTEM_JSON" \
--argjson tasks "$TASKS_JSON" \
'{hostname: $hostname, os_type: $os_type, reported_at: $reported_at, agent_version: $agent_version, system: $system, tasks: $tasks}')
if [[ "$DRY_RUN" == "1" ]]; then
echo "$PAYLOAD" | jq .
exit 0
fi
HLM_TMPDIR=$(mktemp -d)
trap 'rm -rf "$HLM_TMPDIR"' EXIT
response=$(post_report)
if [[ "$response" -lt 200 || "$response" -ge 300 ]]; then
echo "Report failed with HTTP $response:" >&2
cat "$HLM_TMPDIR/response.json" >&2
exit 1
fi
echo "Reported $(jq 'length' <<<"$TASKS_JSON") task(s) successfully."