Linux agent 8 and Windows agent 7 say how long the machine has been up; the server stores it (migration 0086) and shows it as Up for in the status strip (left out while offline) and Uptime on the server page. The Integrations header wraps and its button no longer squeezes, and .table-responsive is positioned so hidden text in scrolled-off columns cannot widen the page. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
758 lines
36 KiB
Bash
758 lines
36 KiB
Bash
#!/usr/bin/env bash
|
|
# Collects cron jobs and systemd timers on this host and POSTs them to the
|
|
# Homelab Manager API. Intended to run as root via a periodic systemd timer
|
|
# (see install.sh) but can be run manually for testing:
|
|
#
|
|
# API_URL=https://homelab.example.lan API_TOKEN=hlm_xxx ./report-tasks.sh --dry-run
|
|
#
|
|
# Set API_INSECURE=true (also written to the env file by install.sh when
|
|
# passed there) if Homelab Manager uses a self-signed certificate.
|
|
#
|
|
set -euo pipefail
|
|
|
|
# Which version of this script this is, sent with every report so Homelab Manager can tell which agents are behind. Raise it whenever a
|
|
# change here is worth installing the agent again for (and add it to the list at the top of server/src/services/agentVersions.ts).
|
|
# 1 = before the number was reported 2 = reports pending updates, and its own version
|
|
# 3 = the package manager is asked outside the sandbox (see --prepare), so dnf and zypper can answer there too
|
|
# 4 = also reports failed services, disk health (SMART) and ZFS pools, reads pacman, and can update itself when told to
|
|
# 5 = also says what the machine is (maker, model, serial number), from the firmware, for the inventory
|
|
# 6 = also says what it could not collect and why (no smartctl, no systemd, a package manager that failed ...), so the server can say what to fix
|
|
AGENT_VERSION=8
|
|
|
|
ENV_FILE="${ENV_FILE:-/etc/homelab-manager-agent.env}"
|
|
if [[ -f "$ENV_FILE" ]]; then
|
|
# shellcheck disable=SC1090
|
|
source "$ENV_FILE"
|
|
fi
|
|
|
|
API_URL="${API_URL:-}"
|
|
API_TOKEN="${API_TOKEN:-}"
|
|
API_INSECURE="${API_INSECURE:-false}"
|
|
DRY_RUN=0
|
|
[[ "${1:-}" == "--dry-run" ]] && DRY_RUN=1
|
|
|
|
# Two extra ways to run it, neither of which reports anything:
|
|
# --prepare the things that can't be done inside the systemd sandbox: asks the package manager what is waiting (dnf and zypper
|
|
# write to /var and /run even when only asked), reads disk health with smartctl and pool state with zpool (both need the
|
|
# devices, which the sandbox closes), and — if an administrator has switched it on — replaces this script with a newer
|
|
# one. The answers are left in the unit's private directory for the report to pick up. The unit runs this step outside
|
|
# its sandbox (ExecStartPre=-+); a failure here only means those figures are missing. --prepare-updates is the same.
|
|
# --check-updates for a person at a terminal: runs the same checks and shows what each said, and whether it would update itself.
|
|
MODE=report
|
|
case "${1:-}" in
|
|
--prepare|--prepare-updates) MODE=prepare ;;
|
|
--check-updates|--check) MODE=check ;;
|
|
esac
|
|
UPDATES_FILE="${UPDATES_FILE:-/run/homelab-manager-agent/updates.json}"
|
|
HEALTH_FILE="${HEALTH_FILE:-$(dirname "$UPDATES_FILE")/health.json}"
|
|
# What --prepare could not collect and why, left for the report to carry along.
|
|
DIAG_FILE="${DIAG_FILE:-$(dirname "$UPDATES_FILE")/diagnostics.json}"
|
|
# Kept between runs (the unit's private directory is not): a SMART reading is only taken about once an hour, so sleeping disks are left alone.
|
|
STATE_DIR="${STATE_DIR:-/var/lib/homelab-manager-agent}"
|
|
# Where the package managers' own error output goes: nowhere, unless a person asked to see it.
|
|
PM_ERR=/dev/null
|
|
[[ "$MODE" == "check" ]] && PM_ERR=/dev/stderr
|
|
|
|
if [[ "$MODE" == "report" && ( -z "$API_URL" || -z "$API_TOKEN" ) ]]; then
|
|
echo "API_URL and API_TOKEN must be set (env vars or $ENV_FILE)" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Things that could not be collected, each a small JSON object: which check, whether it is "unavailable" (nothing to read it with, or nothing
|
|
# there to read) or "failed" (it tried and could not), why in a word, and at most a line of what the tool said. Only problems are noted: a
|
|
# check that is not mentioned went fine. The server turns these into what to do about them.
|
|
DIAG_ITEMS=()
|
|
diag() {
|
|
local check="$1" status="$2" reason="${3:-}" detail="${4:-}"
|
|
detail=$(printf '%s' "$detail" | tr -d '\000-\037' | head -c 160)
|
|
DIAG_ITEMS+=("$(jq -n -c --arg c "$check" --arg s "$status" --arg r "$reason" --arg d "$detail" \
|
|
'{check: $c, status: $s} + (if $r != "" then {reason: $r} else {} end) + (if $d != "" then {detail: $d} else {} end)')")
|
|
}
|
|
|
|
# The notes made so far as one JSON array ([] when there are none).
|
|
diag_json() {
|
|
if (( ${#DIAG_ITEMS[@]} == 0 )); then echo "[]"; return 0; fi
|
|
printf '%s\n' "${DIAG_ITEMS[@]}" | jq -s -c '.'
|
|
}
|
|
|
|
CURL_INSECURE_FLAG=()
|
|
case "${API_INSECURE,,}" in
|
|
1|true|yes) CURL_INSECURE_FLAG=(-k) ;;
|
|
esac
|
|
|
|
suggest_package_install() {
|
|
local pkg="$1"
|
|
if command -v apt-get >/dev/null 2>&1; then
|
|
echo " sudo apt-get update && sudo apt-get install -y $pkg"
|
|
elif command -v dnf >/dev/null 2>&1; then
|
|
echo " sudo dnf install -y $pkg"
|
|
elif command -v yum >/dev/null 2>&1; then
|
|
echo " sudo yum install -y $pkg"
|
|
elif command -v apk >/dev/null 2>&1; then
|
|
echo " sudo apk add $pkg"
|
|
elif command -v pacman >/dev/null 2>&1; then
|
|
echo " sudo pacman -S $pkg"
|
|
elif command -v zypper >/dev/null 2>&1; then
|
|
echo " sudo zypper install -y $pkg"
|
|
fi
|
|
}
|
|
|
|
for bin in curl jq; do
|
|
if ! command -v "$bin" >/dev/null 2>&1; then
|
|
echo "Required dependency '$bin' is not installed. Try:" >&2
|
|
suggest_package_install "$bin" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
TASKS_JSON="[]"
|
|
|
|
add_task() {
|
|
local schedule_type="$1" name="$2" command="$3" schedule_expression="$4" source="$5" enabled="$6"
|
|
TASKS_JSON=$(jq -c \
|
|
--arg schedule_type "$schedule_type" \
|
|
--arg name "$name" \
|
|
--arg command "$command" \
|
|
--arg schedule_expression "$schedule_expression" \
|
|
--arg source "$source" \
|
|
--argjson enabled "$enabled" \
|
|
'. + [{
|
|
schedule_type: $schedule_type,
|
|
name: $name,
|
|
command: $command,
|
|
schedule_expression: $schedule_expression,
|
|
source: $source,
|
|
enabled: $enabled
|
|
}]' <<<"$TASKS_JSON")
|
|
}
|
|
|
|
add_task_with_next_run() {
|
|
local schedule_type="$1" name="$2" command="$3" schedule_expression="$4" source="$5" enabled="$6" next_run_at="$7"
|
|
TASKS_JSON=$(jq -c \
|
|
--arg schedule_type "$schedule_type" \
|
|
--arg name "$name" \
|
|
--arg command "$command" \
|
|
--arg schedule_expression "$schedule_expression" \
|
|
--arg source "$source" \
|
|
--argjson enabled "$enabled" \
|
|
--arg next_run_at "$next_run_at" \
|
|
'. + [{
|
|
schedule_type: $schedule_type,
|
|
name: $name,
|
|
command: $command,
|
|
schedule_expression: $schedule_expression,
|
|
source: $source,
|
|
enabled: $enabled,
|
|
next_run_at: $next_run_at
|
|
}]' <<<"$TASKS_JSON")
|
|
}
|
|
|
|
parse_crontab_lines() {
|
|
# Reads 5-field-schedule + command cron lines from stdin.
|
|
# $1 = source label, $2 = "system" (7-field, includes a user column) or "user" (6-field)
|
|
local source="$1" mode="$2"
|
|
while IFS= read -r line; do
|
|
line="${line%%$'\r'}"
|
|
[[ -z "$line" ]] && continue
|
|
[[ "$line" =~ ^[[:space:]]*# ]] && continue
|
|
[[ "$line" =~ ^[[:space:]]*[A-Za-z_][A-Za-z0-9_]*= ]] && continue
|
|
|
|
if [[ "$mode" == "system" ]]; then
|
|
# min hour dom mon dow user command...
|
|
if [[ "$line" =~ ^[[:space:]]*([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+[^[:space:]]+[[:space:]]+(.+)$ ]]; then
|
|
local sched="${BASH_REMATCH[1]} ${BASH_REMATCH[2]} ${BASH_REMATCH[3]} ${BASH_REMATCH[4]} ${BASH_REMATCH[5]}"
|
|
local cmd="${BASH_REMATCH[6]}"
|
|
add_task "cron" "$cmd" "$cmd" "$sched" "$source" true
|
|
fi
|
|
else
|
|
# min hour dom mon dow command...
|
|
if [[ "$line" =~ ^[[:space:]]*([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+(.+)$ ]]; then
|
|
local sched="${BASH_REMATCH[1]} ${BASH_REMATCH[2]} ${BASH_REMATCH[3]} ${BASH_REMATCH[4]} ${BASH_REMATCH[5]}"
|
|
local cmd="${BASH_REMATCH[6]}"
|
|
add_task "cron" "$cmd" "$cmd" "$sched" "$source" true
|
|
fi
|
|
fi
|
|
done
|
|
}
|
|
|
|
collect_cron() {
|
|
[[ -r /etc/crontab ]] && parse_crontab_lines "/etc/crontab" "system" < /etc/crontab
|
|
|
|
if [[ -d /etc/cron.d ]]; then
|
|
for f in /etc/cron.d/*; do
|
|
[[ -f "$f" && -r "$f" ]] || continue
|
|
parse_crontab_lines "$f" "system" < "$f"
|
|
done
|
|
fi
|
|
|
|
if command -v crontab >/dev/null 2>&1 && [[ -r /etc/passwd ]]; then
|
|
while IFS=: read -r user _ uid _ _ _ shell; do
|
|
case "$shell" in
|
|
*/nologin|*/false|"") continue ;;
|
|
esac
|
|
[[ "$uid" -lt 1000 && "$uid" != "0" ]] && continue
|
|
local_crontab=$(crontab -l -u "$user" 2>/dev/null || true)
|
|
[[ -z "$local_crontab" ]] && continue
|
|
parse_crontab_lines "crontab:$user" "user" <<<"$local_crontab"
|
|
done < /etc/passwd
|
|
fi
|
|
}
|
|
|
|
collect_systemd_timers() {
|
|
command -v systemctl >/dev/null 2>&1 || return 0
|
|
|
|
local timers_json
|
|
timers_json=$(systemctl list-timers --all --output=json 2>/dev/null || echo "[]")
|
|
|
|
while IFS= read -r entry; do
|
|
local unit activates next_usec enabled_state schedule_expr next_run_at
|
|
unit=$(jq -r '.unit' <<<"$entry")
|
|
activates=$(jq -r '.activates // ""' <<<"$entry")
|
|
next_usec=$(jq -r '.next // 0' <<<"$entry")
|
|
|
|
enabled_state=$(systemctl is-enabled "$unit" 2>/dev/null || echo "unknown")
|
|
local enabled_bool="false"
|
|
[[ "$enabled_state" == "enabled" || "$enabled_state" == "static" ]] && enabled_bool="true"
|
|
|
|
schedule_expr=$(systemctl show "$unit" -p TimersCalendar --value 2>/dev/null | sed -n 's/.*OnCalendar=\([^;]*\);.*/\1/p' | sed 's/[[:space:]]*$//')
|
|
[[ -z "$schedule_expr" ]] && schedule_expr="(see: systemctl status $unit)"
|
|
|
|
next_run_at=""
|
|
if [[ "$next_usec" =~ ^[0-9]+$ && "$next_usec" -gt 0 ]]; then
|
|
next_run_at=$(date -u -d "@$((next_usec / 1000000))" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || echo "")
|
|
fi
|
|
|
|
if [[ -n "$next_run_at" ]]; then
|
|
add_task_with_next_run "systemd_timer" "$unit" "$activates" "$schedule_expr" "$unit" "$enabled_bool" "$next_run_at"
|
|
else
|
|
add_task "systemd_timer" "$unit" "$activates" "$schedule_expr" "$unit" "$enabled_bool"
|
|
fi
|
|
done < <(jq -c '.[]' <<<"$timers_json")
|
|
}
|
|
|
|
# Updates waiting to be installed, as JSON in UPDATES_JSON ("null" when this host's package manager isn't one we know, or couldn't answer).
|
|
# Nothing is installed or refreshed: this reads what the package manager already knows about, so it is quick and changes nothing.
|
|
collect_updates() {
|
|
UPDATES_JSON="null"
|
|
local manager="" count="" security="null" reboot="false" out="" status=0
|
|
|
|
if command -v apt-get >/dev/null 2>&1; then
|
|
manager="apt"
|
|
# -s simulates: "Inst" lines are what an upgrade would install. Security ones come from a "-security" or "Security" archive.
|
|
if out=$(LC_ALL=C timeout 60 apt-get -s -o Debug::NoLocking=true upgrade 2>"$PM_ERR"); then
|
|
count=$(grep -c '^Inst ' <<<"$out" || true)
|
|
security=$(grep '^Inst ' <<<"$out" | grep -ci 'security' || true)
|
|
fi
|
|
[[ -f /var/run/reboot-required ]] && reboot="true"
|
|
elif command -v dnf >/dev/null 2>&1; then
|
|
manager="dnf"
|
|
# check-update exits 100 when there are updates, 0 when none, 1 on an error. --cacheonly: never goes to the network from here.
|
|
out=$(LC_ALL=C timeout 120 dnf -q check-update --cacheonly 2>"$PM_ERR") || status=$?
|
|
if [[ "$status" -eq 0 || "$status" -eq 100 ]]; then
|
|
count=$(awk 'NF >= 3 && $1 ~ /^[A-Za-z0-9_.+-]+\.[A-Za-z0-9_]+$/ && /Obsoleting/ == 0 { n++ } /^Obsoleting/ { exit } END { print n + 0 }' <<<"$out")
|
|
security=$(LC_ALL=C timeout 120 dnf -q updateinfo list --security --cacheonly 2>"$PM_ERR" | grep -c '/Sec\.' || true)
|
|
fi
|
|
if command -v needs-restarting >/dev/null 2>&1; then
|
|
needs-restarting -r >/dev/null 2>&1 || reboot="true"
|
|
fi
|
|
elif command -v pacman >/dev/null 2>&1; then
|
|
manager="pacman"
|
|
# -Qu is what the local copy of the sync databases says could be upgraded (what `checkupdates` lists, without changing anything). It
|
|
# exits 1 when there is nothing to upgrade, so an empty answer with status 1 is "none", not an error. Packages held back are left out.
|
|
out=$(LC_ALL=C timeout 60 pacman -Qu 2>"$PM_ERR") || status=$?
|
|
if [[ "$status" -eq 0 || ( "$status" -eq 1 && -z "$out" ) ]]; then
|
|
count=$(grep -v '\[ignored\]' <<<"$out" | grep -c . || true)
|
|
fi
|
|
# No security figure (pacman doesn't say). A restart is due when the kernel running is no longer the one installed.
|
|
[[ -d "/usr/lib/modules/$(uname -r)" ]] || reboot="true"
|
|
elif command -v zypper >/dev/null 2>&1; then
|
|
manager="zypper"
|
|
if out=$(LC_ALL=C timeout 120 zypper -q --no-refresh list-updates 2>"$PM_ERR"); then
|
|
count=$(grep -c '^v |' <<<"$out" || true)
|
|
fi
|
|
[[ -f /run/reboot-needed ]] && reboot="true"
|
|
fi
|
|
|
|
if [[ -z "$manager" ]]; then
|
|
diag updates unavailable no_package_manager
|
|
elif [[ ! "$count" =~ ^[0-9]+$ ]]; then
|
|
# In the journal (systemctl status / journalctl -u homelab-manager-agent) so "not reported" has a reason to look for.
|
|
echo "Note: couldn't read the pending updates from $manager - the report goes without them." >&2
|
|
diag updates failed error "$manager did not answer"
|
|
fi
|
|
|
|
if [[ -n "$manager" && "$count" =~ ^[0-9]+$ ]]; then
|
|
[[ "$security" =~ ^[0-9]+$ ]] || security="null"
|
|
UPDATES_JSON=$(jq -n --arg manager "$manager" --argjson count "$count" --argjson security "$security" --argjson reboot "$reboot" \
|
|
'{manager: $manager, count: $count, security: $security, reboot_required: $reboot}')
|
|
fi
|
|
}
|
|
|
|
# >>> post_report
|
|
# Sends $PAYLOAD to the server and prints the HTTP status code.
|
|
#
|
|
# Nothing sensitive goes on curl's command line: any user on this machine can read other processes' command lines, and the
|
|
# token would be there for the length of every report. So the token is handed to curl on its standard input, as a line of
|
|
# curl's own config format, and the payload and the server's reply go through files in a private directory (mktemp -d is
|
|
# readable by root only). A fixed name in /tmp is no good either — another user could plant a symlink there and have this
|
|
# script, running as root, overwrite whatever it points at.
|
|
curl_quote() {
|
|
local s=$1
|
|
s=${s//\\/\\\\}
|
|
s=${s//\"/\\\"}
|
|
printf '%s' "$s"
|
|
}
|
|
|
|
post_report() {
|
|
case "$API_TOKEN" in
|
|
*$'\n'*|*$'\r'*) echo "API_TOKEN must not contain a line break." >&2; return 1 ;;
|
|
esac
|
|
printf '%s' "$PAYLOAD" > "$HLM_TMPDIR/payload.json"
|
|
printf 'header = "Authorization: Bearer %s"\n' "$(curl_quote "$API_TOKEN")" | curl -sS "${CURL_INSECURE_FLAG[@]}" -K - \
|
|
-o "$HLM_TMPDIR/response.json" -w "%{http_code}" \
|
|
-X POST "$API_URL/api/agent/report" \
|
|
-H "Content-Type: application/json" \
|
|
--data-binary @"$HLM_TMPDIR/payload.json"
|
|
}
|
|
# <<< post_report
|
|
|
|
# A GET with the token handed to curl on its standard input (as the upload does), the reply into the file named; prints the HTTP status.
|
|
api_get() {
|
|
printf 'header = "Authorization: Bearer %s"\n' "$(curl_quote "$API_TOKEN")" | curl -sS "${CURL_INSECURE_FLAG[@]}" -K - -m 30 -o "$2" -w "%{http_code}" "$API_URL$1"
|
|
}
|
|
|
|
# One value from the firmware's table (the kernel shows it under /sys/class/dmi/id), trying the named files in turn and leaving out what a
|
|
# maker writes when it has nothing to say ("To Be Filled By O.E.M.", "Default string", all zeros ...). DMI_DIR is for the tests.
|
|
dmi_value() {
|
|
local name value
|
|
for name in "$@"; do
|
|
# (the file is checked first: a failing redirection prints its complaint before any 2>/dev/null applies)
|
|
[[ -r "${DMI_DIR:-/sys/class/dmi/id}/$name" ]] || continue
|
|
value=$(tr -d '\0' < "${DMI_DIR:-/sys/class/dmi/id}/$name" 2>/dev/null | head -c 200 | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') || value=""
|
|
[[ -n "$value" ]] || continue
|
|
case "${value,,}" in
|
|
"to be filled by o.e.m."|"default string"|"system serial number"|"system product name"|"system manufacturer"|"not specified"|"not applicable"|"none"|"n/a"|"unknown"|"o.e.m."|"oem") continue ;;
|
|
esac
|
|
[[ "$value" =~ ^0+$ ]] && continue
|
|
printf '%s' "$value"
|
|
return 0
|
|
done
|
|
return 0
|
|
}
|
|
|
|
# What the machine is: maker, model and serial number (a serial is only readable by root). "null" when the firmware has nothing to say, as on
|
|
# many virtual machines and single-board computers.
|
|
collect_hardware() {
|
|
HARDWARE_JSON="null"
|
|
local maker model serial
|
|
maker=$(dmi_value sys_vendor board_vendor)
|
|
model=$(dmi_value product_name board_name)
|
|
serial=$(dmi_value product_serial board_serial)
|
|
if [[ -z "$maker$model$serial" ]]; then
|
|
diag hardware unavailable not_found
|
|
return 0
|
|
fi
|
|
HARDWARE_JSON=$(jq -n -c --arg maker "$maker" --arg model "$model" --arg serial "$serial" \
|
|
'{} | (if $maker != "" then . + {manufacturer: $maker} else . end) | (if $model != "" then . + {model: $model} else . end) | (if $serial != "" then . + {serial: $serial} else . end)') || HARDWARE_JSON="null"
|
|
}
|
|
|
|
# What the machine runs, as /etc/os-release says (OS_RELEASE_FILE is for the tests): the distribution's id and version, so the app can say when that release stops
|
|
# being supported. Proxmox VE is Debian underneath, so it is asked by its own command. "null" when there is nothing to say.
|
|
collect_os() {
|
|
OS_JSON="null"
|
|
local file="${OS_RELEASE_FILE:-/etc/os-release}" id="" version="" pretty=""
|
|
if [[ -r "$file" ]]; then
|
|
id=$(sed -n 's/^ID=//p' "$file" | head -n 1 | tr -d '"' | tr 'A-Z' 'a-z' | head -c 40)
|
|
version=$(sed -n 's/^VERSION_ID=//p' "$file" | head -n 1 | tr -d '"' | head -c 40)
|
|
pretty=$(sed -n 's/^PRETTY_NAME=//p' "$file" | head -n 1 | tr -d '"' | head -c 80)
|
|
fi
|
|
if command -v pveversion >/dev/null 2>&1; then
|
|
local pve
|
|
pve=$(pveversion 2>/dev/null | sed -n 's#^pve-manager/\([0-9]*\.[0-9]*\).*#\1#p' | head -n 1)
|
|
if [[ -n "$pve" ]]; then
|
|
id="proxmox-ve"
|
|
version="$pve"
|
|
pretty="Proxmox VE $pve"
|
|
fi
|
|
fi
|
|
[[ -n "$id" ]] || return 0
|
|
OS_JSON=$(jq -n -c --arg id "$id" --arg version "$version" --arg name "$pretty" \
|
|
'{id: $id} | (if $version != "" then . + {version: $version} else . end) | (if $name != "" then . + {name: $name} else . end)') || OS_JSON="null"
|
|
}
|
|
|
|
# Services (systemd units) that have failed. systemctl is asked the way the timers are, so this works inside the sandbox.
|
|
collect_failed_units() {
|
|
FAILED_UNITS_JSON="null"
|
|
if ! command -v systemctl >/dev/null 2>&1; then
|
|
diag failed_units unavailable no_systemd
|
|
return 0
|
|
fi
|
|
local out
|
|
if ! out=$(timeout 20 systemctl list-units --state=failed --plain --no-legend --no-pager --full 2>"$PM_ERR"); then
|
|
diag failed_units failed error "systemctl list-units did not answer"
|
|
return 0
|
|
fi
|
|
FAILED_UNITS_JSON=$(awk 'NF > 0 { unit = $1; $1 = $2 = $3 = $4 = ""; sub(/^ +/, ""); print unit "\t" $0 }' <<<"$out" \
|
|
| jq -R -s -c 'split("\n") | map(select(length > 0) | split("\t")) | map({unit: .[0], description: (.[1] // "")}) | .[0:100]') || FAILED_UNITS_JSON="null"
|
|
}
|
|
|
|
# Each disk's own SMART verdict, temperature and (where it says) how much of its rated life is used. Needs the devices, so it runs from
|
|
# --prepare. Asked at most about once an hour, and with -n standby so a sleeping disk is not woken: what was last known of it is kept.
|
|
collect_smart() {
|
|
SMART_JSON="null"
|
|
if ! command -v smartctl >/dev/null 2>&1; then
|
|
diag smart unavailable tool_missing smartctl
|
|
return 0
|
|
fi
|
|
if [[ "$(id -u)" != "0" ]]; then
|
|
diag smart unavailable not_root
|
|
return 0
|
|
fi
|
|
local cache="$STATE_DIR/smart.json" now_s previous="[]" items="[]" dev out item scan checked
|
|
now_s=$(date +%s)
|
|
if [[ -r "$cache" ]]; then
|
|
checked=$(jq -r '.checked_at // 0' "$cache" 2>/dev/null || echo 0)
|
|
previous=$(jq -c '.devices // []' "$cache" 2>/dev/null || echo "[]")
|
|
if [[ "$checked" =~ ^[0-9]+$ ]] && (( now_s - checked < 3300 )) && [[ "$MODE" != "check" ]]; then
|
|
SMART_JSON=$previous
|
|
return 0
|
|
fi
|
|
fi
|
|
scan=$(timeout 30 smartctl --scan -j 2>"$PM_ERR") || true
|
|
while IFS= read -r dev; do
|
|
[[ -n "$dev" ]] || continue
|
|
# smartctl's exit status is a bit mask that says "something was noted" more often than "it failed", so it is read from the JSON instead.
|
|
out=$(timeout 30 smartctl -j -H -A -n standby "$dev" 2>"$PM_ERR") || true
|
|
item=$(jq -c --arg dev "$dev" 'select(.smart_status != null) | {
|
|
device: $dev,
|
|
model: (.model_name // .scsi_model_name // ""),
|
|
healthy: .smart_status.passed,
|
|
temperature_c: (.temperature.current // null),
|
|
wear_percent: (.nvme_smart_health_information_log.percentage_used // ([.ata_smart_attributes.table[]? | select(.id == 177 or .id == 231 or .id == 233) | (100 - .value)] | first) // null)
|
|
}' <<<"$out" 2>/dev/null) || item=""
|
|
if [[ -z "$item" ]]; then
|
|
# Asleep, or no SMART at all (a virtual disk): keep what was known of it, if anything.
|
|
item=$(jq -c --arg dev "$dev" '.[] | select(.device == $dev)' <<<"$previous" 2>/dev/null | head -n 1) || item=""
|
|
fi
|
|
if [[ -n "$item" ]]; then items=$(jq -c --argjson i "$item" '. + [$i]' <<<"$items"); fi
|
|
done < <(jq -r '.devices[]?.name' <<<"$scan" 2>/dev/null)
|
|
SMART_JSON=$items
|
|
if mkdir -p "$STATE_DIR" 2>/dev/null; then
|
|
chmod 700 "$STATE_DIR" 2>/dev/null || true
|
|
jq -c -n --argjson at "$now_s" --argjson devices "$items" '{checked_at: $at, devices: $devices}' > "$cache.tmp" 2>/dev/null && mv -f "$cache.tmp" "$cache" 2>/dev/null || true
|
|
fi
|
|
}
|
|
|
|
# Each ZFS pool and whether it is ONLINE. Needs /dev/zfs, so it runs from --prepare too.
|
|
collect_zfs() {
|
|
ZFS_JSON="null"
|
|
if ! command -v zpool >/dev/null 2>&1; then
|
|
diag zfs unavailable tool_missing zpool
|
|
return 0
|
|
fi
|
|
local out
|
|
if ! out=$(timeout 20 zpool list -H -o name,health,capacity 2>"$PM_ERR"); then
|
|
diag zfs failed error "zpool list did not answer"
|
|
return 0
|
|
fi
|
|
ZFS_JSON=$(jq -R -s -c 'split("\n") | map(select(length > 0) | split("\t")) | map({name: .[0], health: (.[1] // "UNKNOWN"), capacity_percent: ((.[2] // "" | rtrimstr("%") | tonumber?) // null)})' <<<"$out") || ZFS_JSON="null"
|
|
}
|
|
|
|
# The disk and pool readings --prepare left for this report, if they are there and recent. Re-shaped by jq, never trusted as they stand.
|
|
use_prepared_health() {
|
|
[[ -r "$HEALTH_FILE" ]] || return 1
|
|
[[ -n "$(find "$HEALTH_FILE" -mmin -10 -print 2>/dev/null)" ]] || return 1
|
|
SMART_JSON=$(jq -c 'if (.smart | type) == "array" then [.smart[] | {device: (.device | tostring), model: ((.model // "") | tostring),
|
|
healthy: (if (.healthy | type) == "boolean" then .healthy else null end),
|
|
temperature_c: (if (.temperature_c | type) == "number" then .temperature_c else null end),
|
|
wear_percent: (if (.wear_percent | type) == "number" then .wear_percent else null end)}] else null end' "$HEALTH_FILE" 2>/dev/null) || SMART_JSON="null"
|
|
ZFS_JSON=$(jq -c 'if (.zfs | type) == "array" then [.zfs[] | {name: (.name | tostring), health: ((.health // "UNKNOWN") | tostring),
|
|
capacity_percent: (if (.capacity_percent | type) == "number" then .capacity_percent else null end)}] else null end' "$HEALTH_FILE" 2>/dev/null) || ZFS_JSON="null"
|
|
return 0
|
|
}
|
|
|
|
# Replaces this script with the one this Homelab Manager serves, when an administrator has switched automatic updates on and this agent is
|
|
# behind. Nothing is trusted but the checksum: the (authenticated) answer gives the SHA-256 of the script, and a download that doesn't
|
|
# match it, doesn't look like this script, doesn't pass bash's syntax check or isn't newer is thrown away. The new file is put beside the
|
|
# old and moved over it in one step, and the new version then carries on from here. Runs only from --prepare (outside the sandbox, which
|
|
# keeps /usr read-only for the report itself); with --check-updates it says what it would do.
|
|
self_update() {
|
|
[[ -n "$API_URL" && -n "$API_TOKEN" ]] || return 0
|
|
[[ -z "${HLM_JUST_UPDATED:-}" ]] || return 0
|
|
local self work code reply_update version sha got
|
|
self=$(readlink -f "${BASH_SOURCE[0]}" 2>/dev/null) || return 0
|
|
[[ -f "$self" ]] || return 0
|
|
work=$(mktemp -d) || return 0
|
|
code=$(api_get "/api/agent/report/update?version=$AGENT_VERSION" "$work/reply.json" 2>/dev/null) || code=""
|
|
reply_update=$(jq -r '.update // false' "$work/reply.json" 2>/dev/null || echo false)
|
|
if [[ "$code" != "200" || "$reply_update" != "true" ]]; then
|
|
[[ "$MODE" == "check" ]] && echo "Automatic update: nothing to do (it is off, or this agent is current)."
|
|
rm -rf "$work"
|
|
return 0
|
|
fi
|
|
version=$(jq -r '.version // empty' "$work/reply.json" 2>/dev/null || true)
|
|
sha=$(jq -r '.sha256 // empty' "$work/reply.json" 2>/dev/null || true)
|
|
if [[ ! "$version" =~ ^[0-9]+$ || ! "$sha" =~ ^[0-9a-f]{64}$ ]] || (( version <= AGENT_VERSION )); then
|
|
rm -rf "$work"
|
|
return 0
|
|
fi
|
|
if ! curl -fsS "${CURL_INSECURE_FLAG[@]}" -m 60 -o "$work/new.sh" "$API_URL/agent/linux/report-tasks.sh" 2>"$PM_ERR"; then
|
|
echo "Note: couldn't download the newer agent (version $version), so it was not installed." >&2
|
|
rm -rf "$work"
|
|
return 0
|
|
fi
|
|
got=$(sha256sum "$work/new.sh" 2>/dev/null | awk '{print $1}')
|
|
if [[ "$got" != "$sha" ]] || [[ "$(head -n 1 "$work/new.sh")" != "#!/usr/bin/env bash" ]] || ! bash -n "$work/new.sh" 2>/dev/null || ! grep -q "^AGENT_VERSION=$version\$" "$work/new.sh"; then
|
|
echo "Note: the newer agent (version $version) didn't check out against what the server said, so it was not installed." >&2
|
|
rm -rf "$work"
|
|
return 0
|
|
fi
|
|
if [[ "$MODE" == "check" ]]; then
|
|
echo "Automatic update: version $version is available and would be installed on the next run."
|
|
rm -rf "$work"
|
|
return 0
|
|
fi
|
|
if install -m 755 "$work/new.sh" "$self.new" && mv -f "$self.new" "$self"; then
|
|
echo "Updated the agent from version $AGENT_VERSION to version $version." >&2
|
|
rm -rf "$work"
|
|
HLM_JUST_UPDATED=1 exec "$self" --prepare
|
|
fi
|
|
rm -f "$self.new"
|
|
echo "Note: couldn't replace the agent with version $version." >&2
|
|
rm -rf "$work"
|
|
return 0
|
|
}
|
|
|
|
# The answer --prepare left for this report, if it is there, recent and looks like what it should. Read back through jq rather than
|
|
# trusted: the report only ever carries these four fields, whatever the file holds.
|
|
use_prepared_updates() {
|
|
[[ -r "$UPDATES_FILE" ]] || return 1
|
|
[[ -n "$(find "$UPDATES_FILE" -mmin -10 -print 2>/dev/null)" ]] || return 1
|
|
local json
|
|
json=$(jq -ce 'select(type == "object" and (.manager | type) == "string" and (.count | type) == "number")
|
|
| {manager: .manager, count: .count, security: (if (.security | type) == "number" then .security else null end), reboot_required: (.reboot_required == true)}' "$UPDATES_FILE" 2>/dev/null) || return 1
|
|
[[ -n "$json" ]] || return 1
|
|
UPDATES_JSON=$json
|
|
}
|
|
|
|
# What this report could not collect: the notes made here, then those --prepare left (it does the outside-the-sandbox part), one for each check.
|
|
# With nothing left by --prepare — on a real run, not a dry one — that step did not run, which is worth saying: disk health and ZFS are only read there.
|
|
collect_diagnostics() {
|
|
local from_prepare="[]" shaped
|
|
if [[ -r "$DIAG_FILE" && -n "$(find "$DIAG_FILE" -mmin -10 -print 2>/dev/null)" ]]; then
|
|
from_prepare=$(jq -c 'if type == "array" then [.[] | select(type == "object" and (.check | type) == "string" and (.status | type) == "string")
|
|
| {check: (.check | tostring), status: (.status | tostring)}
|
|
+ (if ((.reason // "") | tostring) != "" then {reason: (.reason | tostring)} else {} end)
|
|
+ (if ((.detail // "") | tostring) != "" then {detail: (.detail | tostring | .[0:160])} else {} end)] | .[0:20] else [] end' "$DIAG_FILE" 2>/dev/null) || from_prepare="[]"
|
|
elif [[ "$DRY_RUN" != "1" ]]; then
|
|
diag prepare unavailable step_missing
|
|
fi
|
|
shaped=$(jq -c -n --argjson here "$(diag_json)" --argjson there "$from_prepare" '($here + $there) | unique_by(.check) | .[0:20]') || shaped="[]"
|
|
DIAGNOSTICS_JSON=$shaped
|
|
}
|
|
|
|
collect_system_info() {
|
|
local ip_json="[]"
|
|
if command -v ip >/dev/null 2>&1; then
|
|
ip_json=$(ip -4 -o addr show scope global 2>/dev/null \
|
|
| awk '{print $4}' | cut -d/ -f1 \
|
|
| jq -R -s -c 'split("\n") | map(select(length > 0))')
|
|
fi
|
|
|
|
local cpu_model="" cpu_cores=0 cpu_load_percent="null"
|
|
# x86 /proc/cpuinfo has a per-core "model name" line. Most 64-bit ARM
|
|
# kernels (Raspberry Pi included) don't — they instead have a single
|
|
# "Model" line at the very end, or nothing at all, in which case
|
|
# /proc/device-tree/model (present on any device-tree/SBC board) has the
|
|
# board's friendly name. Try each in turn; leave blank if none exist.
|
|
cpu_model=$(grep -m1 "^model name" /proc/cpuinfo 2>/dev/null | cut -d: -f2- | sed 's/^ *//' || true)
|
|
if [[ -z "$cpu_model" ]]; then
|
|
cpu_model=$(grep -m1 "^Model" /proc/cpuinfo 2>/dev/null | cut -d: -f2- | sed 's/^ *//' || true)
|
|
fi
|
|
if [[ -z "$cpu_model" && -r /proc/device-tree/model ]]; then
|
|
cpu_model=$(tr -d '\0' < /proc/device-tree/model 2>/dev/null || true)
|
|
fi
|
|
cpu_cores=$(nproc 2>/dev/null || echo 0)
|
|
if [[ -r /proc/loadavg && "$cpu_cores" -gt 0 ]]; then
|
|
local load1
|
|
load1=$(awk '{print $1}' /proc/loadavg)
|
|
cpu_load_percent=$(awk -v l="$load1" -v c="$cpu_cores" 'BEGIN { printf "%.1f", (l/c)*100 }')
|
|
fi
|
|
|
|
# How long the host has been up: the first figure in /proc/uptime, in seconds. null if it cannot be read.
|
|
local uptime_seconds="null"
|
|
if [[ -r /proc/uptime ]]; then
|
|
uptime_seconds=$(awk '{ printf "%d", $1 }' /proc/uptime 2>/dev/null || true)
|
|
[[ "$uptime_seconds" =~ ^[0-9]+$ ]] || uptime_seconds="null"
|
|
fi
|
|
|
|
local mem_total=0 mem_used=0
|
|
if command -v free >/dev/null 2>&1; then
|
|
read -r mem_total mem_used < <(free -b | awk '/^Mem:/ {print $2, $3}')
|
|
fi
|
|
|
|
local disks_json="[]"
|
|
if command -v df >/dev/null 2>&1; then
|
|
disks_json=$(df -B1 --output=target,size,used -x tmpfs -x devtmpfs -x squashfs -x overlay 2>/dev/null \
|
|
| tail -n +2 \
|
|
| awk '{print $1"\t"$2"\t"$3}' \
|
|
| jq -R -s -c '
|
|
split("\n") | map(select(length > 0) | split("\t")) |
|
|
map({mount: .[0], size_bytes: (.[1]|tonumber), used_bytes: (.[2]|tonumber)})
|
|
')
|
|
fi
|
|
|
|
# Everything bound to a port on this host, including services listening on localhost only (which a network
|
|
# scan from elsewhere can't see). `ss -p` needs root to name the process; without it the process is blank.
|
|
# One row per socket — the server groups them per port.
|
|
local ports_json="[]"
|
|
if command -v ss >/dev/null 2>&1; then
|
|
ports_json=$(ss -H -tulnp 2>/dev/null \
|
|
| awk '{
|
|
local = $5; port = local; sub(/.*:/, "", port); addr = local; sub(/:[0-9]+$/, "", addr);
|
|
proc = ""; if (match($0, /users:\(\("[^"]+"/)) { proc = substr($0, RSTART + 9, RLENGTH - 10) }
|
|
if (port ~ /^[0-9]+$/) print $1 "\t" port "\t" addr "\t" proc
|
|
}' \
|
|
| jq -R -s -c '
|
|
split("\n") | map(select(length > 0) | split("\t")) |
|
|
map({protocol: .[0], port: (.[1]|tonumber), address: .[2], process: (.[3] // "")})
|
|
')
|
|
fi
|
|
|
|
# Best-effort like the rest: a package manager that misbehaves must not cost the report.
|
|
UPDATES_JSON="null"
|
|
use_prepared_updates || collect_updates || UPDATES_JSON="null"
|
|
|
|
# Failed services are asked here (systemctl works inside the sandbox); disk and pool health come from --prepare. A person running --dry-run
|
|
# isn't in the sandbox, so there they are read directly.
|
|
FAILED_UNITS_JSON="null"
|
|
collect_failed_units || FAILED_UNITS_JSON="null"
|
|
SMART_JSON="null"
|
|
ZFS_JSON="null"
|
|
if ! use_prepared_health && [[ "$DRY_RUN" == "1" ]]; then
|
|
collect_smart || SMART_JSON="null"
|
|
collect_zfs || ZFS_JSON="null"
|
|
fi
|
|
HARDWARE_JSON="null"
|
|
collect_hardware || HARDWARE_JSON="null"
|
|
OS_JSON="null"
|
|
collect_os || OS_JSON="null"
|
|
collect_diagnostics
|
|
HEALTH_JSON=$(jq -n -c --argjson failed "$FAILED_UNITS_JSON" --argjson smart "$SMART_JSON" --argjson zfs "$ZFS_JSON" \
|
|
'{} | (if $failed != null then . + {failed_units: $failed} else . end) | (if $smart != null then . + {smart: $smart} else . end) | (if $zfs != null then . + {zfs: $zfs} else . end)') || HEALTH_JSON="{}"
|
|
[[ "$HEALTH_JSON" == "{}" ]] && HEALTH_JSON="null"
|
|
|
|
SYSTEM_JSON=$(jq -n \
|
|
--argjson updates "$UPDATES_JSON" \
|
|
--argjson health "$HEALTH_JSON" \
|
|
--argjson hardware "$HARDWARE_JSON" \
|
|
--argjson os "$OS_JSON" \
|
|
--argjson diagnostics "$DIAGNOSTICS_JSON" \
|
|
--argjson ip_addresses "$ip_json" \
|
|
--argjson listening_ports "$ports_json" \
|
|
--arg cpu_model "$cpu_model" \
|
|
--argjson cpu_cores "${cpu_cores:-0}" \
|
|
--argjson cpu_load_percent "$cpu_load_percent" \
|
|
--argjson mem_total_bytes "${mem_total:-0}" \
|
|
--argjson mem_used_bytes "${mem_used:-0}" \
|
|
--argjson uptime_seconds "$uptime_seconds" \
|
|
--argjson disks "$disks_json" \
|
|
'{
|
|
ip_addresses: $ip_addresses,
|
|
cpu: { model: $cpu_model, cores: $cpu_cores, load_percent: $cpu_load_percent },
|
|
memory: { total_bytes: $mem_total_bytes, used_bytes: $mem_used_bytes },
|
|
uptime_seconds: $uptime_seconds,
|
|
disks: $disks,
|
|
listening_ports: $listening_ports,
|
|
updates: $updates,
|
|
health: $health,
|
|
hardware: $hardware,
|
|
os: $os,
|
|
diagnostics: $diagnostics
|
|
}')
|
|
}
|
|
|
|
if [[ "$MODE" != "report" ]]; then
|
|
# prepare: first see whether this script is to be replaced (if so it carries on as the new one, from the top).
|
|
[[ "$MODE" == "prepare" ]] && self_update
|
|
collect_updates || UPDATES_JSON="null"
|
|
collect_smart || SMART_JSON="null"
|
|
collect_zfs || ZFS_JSON="null"
|
|
if [[ "$MODE" == "check" ]]; then
|
|
collect_failed_units || FAILED_UNITS_JSON="null"
|
|
echo "Pending updates as they would be reported:"
|
|
jq . <<<"$UPDATES_JSON"
|
|
echo "Failed services:"
|
|
jq . <<<"$FAILED_UNITS_JSON"
|
|
echo "Disk health (SMART; null means smartctl isn't installed or this isn't root):"
|
|
jq . <<<"$SMART_JSON"
|
|
echo "ZFS pools (null means there is no zpool here):"
|
|
jq . <<<"$ZFS_JSON"
|
|
collect_hardware || true
|
|
collect_os || true
|
|
echo "What it runs (null means there is no os-release to read):"
|
|
jq . <<<"$OS_JSON"
|
|
echo "What could not be collected (empty means nothing was missed):"
|
|
diag_json | jq .
|
|
self_update
|
|
exit 0
|
|
fi
|
|
# Leave the answers (or no file at all) for the report that follows. Each is written whole and moved into place, so a report never reads half of one.
|
|
rm -f "$UPDATES_FILE" "$HEALTH_FILE" "$DIAG_FILE"
|
|
mkdir -p "$(dirname "$UPDATES_FILE")" 2>/dev/null || exit 0
|
|
# The notes always go, even when there are none: a missing file is how the report knows this step didn't run.
|
|
diag_json > "$DIAG_FILE.tmp" 2>/dev/null && mv -f "$DIAG_FILE.tmp" "$DIAG_FILE" 2>/dev/null || true
|
|
if [[ "$UPDATES_JSON" != "null" ]]; then
|
|
printf '%s\n' "$UPDATES_JSON" > "$UPDATES_FILE.tmp" 2>/dev/null && mv -f "$UPDATES_FILE.tmp" "$UPDATES_FILE" 2>/dev/null || true
|
|
fi
|
|
HEALTH_FILE_JSON=$(jq -n -c --argjson smart "${SMART_JSON:-null}" --argjson zfs "${ZFS_JSON:-null}" '{} | (if $smart != null then . + {smart: $smart} else . end) | (if $zfs != null then . + {zfs: $zfs} else . end)' 2>/dev/null || echo "{}")
|
|
if [[ "$HEALTH_FILE_JSON" != "{}" ]]; then
|
|
printf '%s\n' "$HEALTH_FILE_JSON" > "$HEALTH_FILE.tmp" 2>/dev/null && mv -f "$HEALTH_FILE.tmp" "$HEALTH_FILE" 2>/dev/null || true
|
|
fi
|
|
exit 0
|
|
fi
|
|
|
|
collect_cron
|
|
collect_systemd_timers
|
|
|
|
# Hardware/network facts are best-effort: a quirk on any given host (e.g. no
|
|
# "model name" line in /proc/cpuinfo on some ARM boards) must never abort the
|
|
# whole report over it, so errexit is relaxed just for this call.
|
|
SYSTEM_JSON="null"
|
|
set +e
|
|
collect_system_info
|
|
system_info_status=$?
|
|
set -e
|
|
if [[ "$system_info_status" -ne 0 ]]; then
|
|
echo "Warning: collecting hardware/network info failed (exit $system_info_status) — reporting tasks without it." >&2
|
|
SYSTEM_JSON="null"
|
|
fi
|
|
|
|
HOSTNAME_VALUE=$(hostname -f 2>/dev/null || hostname)
|
|
PAYLOAD=$(jq -n \
|
|
--arg hostname "$HOSTNAME_VALUE" \
|
|
--arg os_type "linux" \
|
|
--arg reported_at "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
|
|
--argjson agent_version "$AGENT_VERSION" \
|
|
--argjson system "$SYSTEM_JSON" \
|
|
--argjson tasks "$TASKS_JSON" \
|
|
'{hostname: $hostname, os_type: $os_type, reported_at: $reported_at, agent_version: $agent_version, system: $system, tasks: $tasks}')
|
|
|
|
if [[ "$DRY_RUN" == "1" ]]; then
|
|
echo "$PAYLOAD" | jq .
|
|
exit 0
|
|
fi
|
|
|
|
HLM_TMPDIR=$(mktemp -d)
|
|
trap 'rm -rf "$HLM_TMPDIR"' EXIT
|
|
|
|
response=$(post_report)
|
|
|
|
if [[ "$response" -lt 200 || "$response" -ge 300 ]]; then
|
|
echo "Report failed with HTTP $response:" >&2
|
|
cat "$HLM_TMPDIR/response.json" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "Reported $(jq 'length' <<<"$TASKS_JSON") task(s) successfully."
|