- Dependencies: servers and integrations can depend on each other; alerts are folded under the root cause, and a failure explained by something down isn't announced - Web push: hand-written VAPID and RFC 8291 encryption, per-device subscriptions, a fifth notification channel, service worker handlers - Inventory: maker/model/serial from the agents fill blank fields, "Refresh from agent", QR label printing and ?asset= deep links - Windows agent (v4) reports failed services, disk health, Storage Spaces pools and hardware identity; Linux agent (v5) reports hardware identity Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
649 lines
30 KiB
Bash
649 lines
30 KiB
Bash
#!/usr/bin/env bash
|
|
# Collects cron jobs and systemd timers on this host and POSTs them to the
|
|
# Homelab Manager API. Intended to run as root via a periodic systemd timer
|
|
# (see install.sh) but can be run manually for testing:
|
|
#
|
|
# API_URL=https://homelab.example.lan API_TOKEN=hlm_xxx ./report-tasks.sh --dry-run
|
|
#
|
|
# Set API_INSECURE=true (also written to the env file by install.sh when
|
|
# passed there) if Homelab Manager uses a self-signed certificate.
|
|
#
|
|
set -euo pipefail
|
|
|
|
# Which version of this script this is, sent with every report so Homelab Manager can tell which agents are behind. Raise it whenever a
|
|
# change here is worth installing the agent again for (and add it to the list at the top of server/src/services/agentVersions.ts).
|
|
# 1 = before the number was reported 2 = reports pending updates, and its own version
|
|
# 3 = the package manager is asked outside the sandbox (see --prepare), so dnf and zypper can answer there too
|
|
# 4 = also reports failed services, disk health (SMART) and ZFS pools, reads pacman, and can update itself when told to
|
|
# 5 = also says what the machine is (maker, model, serial number), from the firmware, for the inventory
|
|
AGENT_VERSION=5
|
|
|
|
ENV_FILE="${ENV_FILE:-/etc/homelab-manager-agent.env}"
|
|
if [[ -f "$ENV_FILE" ]]; then
|
|
# shellcheck disable=SC1090
|
|
source "$ENV_FILE"
|
|
fi
|
|
|
|
API_URL="${API_URL:-}"
|
|
API_TOKEN="${API_TOKEN:-}"
|
|
API_INSECURE="${API_INSECURE:-false}"
|
|
DRY_RUN=0
|
|
[[ "${1:-}" == "--dry-run" ]] && DRY_RUN=1
|
|
|
|
# Two extra ways to run it, neither of which reports anything:
|
|
# --prepare the things that can't be done inside the systemd sandbox: asks the package manager what is waiting (dnf and zypper
|
|
# write to /var and /run even when only asked), reads disk health with smartctl and pool state with zpool (both need the
|
|
# devices, which the sandbox closes), and — if an administrator has switched it on — replaces this script with a newer
|
|
# one. The answers are left in the unit's private directory for the report to pick up. The unit runs this step outside
|
|
# its sandbox (ExecStartPre=-+); a failure here only means those figures are missing. --prepare-updates is the same.
|
|
# --check-updates for a person at a terminal: runs the same checks and shows what each said, and whether it would update itself.
|
|
MODE=report
|
|
case "${1:-}" in
|
|
--prepare|--prepare-updates) MODE=prepare ;;
|
|
--check-updates|--check) MODE=check ;;
|
|
esac
|
|
UPDATES_FILE="${UPDATES_FILE:-/run/homelab-manager-agent/updates.json}"
|
|
HEALTH_FILE="${HEALTH_FILE:-$(dirname "$UPDATES_FILE")/health.json}"
|
|
# Kept between runs (the unit's private directory is not): a SMART reading is only taken about once an hour, so sleeping disks are left alone.
|
|
STATE_DIR="${STATE_DIR:-/var/lib/homelab-manager-agent}"
|
|
# Where the package managers' own error output goes: nowhere, unless a person asked to see it.
|
|
PM_ERR=/dev/null
|
|
[[ "$MODE" == "check" ]] && PM_ERR=/dev/stderr
|
|
|
|
if [[ "$MODE" == "report" && ( -z "$API_URL" || -z "$API_TOKEN" ) ]]; then
|
|
echo "API_URL and API_TOKEN must be set (env vars or $ENV_FILE)" >&2
|
|
exit 1
|
|
fi
|
|
|
|
CURL_INSECURE_FLAG=()
|
|
case "${API_INSECURE,,}" in
|
|
1|true|yes) CURL_INSECURE_FLAG=(-k) ;;
|
|
esac
|
|
|
|
suggest_package_install() {
|
|
local pkg="$1"
|
|
if command -v apt-get >/dev/null 2>&1; then
|
|
echo " sudo apt-get update && sudo apt-get install -y $pkg"
|
|
elif command -v dnf >/dev/null 2>&1; then
|
|
echo " sudo dnf install -y $pkg"
|
|
elif command -v yum >/dev/null 2>&1; then
|
|
echo " sudo yum install -y $pkg"
|
|
elif command -v apk >/dev/null 2>&1; then
|
|
echo " sudo apk add $pkg"
|
|
elif command -v pacman >/dev/null 2>&1; then
|
|
echo " sudo pacman -S $pkg"
|
|
elif command -v zypper >/dev/null 2>&1; then
|
|
echo " sudo zypper install -y $pkg"
|
|
fi
|
|
}
|
|
|
|
for bin in curl jq; do
|
|
if ! command -v "$bin" >/dev/null 2>&1; then
|
|
echo "Required dependency '$bin' is not installed. Try:" >&2
|
|
suggest_package_install "$bin" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
TASKS_JSON="[]"
|
|
|
|
add_task() {
|
|
local schedule_type="$1" name="$2" command="$3" schedule_expression="$4" source="$5" enabled="$6"
|
|
TASKS_JSON=$(jq -c \
|
|
--arg schedule_type "$schedule_type" \
|
|
--arg name "$name" \
|
|
--arg command "$command" \
|
|
--arg schedule_expression "$schedule_expression" \
|
|
--arg source "$source" \
|
|
--argjson enabled "$enabled" \
|
|
'. + [{
|
|
schedule_type: $schedule_type,
|
|
name: $name,
|
|
command: $command,
|
|
schedule_expression: $schedule_expression,
|
|
source: $source,
|
|
enabled: $enabled
|
|
}]' <<<"$TASKS_JSON")
|
|
}
|
|
|
|
add_task_with_next_run() {
|
|
local schedule_type="$1" name="$2" command="$3" schedule_expression="$4" source="$5" enabled="$6" next_run_at="$7"
|
|
TASKS_JSON=$(jq -c \
|
|
--arg schedule_type "$schedule_type" \
|
|
--arg name "$name" \
|
|
--arg command "$command" \
|
|
--arg schedule_expression "$schedule_expression" \
|
|
--arg source "$source" \
|
|
--argjson enabled "$enabled" \
|
|
--arg next_run_at "$next_run_at" \
|
|
'. + [{
|
|
schedule_type: $schedule_type,
|
|
name: $name,
|
|
command: $command,
|
|
schedule_expression: $schedule_expression,
|
|
source: $source,
|
|
enabled: $enabled,
|
|
next_run_at: $next_run_at
|
|
}]' <<<"$TASKS_JSON")
|
|
}
|
|
|
|
parse_crontab_lines() {
|
|
# Reads 5-field-schedule + command cron lines from stdin.
|
|
# $1 = source label, $2 = "system" (7-field, includes a user column) or "user" (6-field)
|
|
local source="$1" mode="$2"
|
|
while IFS= read -r line; do
|
|
line="${line%%$'\r'}"
|
|
[[ -z "$line" ]] && continue
|
|
[[ "$line" =~ ^[[:space:]]*# ]] && continue
|
|
[[ "$line" =~ ^[[:space:]]*[A-Za-z_][A-Za-z0-9_]*= ]] && continue
|
|
|
|
if [[ "$mode" == "system" ]]; then
|
|
# min hour dom mon dow user command...
|
|
if [[ "$line" =~ ^[[:space:]]*([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+[^[:space:]]+[[:space:]]+(.+)$ ]]; then
|
|
local sched="${BASH_REMATCH[1]} ${BASH_REMATCH[2]} ${BASH_REMATCH[3]} ${BASH_REMATCH[4]} ${BASH_REMATCH[5]}"
|
|
local cmd="${BASH_REMATCH[6]}"
|
|
add_task "cron" "$cmd" "$cmd" "$sched" "$source" true
|
|
fi
|
|
else
|
|
# min hour dom mon dow command...
|
|
if [[ "$line" =~ ^[[:space:]]*([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+(.+)$ ]]; then
|
|
local sched="${BASH_REMATCH[1]} ${BASH_REMATCH[2]} ${BASH_REMATCH[3]} ${BASH_REMATCH[4]} ${BASH_REMATCH[5]}"
|
|
local cmd="${BASH_REMATCH[6]}"
|
|
add_task "cron" "$cmd" "$cmd" "$sched" "$source" true
|
|
fi
|
|
fi
|
|
done
|
|
}
|
|
|
|
collect_cron() {
|
|
[[ -r /etc/crontab ]] && parse_crontab_lines "/etc/crontab" "system" < /etc/crontab
|
|
|
|
if [[ -d /etc/cron.d ]]; then
|
|
for f in /etc/cron.d/*; do
|
|
[[ -f "$f" && -r "$f" ]] || continue
|
|
parse_crontab_lines "$f" "system" < "$f"
|
|
done
|
|
fi
|
|
|
|
if command -v crontab >/dev/null 2>&1 && [[ -r /etc/passwd ]]; then
|
|
while IFS=: read -r user _ uid _ _ _ shell; do
|
|
case "$shell" in
|
|
*/nologin|*/false|"") continue ;;
|
|
esac
|
|
[[ "$uid" -lt 1000 && "$uid" != "0" ]] && continue
|
|
local_crontab=$(crontab -l -u "$user" 2>/dev/null || true)
|
|
[[ -z "$local_crontab" ]] && continue
|
|
parse_crontab_lines "crontab:$user" "user" <<<"$local_crontab"
|
|
done < /etc/passwd
|
|
fi
|
|
}
|
|
|
|
collect_systemd_timers() {
|
|
command -v systemctl >/dev/null 2>&1 || return 0
|
|
|
|
local timers_json
|
|
timers_json=$(systemctl list-timers --all --output=json 2>/dev/null || echo "[]")
|
|
|
|
while IFS= read -r entry; do
|
|
local unit activates next_usec enabled_state schedule_expr next_run_at
|
|
unit=$(jq -r '.unit' <<<"$entry")
|
|
activates=$(jq -r '.activates // ""' <<<"$entry")
|
|
next_usec=$(jq -r '.next // 0' <<<"$entry")
|
|
|
|
enabled_state=$(systemctl is-enabled "$unit" 2>/dev/null || echo "unknown")
|
|
local enabled_bool="false"
|
|
[[ "$enabled_state" == "enabled" || "$enabled_state" == "static" ]] && enabled_bool="true"
|
|
|
|
schedule_expr=$(systemctl show "$unit" -p TimersCalendar --value 2>/dev/null | sed -n 's/.*OnCalendar=\([^;]*\);.*/\1/p' | sed 's/[[:space:]]*$//')
|
|
[[ -z "$schedule_expr" ]] && schedule_expr="(see: systemctl status $unit)"
|
|
|
|
next_run_at=""
|
|
if [[ "$next_usec" =~ ^[0-9]+$ && "$next_usec" -gt 0 ]]; then
|
|
next_run_at=$(date -u -d "@$((next_usec / 1000000))" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || echo "")
|
|
fi
|
|
|
|
if [[ -n "$next_run_at" ]]; then
|
|
add_task_with_next_run "systemd_timer" "$unit" "$activates" "$schedule_expr" "$unit" "$enabled_bool" "$next_run_at"
|
|
else
|
|
add_task "systemd_timer" "$unit" "$activates" "$schedule_expr" "$unit" "$enabled_bool"
|
|
fi
|
|
done < <(jq -c '.[]' <<<"$timers_json")
|
|
}
|
|
|
|
# Updates waiting to be installed, as JSON in UPDATES_JSON ("null" when this host's package manager isn't one we know, or couldn't answer).
|
|
# Nothing is installed or refreshed: this reads what the package manager already knows about, so it is quick and changes nothing.
|
|
collect_updates() {
|
|
UPDATES_JSON="null"
|
|
local manager="" count="" security="null" reboot="false" out="" status=0
|
|
|
|
if command -v apt-get >/dev/null 2>&1; then
|
|
manager="apt"
|
|
# -s simulates: "Inst" lines are what an upgrade would install. Security ones come from a "-security" or "Security" archive.
|
|
if out=$(LC_ALL=C timeout 60 apt-get -s -o Debug::NoLocking=true upgrade 2>"$PM_ERR"); then
|
|
count=$(grep -c '^Inst ' <<<"$out" || true)
|
|
security=$(grep '^Inst ' <<<"$out" | grep -ci 'security' || true)
|
|
fi
|
|
[[ -f /var/run/reboot-required ]] && reboot="true"
|
|
elif command -v dnf >/dev/null 2>&1; then
|
|
manager="dnf"
|
|
# check-update exits 100 when there are updates, 0 when none, 1 on an error. --cacheonly: never goes to the network from here.
|
|
out=$(LC_ALL=C timeout 120 dnf -q check-update --cacheonly 2>"$PM_ERR") || status=$?
|
|
if [[ "$status" -eq 0 || "$status" -eq 100 ]]; then
|
|
count=$(awk 'NF >= 3 && $1 ~ /^[A-Za-z0-9_.+-]+\.[A-Za-z0-9_]+$/ && /Obsoleting/ == 0 { n++ } /^Obsoleting/ { exit } END { print n + 0 }' <<<"$out")
|
|
security=$(LC_ALL=C timeout 120 dnf -q updateinfo list --security --cacheonly 2>"$PM_ERR" | grep -c '/Sec\.' || true)
|
|
fi
|
|
if command -v needs-restarting >/dev/null 2>&1; then
|
|
needs-restarting -r >/dev/null 2>&1 || reboot="true"
|
|
fi
|
|
elif command -v pacman >/dev/null 2>&1; then
|
|
manager="pacman"
|
|
# -Qu is what the local copy of the sync databases says could be upgraded (what `checkupdates` lists, without changing anything). It
|
|
# exits 1 when there is nothing to upgrade, so an empty answer with status 1 is "none", not an error. Packages held back are left out.
|
|
out=$(LC_ALL=C timeout 60 pacman -Qu 2>"$PM_ERR") || status=$?
|
|
if [[ "$status" -eq 0 || ( "$status" -eq 1 && -z "$out" ) ]]; then
|
|
count=$(grep -v '\[ignored\]' <<<"$out" | grep -c . || true)
|
|
fi
|
|
# No security figure (pacman doesn't say). A restart is due when the kernel running is no longer the one installed.
|
|
[[ -d "/usr/lib/modules/$(uname -r)" ]] || reboot="true"
|
|
elif command -v zypper >/dev/null 2>&1; then
|
|
manager="zypper"
|
|
if out=$(LC_ALL=C timeout 120 zypper -q --no-refresh list-updates 2>"$PM_ERR"); then
|
|
count=$(grep -c '^v |' <<<"$out" || true)
|
|
fi
|
|
[[ -f /run/reboot-needed ]] && reboot="true"
|
|
fi
|
|
|
|
if [[ -n "$manager" && ! "$count" =~ ^[0-9]+$ ]]; then
|
|
# In the journal (systemctl status / journalctl -u homelab-manager-agent) so "not reported" has a reason to look for.
|
|
echo "Note: couldn't read the pending updates from $manager - the report goes without them." >&2
|
|
fi
|
|
|
|
if [[ -n "$manager" && "$count" =~ ^[0-9]+$ ]]; then
|
|
[[ "$security" =~ ^[0-9]+$ ]] || security="null"
|
|
UPDATES_JSON=$(jq -n --arg manager "$manager" --argjson count "$count" --argjson security "$security" --argjson reboot "$reboot" \
|
|
'{manager: $manager, count: $count, security: $security, reboot_required: $reboot}')
|
|
fi
|
|
}
|
|
|
|
# >>> post_report
|
|
# Sends $PAYLOAD to the server and prints the HTTP status code.
|
|
#
|
|
# Nothing sensitive goes on curl's command line: any user on this machine can read other processes' command lines, and the
|
|
# token would be there for the length of every report. So the token is handed to curl on its standard input, as a line of
|
|
# curl's own config format, and the payload and the server's reply go through files in a private directory (mktemp -d is
|
|
# readable by root only). A fixed name in /tmp is no good either — another user could plant a symlink there and have this
|
|
# script, running as root, overwrite whatever it points at.
|
|
curl_quote() {
|
|
local s=$1
|
|
s=${s//\\/\\\\}
|
|
s=${s//\"/\\\"}
|
|
printf '%s' "$s"
|
|
}
|
|
|
|
post_report() {
|
|
case "$API_TOKEN" in
|
|
*$'\n'*|*$'\r'*) echo "API_TOKEN must not contain a line break." >&2; return 1 ;;
|
|
esac
|
|
printf '%s' "$PAYLOAD" > "$HLM_TMPDIR/payload.json"
|
|
printf 'header = "Authorization: Bearer %s"\n' "$(curl_quote "$API_TOKEN")" | curl -sS "${CURL_INSECURE_FLAG[@]}" -K - \
|
|
-o "$HLM_TMPDIR/response.json" -w "%{http_code}" \
|
|
-X POST "$API_URL/api/agent/report" \
|
|
-H "Content-Type: application/json" \
|
|
--data-binary @"$HLM_TMPDIR/payload.json"
|
|
}
|
|
# <<< post_report
|
|
|
|
# A GET with the token handed to curl on its standard input (as the upload does), the reply into the file named; prints the HTTP status.
|
|
api_get() {
|
|
printf 'header = "Authorization: Bearer %s"\n' "$(curl_quote "$API_TOKEN")" | curl -sS "${CURL_INSECURE_FLAG[@]}" -K - -m 30 -o "$2" -w "%{http_code}" "$API_URL$1"
|
|
}
|
|
|
|
# One value from the firmware's table (the kernel shows it under /sys/class/dmi/id), trying the named files in turn and leaving out what a
|
|
# maker writes when it has nothing to say ("To Be Filled By O.E.M.", "Default string", all zeros ...). DMI_DIR is for the tests.
|
|
dmi_value() {
|
|
local name value
|
|
for name in "$@"; do
|
|
# (the file is checked first: a failing redirection prints its complaint before any 2>/dev/null applies)
|
|
[[ -r "${DMI_DIR:-/sys/class/dmi/id}/$name" ]] || continue
|
|
value=$(tr -d '\0' < "${DMI_DIR:-/sys/class/dmi/id}/$name" 2>/dev/null | head -c 200 | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') || value=""
|
|
[[ -n "$value" ]] || continue
|
|
case "${value,,}" in
|
|
"to be filled by o.e.m."|"default string"|"system serial number"|"system product name"|"system manufacturer"|"not specified"|"not applicable"|"none"|"n/a"|"unknown"|"o.e.m."|"oem") continue ;;
|
|
esac
|
|
[[ "$value" =~ ^0+$ ]] && continue
|
|
printf '%s' "$value"
|
|
return 0
|
|
done
|
|
return 0
|
|
}
|
|
|
|
# What the machine is: maker, model and serial number (a serial is only readable by root). "null" when the firmware has nothing to say, as on
|
|
# many virtual machines and single-board computers.
|
|
collect_hardware() {
|
|
HARDWARE_JSON="null"
|
|
local maker model serial
|
|
maker=$(dmi_value sys_vendor board_vendor)
|
|
model=$(dmi_value product_name board_name)
|
|
serial=$(dmi_value product_serial board_serial)
|
|
[[ -n "$maker$model$serial" ]] || return 0
|
|
HARDWARE_JSON=$(jq -n -c --arg maker "$maker" --arg model "$model" --arg serial "$serial" \
|
|
'{} | (if $maker != "" then . + {manufacturer: $maker} else . end) | (if $model != "" then . + {model: $model} else . end) | (if $serial != "" then . + {serial: $serial} else . end)') || HARDWARE_JSON="null"
|
|
}
|
|
|
|
# Services (systemd units) that have failed. systemctl is asked the way the timers are, so this works inside the sandbox.
|
|
collect_failed_units() {
|
|
FAILED_UNITS_JSON="null"
|
|
command -v systemctl >/dev/null 2>&1 || return 0
|
|
local out
|
|
out=$(timeout 20 systemctl list-units --state=failed --plain --no-legend --no-pager --full 2>"$PM_ERR") || return 0
|
|
FAILED_UNITS_JSON=$(awk 'NF > 0 { unit = $1; $1 = $2 = $3 = $4 = ""; sub(/^ +/, ""); print unit "\t" $0 }' <<<"$out" \
|
|
| jq -R -s -c 'split("\n") | map(select(length > 0) | split("\t")) | map({unit: .[0], description: (.[1] // "")}) | .[0:100]') || FAILED_UNITS_JSON="null"
|
|
}
|
|
|
|
# Each disk's own SMART verdict, temperature and (where it says) how much of its rated life is used. Needs the devices, so it runs from
|
|
# --prepare. Asked at most about once an hour, and with -n standby so a sleeping disk is not woken: what was last known of it is kept.
|
|
collect_smart() {
|
|
SMART_JSON="null"
|
|
command -v smartctl >/dev/null 2>&1 || return 0
|
|
local cache="$STATE_DIR/smart.json" now_s previous="[]" items="[]" dev out item scan checked
|
|
now_s=$(date +%s)
|
|
if [[ -r "$cache" ]]; then
|
|
checked=$(jq -r '.checked_at // 0' "$cache" 2>/dev/null || echo 0)
|
|
previous=$(jq -c '.devices // []' "$cache" 2>/dev/null || echo "[]")
|
|
if [[ "$checked" =~ ^[0-9]+$ ]] && (( now_s - checked < 3300 )) && [[ "$MODE" != "check" ]]; then
|
|
SMART_JSON=$previous
|
|
return 0
|
|
fi
|
|
fi
|
|
scan=$(timeout 30 smartctl --scan -j 2>"$PM_ERR") || true
|
|
while IFS= read -r dev; do
|
|
[[ -n "$dev" ]] || continue
|
|
# smartctl's exit status is a bit mask that says "something was noted" more often than "it failed", so it is read from the JSON instead.
|
|
out=$(timeout 30 smartctl -j -H -A -n standby "$dev" 2>"$PM_ERR") || true
|
|
item=$(jq -c --arg dev "$dev" 'select(.smart_status != null) | {
|
|
device: $dev,
|
|
model: (.model_name // .scsi_model_name // ""),
|
|
healthy: .smart_status.passed,
|
|
temperature_c: (.temperature.current // null),
|
|
wear_percent: (.nvme_smart_health_information_log.percentage_used // ([.ata_smart_attributes.table[]? | select(.id == 177 or .id == 231 or .id == 233) | (100 - .value)] | first) // null)
|
|
}' <<<"$out" 2>/dev/null) || item=""
|
|
if [[ -z "$item" ]]; then
|
|
# Asleep, or no SMART at all (a virtual disk): keep what was known of it, if anything.
|
|
item=$(jq -c --arg dev "$dev" '.[] | select(.device == $dev)' <<<"$previous" 2>/dev/null | head -n 1) || item=""
|
|
fi
|
|
if [[ -n "$item" ]]; then items=$(jq -c --argjson i "$item" '. + [$i]' <<<"$items"); fi
|
|
done < <(jq -r '.devices[]?.name' <<<"$scan" 2>/dev/null)
|
|
SMART_JSON=$items
|
|
if mkdir -p "$STATE_DIR" 2>/dev/null; then
|
|
chmod 700 "$STATE_DIR" 2>/dev/null || true
|
|
jq -c -n --argjson at "$now_s" --argjson devices "$items" '{checked_at: $at, devices: $devices}' > "$cache.tmp" 2>/dev/null && mv -f "$cache.tmp" "$cache" 2>/dev/null || true
|
|
fi
|
|
}
|
|
|
|
# Each ZFS pool and whether it is ONLINE. Needs /dev/zfs, so it runs from --prepare too.
|
|
collect_zfs() {
|
|
ZFS_JSON="null"
|
|
command -v zpool >/dev/null 2>&1 || return 0
|
|
local out
|
|
out=$(timeout 20 zpool list -H -o name,health,capacity 2>"$PM_ERR") || return 0
|
|
ZFS_JSON=$(jq -R -s -c 'split("\n") | map(select(length > 0) | split("\t")) | map({name: .[0], health: (.[1] // "UNKNOWN"), capacity_percent: ((.[2] // "" | rtrimstr("%") | tonumber?) // null)})' <<<"$out") || ZFS_JSON="null"
|
|
}
|
|
|
|
# The disk and pool readings --prepare left for this report, if they are there and recent. Re-shaped by jq, never trusted as they stand.
|
|
use_prepared_health() {
|
|
[[ -r "$HEALTH_FILE" ]] || return 1
|
|
[[ -n "$(find "$HEALTH_FILE" -mmin -10 -print 2>/dev/null)" ]] || return 1
|
|
SMART_JSON=$(jq -c 'if (.smart | type) == "array" then [.smart[] | {device: (.device | tostring), model: ((.model // "") | tostring),
|
|
healthy: (if (.healthy | type) == "boolean" then .healthy else null end),
|
|
temperature_c: (if (.temperature_c | type) == "number" then .temperature_c else null end),
|
|
wear_percent: (if (.wear_percent | type) == "number" then .wear_percent else null end)}] else null end' "$HEALTH_FILE" 2>/dev/null) || SMART_JSON="null"
|
|
ZFS_JSON=$(jq -c 'if (.zfs | type) == "array" then [.zfs[] | {name: (.name | tostring), health: ((.health // "UNKNOWN") | tostring),
|
|
capacity_percent: (if (.capacity_percent | type) == "number" then .capacity_percent else null end)}] else null end' "$HEALTH_FILE" 2>/dev/null) || ZFS_JSON="null"
|
|
return 0
|
|
}
|
|
|
|
# Replaces this script with the one this Homelab Manager serves, when an administrator has switched automatic updates on and this agent is
|
|
# behind. Nothing is trusted but the checksum: the (authenticated) answer gives the SHA-256 of the script, and a download that doesn't
|
|
# match it, doesn't look like this script, doesn't pass bash's syntax check or isn't newer is thrown away. The new file is put beside the
|
|
# old and moved over it in one step, and the new version then carries on from here. Runs only from --prepare (outside the sandbox, which
|
|
# keeps /usr read-only for the report itself); with --check-updates it says what it would do.
|
|
self_update() {
|
|
[[ -n "$API_URL" && -n "$API_TOKEN" ]] || return 0
|
|
[[ -z "${HLM_JUST_UPDATED:-}" ]] || return 0
|
|
local self work code reply_update version sha got
|
|
self=$(readlink -f "${BASH_SOURCE[0]}" 2>/dev/null) || return 0
|
|
[[ -f "$self" ]] || return 0
|
|
work=$(mktemp -d) || return 0
|
|
code=$(api_get "/api/agent/report/update?version=$AGENT_VERSION" "$work/reply.json" 2>/dev/null) || code=""
|
|
reply_update=$(jq -r '.update // false' "$work/reply.json" 2>/dev/null || echo false)
|
|
if [[ "$code" != "200" || "$reply_update" != "true" ]]; then
|
|
[[ "$MODE" == "check" ]] && echo "Automatic update: nothing to do (it is off, or this agent is current)."
|
|
rm -rf "$work"
|
|
return 0
|
|
fi
|
|
version=$(jq -r '.version // empty' "$work/reply.json" 2>/dev/null || true)
|
|
sha=$(jq -r '.sha256 // empty' "$work/reply.json" 2>/dev/null || true)
|
|
if [[ ! "$version" =~ ^[0-9]+$ || ! "$sha" =~ ^[0-9a-f]{64}$ ]] || (( version <= AGENT_VERSION )); then
|
|
rm -rf "$work"
|
|
return 0
|
|
fi
|
|
if ! curl -fsS "${CURL_INSECURE_FLAG[@]}" -m 60 -o "$work/new.sh" "$API_URL/agent/linux/report-tasks.sh" 2>"$PM_ERR"; then
|
|
echo "Note: couldn't download the newer agent (version $version), so it was not installed." >&2
|
|
rm -rf "$work"
|
|
return 0
|
|
fi
|
|
got=$(sha256sum "$work/new.sh" 2>/dev/null | awk '{print $1}')
|
|
if [[ "$got" != "$sha" ]] || [[ "$(head -n 1 "$work/new.sh")" != "#!/usr/bin/env bash" ]] || ! bash -n "$work/new.sh" 2>/dev/null || ! grep -q "^AGENT_VERSION=$version\$" "$work/new.sh"; then
|
|
echo "Note: the newer agent (version $version) didn't check out against what the server said, so it was not installed." >&2
|
|
rm -rf "$work"
|
|
return 0
|
|
fi
|
|
if [[ "$MODE" == "check" ]]; then
|
|
echo "Automatic update: version $version is available and would be installed on the next run."
|
|
rm -rf "$work"
|
|
return 0
|
|
fi
|
|
if install -m 755 "$work/new.sh" "$self.new" && mv -f "$self.new" "$self"; then
|
|
echo "Updated the agent from version $AGENT_VERSION to version $version." >&2
|
|
rm -rf "$work"
|
|
HLM_JUST_UPDATED=1 exec "$self" --prepare
|
|
fi
|
|
rm -f "$self.new"
|
|
echo "Note: couldn't replace the agent with version $version." >&2
|
|
rm -rf "$work"
|
|
return 0
|
|
}
|
|
|
|
# The answer --prepare left for this report, if it is there, recent and looks like what it should. Read back through jq rather than
|
|
# trusted: the report only ever carries these four fields, whatever the file holds.
|
|
use_prepared_updates() {
|
|
[[ -r "$UPDATES_FILE" ]] || return 1
|
|
[[ -n "$(find "$UPDATES_FILE" -mmin -10 -print 2>/dev/null)" ]] || return 1
|
|
local json
|
|
json=$(jq -ce 'select(type == "object" and (.manager | type) == "string" and (.count | type) == "number")
|
|
| {manager: .manager, count: .count, security: (if (.security | type) == "number" then .security else null end), reboot_required: (.reboot_required == true)}' "$UPDATES_FILE" 2>/dev/null) || return 1
|
|
[[ -n "$json" ]] || return 1
|
|
UPDATES_JSON=$json
|
|
}
|
|
|
|
collect_system_info() {
|
|
local ip_json="[]"
|
|
if command -v ip >/dev/null 2>&1; then
|
|
ip_json=$(ip -4 -o addr show scope global 2>/dev/null \
|
|
| awk '{print $4}' | cut -d/ -f1 \
|
|
| jq -R -s -c 'split("\n") | map(select(length > 0))')
|
|
fi
|
|
|
|
local cpu_model="" cpu_cores=0 cpu_load_percent="null"
|
|
# x86 /proc/cpuinfo has a per-core "model name" line. Most 64-bit ARM
|
|
# kernels (Raspberry Pi included) don't — they instead have a single
|
|
# "Model" line at the very end, or nothing at all, in which case
|
|
# /proc/device-tree/model (present on any device-tree/SBC board) has the
|
|
# board's friendly name. Try each in turn; leave blank if none exist.
|
|
cpu_model=$(grep -m1 "^model name" /proc/cpuinfo 2>/dev/null | cut -d: -f2- | sed 's/^ *//' || true)
|
|
if [[ -z "$cpu_model" ]]; then
|
|
cpu_model=$(grep -m1 "^Model" /proc/cpuinfo 2>/dev/null | cut -d: -f2- | sed 's/^ *//' || true)
|
|
fi
|
|
if [[ -z "$cpu_model" && -r /proc/device-tree/model ]]; then
|
|
cpu_model=$(tr -d '\0' < /proc/device-tree/model 2>/dev/null || true)
|
|
fi
|
|
cpu_cores=$(nproc 2>/dev/null || echo 0)
|
|
if [[ -r /proc/loadavg && "$cpu_cores" -gt 0 ]]; then
|
|
local load1
|
|
load1=$(awk '{print $1}' /proc/loadavg)
|
|
cpu_load_percent=$(awk -v l="$load1" -v c="$cpu_cores" 'BEGIN { printf "%.1f", (l/c)*100 }')
|
|
fi
|
|
|
|
local mem_total=0 mem_used=0
|
|
if command -v free >/dev/null 2>&1; then
|
|
read -r mem_total mem_used < <(free -b | awk '/^Mem:/ {print $2, $3}')
|
|
fi
|
|
|
|
local disks_json="[]"
|
|
if command -v df >/dev/null 2>&1; then
|
|
disks_json=$(df -B1 --output=target,size,used -x tmpfs -x devtmpfs -x squashfs -x overlay 2>/dev/null \
|
|
| tail -n +2 \
|
|
| awk '{print $1"\t"$2"\t"$3}' \
|
|
| jq -R -s -c '
|
|
split("\n") | map(select(length > 0) | split("\t")) |
|
|
map({mount: .[0], size_bytes: (.[1]|tonumber), used_bytes: (.[2]|tonumber)})
|
|
')
|
|
fi
|
|
|
|
# Everything bound to a port on this host, including services listening on localhost only (which a network
|
|
# scan from elsewhere can't see). `ss -p` needs root to name the process; without it the process is blank.
|
|
# One row per socket — the server groups them per port.
|
|
local ports_json="[]"
|
|
if command -v ss >/dev/null 2>&1; then
|
|
ports_json=$(ss -H -tulnp 2>/dev/null \
|
|
| awk '{
|
|
local = $5; port = local; sub(/.*:/, "", port); addr = local; sub(/:[0-9]+$/, "", addr);
|
|
proc = ""; if (match($0, /users:\(\("[^"]+"/)) { proc = substr($0, RSTART + 9, RLENGTH - 10) }
|
|
if (port ~ /^[0-9]+$/) print $1 "\t" port "\t" addr "\t" proc
|
|
}' \
|
|
| jq -R -s -c '
|
|
split("\n") | map(select(length > 0) | split("\t")) |
|
|
map({protocol: .[0], port: (.[1]|tonumber), address: .[2], process: (.[3] // "")})
|
|
')
|
|
fi
|
|
|
|
# Best-effort like the rest: a package manager that misbehaves must not cost the report.
|
|
UPDATES_JSON="null"
|
|
use_prepared_updates || collect_updates || UPDATES_JSON="null"
|
|
|
|
# Failed services are asked here (systemctl works inside the sandbox); disk and pool health come from --prepare. A person running --dry-run
|
|
# isn't in the sandbox, so there they are read directly.
|
|
FAILED_UNITS_JSON="null"
|
|
collect_failed_units || FAILED_UNITS_JSON="null"
|
|
SMART_JSON="null"
|
|
ZFS_JSON="null"
|
|
if ! use_prepared_health && [[ "$DRY_RUN" == "1" ]]; then
|
|
collect_smart || SMART_JSON="null"
|
|
collect_zfs || ZFS_JSON="null"
|
|
fi
|
|
HARDWARE_JSON="null"
|
|
collect_hardware || HARDWARE_JSON="null"
|
|
HEALTH_JSON=$(jq -n -c --argjson failed "$FAILED_UNITS_JSON" --argjson smart "$SMART_JSON" --argjson zfs "$ZFS_JSON" \
|
|
'{} | (if $failed != null then . + {failed_units: $failed} else . end) | (if $smart != null then . + {smart: $smart} else . end) | (if $zfs != null then . + {zfs: $zfs} else . end)') || HEALTH_JSON="{}"
|
|
[[ "$HEALTH_JSON" == "{}" ]] && HEALTH_JSON="null"
|
|
|
|
SYSTEM_JSON=$(jq -n \
|
|
--argjson updates "$UPDATES_JSON" \
|
|
--argjson health "$HEALTH_JSON" \
|
|
--argjson hardware "$HARDWARE_JSON" \
|
|
--argjson ip_addresses "$ip_json" \
|
|
--argjson listening_ports "$ports_json" \
|
|
--arg cpu_model "$cpu_model" \
|
|
--argjson cpu_cores "${cpu_cores:-0}" \
|
|
--argjson cpu_load_percent "$cpu_load_percent" \
|
|
--argjson mem_total_bytes "${mem_total:-0}" \
|
|
--argjson mem_used_bytes "${mem_used:-0}" \
|
|
--argjson disks "$disks_json" \
|
|
'{
|
|
ip_addresses: $ip_addresses,
|
|
cpu: { model: $cpu_model, cores: $cpu_cores, load_percent: $cpu_load_percent },
|
|
memory: { total_bytes: $mem_total_bytes, used_bytes: $mem_used_bytes },
|
|
disks: $disks,
|
|
listening_ports: $listening_ports,
|
|
updates: $updates,
|
|
health: $health,
|
|
hardware: $hardware
|
|
}')
|
|
}
|
|
|
|
if [[ "$MODE" != "report" ]]; then
|
|
# prepare: first see whether this script is to be replaced (if so it carries on as the new one, from the top).
|
|
[[ "$MODE" == "prepare" ]] && self_update
|
|
collect_updates || UPDATES_JSON="null"
|
|
collect_smart || SMART_JSON="null"
|
|
collect_zfs || ZFS_JSON="null"
|
|
if [[ "$MODE" == "check" ]]; then
|
|
collect_failed_units || FAILED_UNITS_JSON="null"
|
|
echo "Pending updates as they would be reported:"
|
|
jq . <<<"$UPDATES_JSON"
|
|
echo "Failed services:"
|
|
jq . <<<"$FAILED_UNITS_JSON"
|
|
echo "Disk health (SMART; null means smartctl isn't installed or this isn't root):"
|
|
jq . <<<"$SMART_JSON"
|
|
echo "ZFS pools (null means there is no zpool here):"
|
|
jq . <<<"$ZFS_JSON"
|
|
self_update
|
|
exit 0
|
|
fi
|
|
# Leave the answers (or no file at all) for the report that follows. Each is written whole and moved into place, so a report never reads half of one.
|
|
rm -f "$UPDATES_FILE" "$HEALTH_FILE"
|
|
mkdir -p "$(dirname "$UPDATES_FILE")" 2>/dev/null || exit 0
|
|
if [[ "$UPDATES_JSON" != "null" ]]; then
|
|
printf '%s\n' "$UPDATES_JSON" > "$UPDATES_FILE.tmp" 2>/dev/null && mv -f "$UPDATES_FILE.tmp" "$UPDATES_FILE" 2>/dev/null || true
|
|
fi
|
|
HEALTH_FILE_JSON=$(jq -n -c --argjson smart "${SMART_JSON:-null}" --argjson zfs "${ZFS_JSON:-null}" '{} | (if $smart != null then . + {smart: $smart} else . end) | (if $zfs != null then . + {zfs: $zfs} else . end)' 2>/dev/null || echo "{}")
|
|
if [[ "$HEALTH_FILE_JSON" != "{}" ]]; then
|
|
printf '%s\n' "$HEALTH_FILE_JSON" > "$HEALTH_FILE.tmp" 2>/dev/null && mv -f "$HEALTH_FILE.tmp" "$HEALTH_FILE" 2>/dev/null || true
|
|
fi
|
|
exit 0
|
|
fi
|
|
|
|
collect_cron
|
|
collect_systemd_timers
|
|
|
|
# Hardware/network facts are best-effort: a quirk on any given host (e.g. no
|
|
# "model name" line in /proc/cpuinfo on some ARM boards) must never abort the
|
|
# whole report over it, so errexit is relaxed just for this call.
|
|
SYSTEM_JSON="null"
|
|
set +e
|
|
collect_system_info
|
|
system_info_status=$?
|
|
set -e
|
|
if [[ "$system_info_status" -ne 0 ]]; then
|
|
echo "Warning: collecting hardware/network info failed (exit $system_info_status) — reporting tasks without it." >&2
|
|
SYSTEM_JSON="null"
|
|
fi
|
|
|
|
HOSTNAME_VALUE=$(hostname -f 2>/dev/null || hostname)
|
|
PAYLOAD=$(jq -n \
|
|
--arg hostname "$HOSTNAME_VALUE" \
|
|
--arg os_type "linux" \
|
|
--arg reported_at "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
|
|
--argjson agent_version "$AGENT_VERSION" \
|
|
--argjson system "$SYSTEM_JSON" \
|
|
--argjson tasks "$TASKS_JSON" \
|
|
'{hostname: $hostname, os_type: $os_type, reported_at: $reported_at, agent_version: $agent_version, system: $system, tasks: $tasks}')
|
|
|
|
if [[ "$DRY_RUN" == "1" ]]; then
|
|
echo "$PAYLOAD" | jq .
|
|
exit 0
|
|
fi
|
|
|
|
HLM_TMPDIR=$(mktemp -d)
|
|
trap 'rm -rf "$HLM_TMPDIR"' EXIT
|
|
|
|
response=$(post_report)
|
|
|
|
if [[ "$response" -lt 200 || "$response" -ge 300 ]]; then
|
|
echo "Report failed with HTTP $response:" >&2
|
|
cat "$HLM_TMPDIR/response.json" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "Reported $(jq 'length' <<<"$TASKS_JSON") task(s) successfully."
|