#!/usr/bin/env bash # Collects cron jobs and systemd timers on this host and POSTs them to the # Homelab Manager API. Intended to run as root via a periodic systemd timer # (see install.sh) but can be run manually for testing: # # API_URL=https://homelab.example.lan API_TOKEN=hlm_xxx ./report-tasks.sh --dry-run # # Set API_INSECURE=true (also written to the env file by install.sh when # passed there) if Homelab Manager uses a self-signed certificate. # set -euo pipefail # Which version of this script this is, sent with every report so Homelab Manager can tell which agents are behind. Raise it whenever a # change here is worth installing the agent again for (and add it to the list at the top of server/src/services/agentVersions.ts). # 1 = before the number was reported 2 = reports pending updates, and its own version # 3 = the package manager is asked outside the sandbox (see --prepare), so dnf and zypper can answer there too # 4 = also reports failed services, disk health (SMART) and ZFS pools, reads pacman, and can update itself when told to # 5 = also says what the machine is (maker, model, serial number), from the firmware, for the inventory # 6 = also says what it could not collect and why (no smartctl, no systemd, a package manager that failed ...), so the server can say what to fix AGENT_VERSION=6 ENV_FILE="${ENV_FILE:-/etc/homelab-manager-agent.env}" if [[ -f "$ENV_FILE" ]]; then # shellcheck disable=SC1090 source "$ENV_FILE" fi API_URL="${API_URL:-}" API_TOKEN="${API_TOKEN:-}" API_INSECURE="${API_INSECURE:-false}" DRY_RUN=0 [[ "${1:-}" == "--dry-run" ]] && DRY_RUN=1 # Two extra ways to run it, neither of which reports anything: # --prepare the things that can't be done inside the systemd sandbox: asks the package manager what is waiting (dnf and zypper # write to /var and /run even when only asked), reads disk health with smartctl and pool state with zpool (both need the # devices, which the sandbox closes), and — if an administrator has switched it on — replaces this script with a newer # one. The answers are left in the unit's private directory for the report to pick up. The unit runs this step outside # its sandbox (ExecStartPre=-+); a failure here only means those figures are missing. --prepare-updates is the same. # --check-updates for a person at a terminal: runs the same checks and shows what each said, and whether it would update itself. MODE=report case "${1:-}" in --prepare|--prepare-updates) MODE=prepare ;; --check-updates|--check) MODE=check ;; esac UPDATES_FILE="${UPDATES_FILE:-/run/homelab-manager-agent/updates.json}" HEALTH_FILE="${HEALTH_FILE:-$(dirname "$UPDATES_FILE")/health.json}" # What --prepare could not collect and why, left for the report to carry along. DIAG_FILE="${DIAG_FILE:-$(dirname "$UPDATES_FILE")/diagnostics.json}" # Kept between runs (the unit's private directory is not): a SMART reading is only taken about once an hour, so sleeping disks are left alone. STATE_DIR="${STATE_DIR:-/var/lib/homelab-manager-agent}" # Where the package managers' own error output goes: nowhere, unless a person asked to see it. PM_ERR=/dev/null [[ "$MODE" == "check" ]] && PM_ERR=/dev/stderr if [[ "$MODE" == "report" && ( -z "$API_URL" || -z "$API_TOKEN" ) ]]; then echo "API_URL and API_TOKEN must be set (env vars or $ENV_FILE)" >&2 exit 1 fi # Things that could not be collected, each a small JSON object: which check, whether it is "unavailable" (nothing to read it with, or nothing # there to read) or "failed" (it tried and could not), why in a word, and at most a line of what the tool said. Only problems are noted: a # check that is not mentioned went fine. The server turns these into what to do about them. DIAG_ITEMS=() diag() { local check="$1" status="$2" reason="${3:-}" detail="${4:-}" detail=$(printf '%s' "$detail" | tr -d '\000-\037' | head -c 160) DIAG_ITEMS+=("$(jq -n -c --arg c "$check" --arg s "$status" --arg r "$reason" --arg d "$detail" \ '{check: $c, status: $s} + (if $r != "" then {reason: $r} else {} end) + (if $d != "" then {detail: $d} else {} end)')") } # The notes made so far as one JSON array ([] when there are none). diag_json() { if (( ${#DIAG_ITEMS[@]} == 0 )); then echo "[]"; return 0; fi printf '%s\n' "${DIAG_ITEMS[@]}" | jq -s -c '.' } CURL_INSECURE_FLAG=() case "${API_INSECURE,,}" in 1|true|yes) CURL_INSECURE_FLAG=(-k) ;; esac suggest_package_install() { local pkg="$1" if command -v apt-get >/dev/null 2>&1; then echo " sudo apt-get update && sudo apt-get install -y $pkg" elif command -v dnf >/dev/null 2>&1; then echo " sudo dnf install -y $pkg" elif command -v yum >/dev/null 2>&1; then echo " sudo yum install -y $pkg" elif command -v apk >/dev/null 2>&1; then echo " sudo apk add $pkg" elif command -v pacman >/dev/null 2>&1; then echo " sudo pacman -S $pkg" elif command -v zypper >/dev/null 2>&1; then echo " sudo zypper install -y $pkg" fi } for bin in curl jq; do if ! command -v "$bin" >/dev/null 2>&1; then echo "Required dependency '$bin' is not installed. Try:" >&2 suggest_package_install "$bin" >&2 exit 1 fi done TASKS_JSON="[]" add_task() { local schedule_type="$1" name="$2" command="$3" schedule_expression="$4" source="$5" enabled="$6" TASKS_JSON=$(jq -c \ --arg schedule_type "$schedule_type" \ --arg name "$name" \ --arg command "$command" \ --arg schedule_expression "$schedule_expression" \ --arg source "$source" \ --argjson enabled "$enabled" \ '. + [{ schedule_type: $schedule_type, name: $name, command: $command, schedule_expression: $schedule_expression, source: $source, enabled: $enabled }]' <<<"$TASKS_JSON") } add_task_with_next_run() { local schedule_type="$1" name="$2" command="$3" schedule_expression="$4" source="$5" enabled="$6" next_run_at="$7" TASKS_JSON=$(jq -c \ --arg schedule_type "$schedule_type" \ --arg name "$name" \ --arg command "$command" \ --arg schedule_expression "$schedule_expression" \ --arg source "$source" \ --argjson enabled "$enabled" \ --arg next_run_at "$next_run_at" \ '. + [{ schedule_type: $schedule_type, name: $name, command: $command, schedule_expression: $schedule_expression, source: $source, enabled: $enabled, next_run_at: $next_run_at }]' <<<"$TASKS_JSON") } parse_crontab_lines() { # Reads 5-field-schedule + command cron lines from stdin. # $1 = source label, $2 = "system" (7-field, includes a user column) or "user" (6-field) local source="$1" mode="$2" while IFS= read -r line; do line="${line%%$'\r'}" [[ -z "$line" ]] && continue [[ "$line" =~ ^[[:space:]]*# ]] && continue [[ "$line" =~ ^[[:space:]]*[A-Za-z_][A-Za-z0-9_]*= ]] && continue if [[ "$mode" == "system" ]]; then # min hour dom mon dow user command... if [[ "$line" =~ ^[[:space:]]*([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+[^[:space:]]+[[:space:]]+(.+)$ ]]; then local sched="${BASH_REMATCH[1]} ${BASH_REMATCH[2]} ${BASH_REMATCH[3]} ${BASH_REMATCH[4]} ${BASH_REMATCH[5]}" local cmd="${BASH_REMATCH[6]}" add_task "cron" "$cmd" "$cmd" "$sched" "$source" true fi else # min hour dom mon dow command... if [[ "$line" =~ ^[[:space:]]*([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+(.+)$ ]]; then local sched="${BASH_REMATCH[1]} ${BASH_REMATCH[2]} ${BASH_REMATCH[3]} ${BASH_REMATCH[4]} ${BASH_REMATCH[5]}" local cmd="${BASH_REMATCH[6]}" add_task "cron" "$cmd" "$cmd" "$sched" "$source" true fi fi done } collect_cron() { [[ -r /etc/crontab ]] && parse_crontab_lines "/etc/crontab" "system" < /etc/crontab if [[ -d /etc/cron.d ]]; then for f in /etc/cron.d/*; do [[ -f "$f" && -r "$f" ]] || continue parse_crontab_lines "$f" "system" < "$f" done fi if command -v crontab >/dev/null 2>&1 && [[ -r /etc/passwd ]]; then while IFS=: read -r user _ uid _ _ _ shell; do case "$shell" in */nologin|*/false|"") continue ;; esac [[ "$uid" -lt 1000 && "$uid" != "0" ]] && continue local_crontab=$(crontab -l -u "$user" 2>/dev/null || true) [[ -z "$local_crontab" ]] && continue parse_crontab_lines "crontab:$user" "user" <<<"$local_crontab" done < /etc/passwd fi } collect_systemd_timers() { command -v systemctl >/dev/null 2>&1 || return 0 local timers_json timers_json=$(systemctl list-timers --all --output=json 2>/dev/null || echo "[]") while IFS= read -r entry; do local unit activates next_usec enabled_state schedule_expr next_run_at unit=$(jq -r '.unit' <<<"$entry") activates=$(jq -r '.activates // ""' <<<"$entry") next_usec=$(jq -r '.next // 0' <<<"$entry") enabled_state=$(systemctl is-enabled "$unit" 2>/dev/null || echo "unknown") local enabled_bool="false" [[ "$enabled_state" == "enabled" || "$enabled_state" == "static" ]] && enabled_bool="true" schedule_expr=$(systemctl show "$unit" -p TimersCalendar --value 2>/dev/null | sed -n 's/.*OnCalendar=\([^;]*\);.*/\1/p' | sed 's/[[:space:]]*$//') [[ -z "$schedule_expr" ]] && schedule_expr="(see: systemctl status $unit)" next_run_at="" if [[ "$next_usec" =~ ^[0-9]+$ && "$next_usec" -gt 0 ]]; then next_run_at=$(date -u -d "@$((next_usec / 1000000))" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || echo "") fi if [[ -n "$next_run_at" ]]; then add_task_with_next_run "systemd_timer" "$unit" "$activates" "$schedule_expr" "$unit" "$enabled_bool" "$next_run_at" else add_task "systemd_timer" "$unit" "$activates" "$schedule_expr" "$unit" "$enabled_bool" fi done < <(jq -c '.[]' <<<"$timers_json") } # Updates waiting to be installed, as JSON in UPDATES_JSON ("null" when this host's package manager isn't one we know, or couldn't answer). # Nothing is installed or refreshed: this reads what the package manager already knows about, so it is quick and changes nothing. collect_updates() { UPDATES_JSON="null" local manager="" count="" security="null" reboot="false" out="" status=0 if command -v apt-get >/dev/null 2>&1; then manager="apt" # -s simulates: "Inst" lines are what an upgrade would install. Security ones come from a "-security" or "Security" archive. if out=$(LC_ALL=C timeout 60 apt-get -s -o Debug::NoLocking=true upgrade 2>"$PM_ERR"); then count=$(grep -c '^Inst ' <<<"$out" || true) security=$(grep '^Inst ' <<<"$out" | grep -ci 'security' || true) fi [[ -f /var/run/reboot-required ]] && reboot="true" elif command -v dnf >/dev/null 2>&1; then manager="dnf" # check-update exits 100 when there are updates, 0 when none, 1 on an error. --cacheonly: never goes to the network from here. out=$(LC_ALL=C timeout 120 dnf -q check-update --cacheonly 2>"$PM_ERR") || status=$? if [[ "$status" -eq 0 || "$status" -eq 100 ]]; then count=$(awk 'NF >= 3 && $1 ~ /^[A-Za-z0-9_.+-]+\.[A-Za-z0-9_]+$/ && /Obsoleting/ == 0 { n++ } /^Obsoleting/ { exit } END { print n + 0 }' <<<"$out") security=$(LC_ALL=C timeout 120 dnf -q updateinfo list --security --cacheonly 2>"$PM_ERR" | grep -c '/Sec\.' || true) fi if command -v needs-restarting >/dev/null 2>&1; then needs-restarting -r >/dev/null 2>&1 || reboot="true" fi elif command -v pacman >/dev/null 2>&1; then manager="pacman" # -Qu is what the local copy of the sync databases says could be upgraded (what `checkupdates` lists, without changing anything). It # exits 1 when there is nothing to upgrade, so an empty answer with status 1 is "none", not an error. Packages held back are left out. out=$(LC_ALL=C timeout 60 pacman -Qu 2>"$PM_ERR") || status=$? if [[ "$status" -eq 0 || ( "$status" -eq 1 && -z "$out" ) ]]; then count=$(grep -v '\[ignored\]' <<<"$out" | grep -c . || true) fi # No security figure (pacman doesn't say). A restart is due when the kernel running is no longer the one installed. [[ -d "/usr/lib/modules/$(uname -r)" ]] || reboot="true" elif command -v zypper >/dev/null 2>&1; then manager="zypper" if out=$(LC_ALL=C timeout 120 zypper -q --no-refresh list-updates 2>"$PM_ERR"); then count=$(grep -c '^v |' <<<"$out" || true) fi [[ -f /run/reboot-needed ]] && reboot="true" fi if [[ -z "$manager" ]]; then diag updates unavailable no_package_manager elif [[ ! "$count" =~ ^[0-9]+$ ]]; then # In the journal (systemctl status / journalctl -u homelab-manager-agent) so "not reported" has a reason to look for. echo "Note: couldn't read the pending updates from $manager - the report goes without them." >&2 diag updates failed error "$manager did not answer" fi if [[ -n "$manager" && "$count" =~ ^[0-9]+$ ]]; then [[ "$security" =~ ^[0-9]+$ ]] || security="null" UPDATES_JSON=$(jq -n --arg manager "$manager" --argjson count "$count" --argjson security "$security" --argjson reboot "$reboot" \ '{manager: $manager, count: $count, security: $security, reboot_required: $reboot}') fi } # >>> post_report # Sends $PAYLOAD to the server and prints the HTTP status code. # # Nothing sensitive goes on curl's command line: any user on this machine can read other processes' command lines, and the # token would be there for the length of every report. So the token is handed to curl on its standard input, as a line of # curl's own config format, and the payload and the server's reply go through files in a private directory (mktemp -d is # readable by root only). A fixed name in /tmp is no good either — another user could plant a symlink there and have this # script, running as root, overwrite whatever it points at. curl_quote() { local s=$1 s=${s//\\/\\\\} s=${s//\"/\\\"} printf '%s' "$s" } post_report() { case "$API_TOKEN" in *$'\n'*|*$'\r'*) echo "API_TOKEN must not contain a line break." >&2; return 1 ;; esac printf '%s' "$PAYLOAD" > "$HLM_TMPDIR/payload.json" printf 'header = "Authorization: Bearer %s"\n' "$(curl_quote "$API_TOKEN")" | curl -sS "${CURL_INSECURE_FLAG[@]}" -K - \ -o "$HLM_TMPDIR/response.json" -w "%{http_code}" \ -X POST "$API_URL/api/agent/report" \ -H "Content-Type: application/json" \ --data-binary @"$HLM_TMPDIR/payload.json" } # <<< post_report # A GET with the token handed to curl on its standard input (as the upload does), the reply into the file named; prints the HTTP status. api_get() { printf 'header = "Authorization: Bearer %s"\n' "$(curl_quote "$API_TOKEN")" | curl -sS "${CURL_INSECURE_FLAG[@]}" -K - -m 30 -o "$2" -w "%{http_code}" "$API_URL$1" } # One value from the firmware's table (the kernel shows it under /sys/class/dmi/id), trying the named files in turn and leaving out what a # maker writes when it has nothing to say ("To Be Filled By O.E.M.", "Default string", all zeros ...). DMI_DIR is for the tests. dmi_value() { local name value for name in "$@"; do # (the file is checked first: a failing redirection prints its complaint before any 2>/dev/null applies) [[ -r "${DMI_DIR:-/sys/class/dmi/id}/$name" ]] || continue value=$(tr -d '\0' < "${DMI_DIR:-/sys/class/dmi/id}/$name" 2>/dev/null | head -c 200 | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') || value="" [[ -n "$value" ]] || continue case "${value,,}" in "to be filled by o.e.m."|"default string"|"system serial number"|"system product name"|"system manufacturer"|"not specified"|"not applicable"|"none"|"n/a"|"unknown"|"o.e.m."|"oem") continue ;; esac [[ "$value" =~ ^0+$ ]] && continue printf '%s' "$value" return 0 done return 0 } # What the machine is: maker, model and serial number (a serial is only readable by root). "null" when the firmware has nothing to say, as on # many virtual machines and single-board computers. collect_hardware() { HARDWARE_JSON="null" local maker model serial maker=$(dmi_value sys_vendor board_vendor) model=$(dmi_value product_name board_name) serial=$(dmi_value product_serial board_serial) if [[ -z "$maker$model$serial" ]]; then diag hardware unavailable not_found return 0 fi HARDWARE_JSON=$(jq -n -c --arg maker "$maker" --arg model "$model" --arg serial "$serial" \ '{} | (if $maker != "" then . + {manufacturer: $maker} else . end) | (if $model != "" then . + {model: $model} else . end) | (if $serial != "" then . + {serial: $serial} else . end)') || HARDWARE_JSON="null" } # Services (systemd units) that have failed. systemctl is asked the way the timers are, so this works inside the sandbox. collect_failed_units() { FAILED_UNITS_JSON="null" if ! command -v systemctl >/dev/null 2>&1; then diag failed_units unavailable no_systemd return 0 fi local out if ! out=$(timeout 20 systemctl list-units --state=failed --plain --no-legend --no-pager --full 2>"$PM_ERR"); then diag failed_units failed error "systemctl list-units did not answer" return 0 fi FAILED_UNITS_JSON=$(awk 'NF > 0 { unit = $1; $1 = $2 = $3 = $4 = ""; sub(/^ +/, ""); print unit "\t" $0 }' <<<"$out" \ | jq -R -s -c 'split("\n") | map(select(length > 0) | split("\t")) | map({unit: .[0], description: (.[1] // "")}) | .[0:100]') || FAILED_UNITS_JSON="null" } # Each disk's own SMART verdict, temperature and (where it says) how much of its rated life is used. Needs the devices, so it runs from # --prepare. Asked at most about once an hour, and with -n standby so a sleeping disk is not woken: what was last known of it is kept. collect_smart() { SMART_JSON="null" if ! command -v smartctl >/dev/null 2>&1; then diag smart unavailable tool_missing smartctl return 0 fi if [[ "$(id -u)" != "0" ]]; then diag smart unavailable not_root return 0 fi local cache="$STATE_DIR/smart.json" now_s previous="[]" items="[]" dev out item scan checked now_s=$(date +%s) if [[ -r "$cache" ]]; then checked=$(jq -r '.checked_at // 0' "$cache" 2>/dev/null || echo 0) previous=$(jq -c '.devices // []' "$cache" 2>/dev/null || echo "[]") if [[ "$checked" =~ ^[0-9]+$ ]] && (( now_s - checked < 3300 )) && [[ "$MODE" != "check" ]]; then SMART_JSON=$previous return 0 fi fi scan=$(timeout 30 smartctl --scan -j 2>"$PM_ERR") || true while IFS= read -r dev; do [[ -n "$dev" ]] || continue # smartctl's exit status is a bit mask that says "something was noted" more often than "it failed", so it is read from the JSON instead. out=$(timeout 30 smartctl -j -H -A -n standby "$dev" 2>"$PM_ERR") || true item=$(jq -c --arg dev "$dev" 'select(.smart_status != null) | { device: $dev, model: (.model_name // .scsi_model_name // ""), healthy: .smart_status.passed, temperature_c: (.temperature.current // null), wear_percent: (.nvme_smart_health_information_log.percentage_used // ([.ata_smart_attributes.table[]? | select(.id == 177 or .id == 231 or .id == 233) | (100 - .value)] | first) // null) }' <<<"$out" 2>/dev/null) || item="" if [[ -z "$item" ]]; then # Asleep, or no SMART at all (a virtual disk): keep what was known of it, if anything. item=$(jq -c --arg dev "$dev" '.[] | select(.device == $dev)' <<<"$previous" 2>/dev/null | head -n 1) || item="" fi if [[ -n "$item" ]]; then items=$(jq -c --argjson i "$item" '. + [$i]' <<<"$items"); fi done < <(jq -r '.devices[]?.name' <<<"$scan" 2>/dev/null) SMART_JSON=$items if mkdir -p "$STATE_DIR" 2>/dev/null; then chmod 700 "$STATE_DIR" 2>/dev/null || true jq -c -n --argjson at "$now_s" --argjson devices "$items" '{checked_at: $at, devices: $devices}' > "$cache.tmp" 2>/dev/null && mv -f "$cache.tmp" "$cache" 2>/dev/null || true fi } # Each ZFS pool and whether it is ONLINE. Needs /dev/zfs, so it runs from --prepare too. collect_zfs() { ZFS_JSON="null" if ! command -v zpool >/dev/null 2>&1; then diag zfs unavailable tool_missing zpool return 0 fi local out if ! out=$(timeout 20 zpool list -H -o name,health,capacity 2>"$PM_ERR"); then diag zfs failed error "zpool list did not answer" return 0 fi ZFS_JSON=$(jq -R -s -c 'split("\n") | map(select(length > 0) | split("\t")) | map({name: .[0], health: (.[1] // "UNKNOWN"), capacity_percent: ((.[2] // "" | rtrimstr("%") | tonumber?) // null)})' <<<"$out") || ZFS_JSON="null" } # The disk and pool readings --prepare left for this report, if they are there and recent. Re-shaped by jq, never trusted as they stand. use_prepared_health() { [[ -r "$HEALTH_FILE" ]] || return 1 [[ -n "$(find "$HEALTH_FILE" -mmin -10 -print 2>/dev/null)" ]] || return 1 SMART_JSON=$(jq -c 'if (.smart | type) == "array" then [.smart[] | {device: (.device | tostring), model: ((.model // "") | tostring), healthy: (if (.healthy | type) == "boolean" then .healthy else null end), temperature_c: (if (.temperature_c | type) == "number" then .temperature_c else null end), wear_percent: (if (.wear_percent | type) == "number" then .wear_percent else null end)}] else null end' "$HEALTH_FILE" 2>/dev/null) || SMART_JSON="null" ZFS_JSON=$(jq -c 'if (.zfs | type) == "array" then [.zfs[] | {name: (.name | tostring), health: ((.health // "UNKNOWN") | tostring), capacity_percent: (if (.capacity_percent | type) == "number" then .capacity_percent else null end)}] else null end' "$HEALTH_FILE" 2>/dev/null) || ZFS_JSON="null" return 0 } # Replaces this script with the one this Homelab Manager serves, when an administrator has switched automatic updates on and this agent is # behind. Nothing is trusted but the checksum: the (authenticated) answer gives the SHA-256 of the script, and a download that doesn't # match it, doesn't look like this script, doesn't pass bash's syntax check or isn't newer is thrown away. The new file is put beside the # old and moved over it in one step, and the new version then carries on from here. Runs only from --prepare (outside the sandbox, which # keeps /usr read-only for the report itself); with --check-updates it says what it would do. self_update() { [[ -n "$API_URL" && -n "$API_TOKEN" ]] || return 0 [[ -z "${HLM_JUST_UPDATED:-}" ]] || return 0 local self work code reply_update version sha got self=$(readlink -f "${BASH_SOURCE[0]}" 2>/dev/null) || return 0 [[ -f "$self" ]] || return 0 work=$(mktemp -d) || return 0 code=$(api_get "/api/agent/report/update?version=$AGENT_VERSION" "$work/reply.json" 2>/dev/null) || code="" reply_update=$(jq -r '.update // false' "$work/reply.json" 2>/dev/null || echo false) if [[ "$code" != "200" || "$reply_update" != "true" ]]; then [[ "$MODE" == "check" ]] && echo "Automatic update: nothing to do (it is off, or this agent is current)." rm -rf "$work" return 0 fi version=$(jq -r '.version // empty' "$work/reply.json" 2>/dev/null || true) sha=$(jq -r '.sha256 // empty' "$work/reply.json" 2>/dev/null || true) if [[ ! "$version" =~ ^[0-9]+$ || ! "$sha" =~ ^[0-9a-f]{64}$ ]] || (( version <= AGENT_VERSION )); then rm -rf "$work" return 0 fi if ! curl -fsS "${CURL_INSECURE_FLAG[@]}" -m 60 -o "$work/new.sh" "$API_URL/agent/linux/report-tasks.sh" 2>"$PM_ERR"; then echo "Note: couldn't download the newer agent (version $version), so it was not installed." >&2 rm -rf "$work" return 0 fi got=$(sha256sum "$work/new.sh" 2>/dev/null | awk '{print $1}') if [[ "$got" != "$sha" ]] || [[ "$(head -n 1 "$work/new.sh")" != "#!/usr/bin/env bash" ]] || ! bash -n "$work/new.sh" 2>/dev/null || ! grep -q "^AGENT_VERSION=$version\$" "$work/new.sh"; then echo "Note: the newer agent (version $version) didn't check out against what the server said, so it was not installed." >&2 rm -rf "$work" return 0 fi if [[ "$MODE" == "check" ]]; then echo "Automatic update: version $version is available and would be installed on the next run." rm -rf "$work" return 0 fi if install -m 755 "$work/new.sh" "$self.new" && mv -f "$self.new" "$self"; then echo "Updated the agent from version $AGENT_VERSION to version $version." >&2 rm -rf "$work" HLM_JUST_UPDATED=1 exec "$self" --prepare fi rm -f "$self.new" echo "Note: couldn't replace the agent with version $version." >&2 rm -rf "$work" return 0 } # The answer --prepare left for this report, if it is there, recent and looks like what it should. Read back through jq rather than # trusted: the report only ever carries these four fields, whatever the file holds. use_prepared_updates() { [[ -r "$UPDATES_FILE" ]] || return 1 [[ -n "$(find "$UPDATES_FILE" -mmin -10 -print 2>/dev/null)" ]] || return 1 local json json=$(jq -ce 'select(type == "object" and (.manager | type) == "string" and (.count | type) == "number") | {manager: .manager, count: .count, security: (if (.security | type) == "number" then .security else null end), reboot_required: (.reboot_required == true)}' "$UPDATES_FILE" 2>/dev/null) || return 1 [[ -n "$json" ]] || return 1 UPDATES_JSON=$json } # What this report could not collect: the notes made here, then those --prepare left (it does the outside-the-sandbox part), one for each check. # With nothing left by --prepare — on a real run, not a dry one — that step did not run, which is worth saying: disk health and ZFS are only read there. collect_diagnostics() { local from_prepare="[]" shaped if [[ -r "$DIAG_FILE" && -n "$(find "$DIAG_FILE" -mmin -10 -print 2>/dev/null)" ]]; then from_prepare=$(jq -c 'if type == "array" then [.[] | select(type == "object" and (.check | type) == "string" and (.status | type) == "string") | {check: (.check | tostring), status: (.status | tostring)} + (if ((.reason // "") | tostring) != "" then {reason: (.reason | tostring)} else {} end) + (if ((.detail // "") | tostring) != "" then {detail: (.detail | tostring | .[0:160])} else {} end)] | .[0:20] else [] end' "$DIAG_FILE" 2>/dev/null) || from_prepare="[]" elif [[ "$DRY_RUN" != "1" ]]; then diag prepare unavailable step_missing fi shaped=$(jq -c -n --argjson here "$(diag_json)" --argjson there "$from_prepare" '($here + $there) | unique_by(.check) | .[0:20]') || shaped="[]" DIAGNOSTICS_JSON=$shaped } collect_system_info() { local ip_json="[]" if command -v ip >/dev/null 2>&1; then ip_json=$(ip -4 -o addr show scope global 2>/dev/null \ | awk '{print $4}' | cut -d/ -f1 \ | jq -R -s -c 'split("\n") | map(select(length > 0))') fi local cpu_model="" cpu_cores=0 cpu_load_percent="null" # x86 /proc/cpuinfo has a per-core "model name" line. Most 64-bit ARM # kernels (Raspberry Pi included) don't — they instead have a single # "Model" line at the very end, or nothing at all, in which case # /proc/device-tree/model (present on any device-tree/SBC board) has the # board's friendly name. Try each in turn; leave blank if none exist. cpu_model=$(grep -m1 "^model name" /proc/cpuinfo 2>/dev/null | cut -d: -f2- | sed 's/^ *//' || true) if [[ -z "$cpu_model" ]]; then cpu_model=$(grep -m1 "^Model" /proc/cpuinfo 2>/dev/null | cut -d: -f2- | sed 's/^ *//' || true) fi if [[ -z "$cpu_model" && -r /proc/device-tree/model ]]; then cpu_model=$(tr -d '\0' < /proc/device-tree/model 2>/dev/null || true) fi cpu_cores=$(nproc 2>/dev/null || echo 0) if [[ -r /proc/loadavg && "$cpu_cores" -gt 0 ]]; then local load1 load1=$(awk '{print $1}' /proc/loadavg) cpu_load_percent=$(awk -v l="$load1" -v c="$cpu_cores" 'BEGIN { printf "%.1f", (l/c)*100 }') fi local mem_total=0 mem_used=0 if command -v free >/dev/null 2>&1; then read -r mem_total mem_used < <(free -b | awk '/^Mem:/ {print $2, $3}') fi local disks_json="[]" if command -v df >/dev/null 2>&1; then disks_json=$(df -B1 --output=target,size,used -x tmpfs -x devtmpfs -x squashfs -x overlay 2>/dev/null \ | tail -n +2 \ | awk '{print $1"\t"$2"\t"$3}' \ | jq -R -s -c ' split("\n") | map(select(length > 0) | split("\t")) | map({mount: .[0], size_bytes: (.[1]|tonumber), used_bytes: (.[2]|tonumber)}) ') fi # Everything bound to a port on this host, including services listening on localhost only (which a network # scan from elsewhere can't see). `ss -p` needs root to name the process; without it the process is blank. # One row per socket — the server groups them per port. local ports_json="[]" if command -v ss >/dev/null 2>&1; then ports_json=$(ss -H -tulnp 2>/dev/null \ | awk '{ local = $5; port = local; sub(/.*:/, "", port); addr = local; sub(/:[0-9]+$/, "", addr); proc = ""; if (match($0, /users:\(\("[^"]+"/)) { proc = substr($0, RSTART + 9, RLENGTH - 10) } if (port ~ /^[0-9]+$/) print $1 "\t" port "\t" addr "\t" proc }' \ | jq -R -s -c ' split("\n") | map(select(length > 0) | split("\t")) | map({protocol: .[0], port: (.[1]|tonumber), address: .[2], process: (.[3] // "")}) ') fi # Best-effort like the rest: a package manager that misbehaves must not cost the report. UPDATES_JSON="null" use_prepared_updates || collect_updates || UPDATES_JSON="null" # Failed services are asked here (systemctl works inside the sandbox); disk and pool health come from --prepare. A person running --dry-run # isn't in the sandbox, so there they are read directly. FAILED_UNITS_JSON="null" collect_failed_units || FAILED_UNITS_JSON="null" SMART_JSON="null" ZFS_JSON="null" if ! use_prepared_health && [[ "$DRY_RUN" == "1" ]]; then collect_smart || SMART_JSON="null" collect_zfs || ZFS_JSON="null" fi HARDWARE_JSON="null" collect_hardware || HARDWARE_JSON="null" collect_diagnostics HEALTH_JSON=$(jq -n -c --argjson failed "$FAILED_UNITS_JSON" --argjson smart "$SMART_JSON" --argjson zfs "$ZFS_JSON" \ '{} | (if $failed != null then . + {failed_units: $failed} else . end) | (if $smart != null then . + {smart: $smart} else . end) | (if $zfs != null then . + {zfs: $zfs} else . end)') || HEALTH_JSON="{}" [[ "$HEALTH_JSON" == "{}" ]] && HEALTH_JSON="null" SYSTEM_JSON=$(jq -n \ --argjson updates "$UPDATES_JSON" \ --argjson health "$HEALTH_JSON" \ --argjson hardware "$HARDWARE_JSON" \ --argjson diagnostics "$DIAGNOSTICS_JSON" \ --argjson ip_addresses "$ip_json" \ --argjson listening_ports "$ports_json" \ --arg cpu_model "$cpu_model" \ --argjson cpu_cores "${cpu_cores:-0}" \ --argjson cpu_load_percent "$cpu_load_percent" \ --argjson mem_total_bytes "${mem_total:-0}" \ --argjson mem_used_bytes "${mem_used:-0}" \ --argjson disks "$disks_json" \ '{ ip_addresses: $ip_addresses, cpu: { model: $cpu_model, cores: $cpu_cores, load_percent: $cpu_load_percent }, memory: { total_bytes: $mem_total_bytes, used_bytes: $mem_used_bytes }, disks: $disks, listening_ports: $listening_ports, updates: $updates, health: $health, hardware: $hardware, diagnostics: $diagnostics }') } if [[ "$MODE" != "report" ]]; then # prepare: first see whether this script is to be replaced (if so it carries on as the new one, from the top). [[ "$MODE" == "prepare" ]] && self_update collect_updates || UPDATES_JSON="null" collect_smart || SMART_JSON="null" collect_zfs || ZFS_JSON="null" if [[ "$MODE" == "check" ]]; then collect_failed_units || FAILED_UNITS_JSON="null" echo "Pending updates as they would be reported:" jq . <<<"$UPDATES_JSON" echo "Failed services:" jq . <<<"$FAILED_UNITS_JSON" echo "Disk health (SMART; null means smartctl isn't installed or this isn't root):" jq . <<<"$SMART_JSON" echo "ZFS pools (null means there is no zpool here):" jq . <<<"$ZFS_JSON" collect_hardware || true echo "What could not be collected (empty means nothing was missed):" diag_json | jq . self_update exit 0 fi # Leave the answers (or no file at all) for the report that follows. Each is written whole and moved into place, so a report never reads half of one. rm -f "$UPDATES_FILE" "$HEALTH_FILE" "$DIAG_FILE" mkdir -p "$(dirname "$UPDATES_FILE")" 2>/dev/null || exit 0 # The notes always go, even when there are none: a missing file is how the report knows this step didn't run. diag_json > "$DIAG_FILE.tmp" 2>/dev/null && mv -f "$DIAG_FILE.tmp" "$DIAG_FILE" 2>/dev/null || true if [[ "$UPDATES_JSON" != "null" ]]; then printf '%s\n' "$UPDATES_JSON" > "$UPDATES_FILE.tmp" 2>/dev/null && mv -f "$UPDATES_FILE.tmp" "$UPDATES_FILE" 2>/dev/null || true fi HEALTH_FILE_JSON=$(jq -n -c --argjson smart "${SMART_JSON:-null}" --argjson zfs "${ZFS_JSON:-null}" '{} | (if $smart != null then . + {smart: $smart} else . end) | (if $zfs != null then . + {zfs: $zfs} else . end)' 2>/dev/null || echo "{}") if [[ "$HEALTH_FILE_JSON" != "{}" ]]; then printf '%s\n' "$HEALTH_FILE_JSON" > "$HEALTH_FILE.tmp" 2>/dev/null && mv -f "$HEALTH_FILE.tmp" "$HEALTH_FILE" 2>/dev/null || true fi exit 0 fi collect_cron collect_systemd_timers # Hardware/network facts are best-effort: a quirk on any given host (e.g. no # "model name" line in /proc/cpuinfo on some ARM boards) must never abort the # whole report over it, so errexit is relaxed just for this call. SYSTEM_JSON="null" set +e collect_system_info system_info_status=$? set -e if [[ "$system_info_status" -ne 0 ]]; then echo "Warning: collecting hardware/network info failed (exit $system_info_status) — reporting tasks without it." >&2 SYSTEM_JSON="null" fi HOSTNAME_VALUE=$(hostname -f 2>/dev/null || hostname) PAYLOAD=$(jq -n \ --arg hostname "$HOSTNAME_VALUE" \ --arg os_type "linux" \ --arg reported_at "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ --argjson agent_version "$AGENT_VERSION" \ --argjson system "$SYSTEM_JSON" \ --argjson tasks "$TASKS_JSON" \ '{hostname: $hostname, os_type: $os_type, reported_at: $reported_at, agent_version: $agent_version, system: $system, tasks: $tasks}') if [[ "$DRY_RUN" == "1" ]]; then echo "$PAYLOAD" | jq . exit 0 fi HLM_TMPDIR=$(mktemp -d) trap 'rm -rf "$HLM_TMPDIR"' EXIT response=$(post_report) if [[ "$response" -lt 200 || "$response" -ge 300 ]]; then echo "Report failed with HTTP $response:" >&2 cat "$HLM_TMPDIR/response.json" >&2 exit 1 fi echo "Reported $(jq 'length' <<<"$TASKS_JSON") task(s) successfully."