import { and, eq, inArray } from "drizzle-orm"; import { db } from "../db/client.js"; import { integrations } from "../db/schema.js"; import { loadIntegrationConfig } from "../integrations/loadIntegration.js"; import { createSemaphoreAdapter } from "../integrations/semaphore/adapter.js"; import { createGiteaAdapter } from "../integrations/gitea/adapter.js"; import { diffConditions, type ActiveState, type HealthCondition } from "./healthMonitor.js"; import { activeSubjects } from "./maintenance.js"; import { notifyAutomationFailed, notifyAutomationRecovered } from "./notify.js"; import { getInternalFlag, setInternalFlag } from "./settingsStore.js"; const STATE_FLAG = "automationActiveConditions"; /** * What a piece of automation's most recent run tells us. "unknown" covers everything that is neither a clear * pass nor a clear failure — still running, waiting, cancelled, skipped, stopped by hand — and means "don't * change what we were saying": a run in progress must not clear a failure it hasn't yet fixed, and a manual * stop isn't a failure. */ export type RunOutcome = "failed" | "passed" | "unknown"; export interface AutomationItem { /** Stable identity: the same template/repo always has the same key. */ key: string; /** Hierarchical source, "semaphore:3:12:45" — held when the run's outcome isn't known. */ source: string; outcome: RunOutcome; /** What to say when it has failed. */ message: string; } // ─── Classification (pure) ────────────────────────────────────────────────── export function classifySemaphoreStatus(status: string | null | undefined): RunOutcome { if (status === "error") return "failed"; if (status === "success") return "passed"; return "unknown"; // waiting, starting, running, stopping, stopped, rejected, confirmed, ... } export function classifyGiteaRun(run: { status: string; conclusion: string | null }): RunOutcome { if (run.conclusion === "failure" || run.status === "failure") return "failed"; if (run.conclusion === "success" || run.status === "success") return "passed"; return "unknown"; // running, waiting, blocked, cancelled, skipped, ... } function endedSuffix(end: string | null): string { if (!end) return ""; const t = Date.parse(end); return Number.isFinite(t) ? ` (${new Date(t).toISOString().slice(0, 16).replace("T", " ")} UTC)` : ""; } /** Splits the items into the failures to report and the sources whose state must be left alone this pass. */ export function evaluateAutomation(items: AutomationItem[]): { conditions: HealthCondition[]; held: Set } { const conditions: HealthCondition[] = []; const held = new Set(); for (const item of items) { if (item.outcome === "failed") conditions.push({ key: item.key, source: item.source, message: item.message }); else if (item.outcome === "unknown") held.add(item.source); } return { conditions, held }; } // ─── Collection (I/O) ─────────────────────────────────────────────────────── export async function collectAutomation(): Promise<{ items: AutomationItem[]; held: Set; readable: number }> { const items: AutomationItem[] = []; const held = new Set(); let readable = 0; const rows = await db .select({ id: integrations.id, name: integrations.name, type: integrations.type }) .from(integrations) .where(and(eq(integrations.enabled, true), inArray(integrations.type, ["semaphore", "gitea"]))); for (const row of rows) { try { const loaded = await loadIntegrationConfig(row.id); if (!loaded) continue; if (row.type === "semaphore") { const { templates, failedProjectIds } = await createSemaphoreAdapter(loaded.config as any).checkTemplates(); // A project that couldn't be listed contributes no templates — that's "couldn't read", not "all fixed". for (const pid of failedProjectIds) held.add(`semaphore:${row.id}:${pid}`); for (const t of templates) { if (!t.lastTask) continue; items.push({ key: `automation:semaphore:${row.id}:${t.projectId}:${t.id}`, source: `semaphore:${row.id}:${t.projectId}:${t.id}`, outcome: classifySemaphoreStatus(t.lastTask.status), message: `${row.name} / ${t.projectName} / ${t.name}: run #${t.lastTask.id} failed${endedSuffix(t.lastTask.end)}`, }); } } else { const repos = await createGiteaAdapter(loaded.config as any).listReposWithStatus(); for (const r of repos) { if (!r.hasActions) continue; const source = `gitea:${row.id}:${r.fullName}`; // The adapter reports a run it couldn't fetch as null, the same as "no runs yet" — either way there's nothing to judge. if (!r.latestRun) { held.add(source); continue; } const run = r.latestRun; items.push({ key: `automation:gitea:${row.id}:${r.fullName}`, source, outcome: classifyGiteaRun(run), message: `${row.name} / ${r.fullName}: "${run.displayTitle || "workflow"}" (run #${run.runNumber}) failed${run.headBranch ? ` on ${run.headBranch}` : ""}${run.htmlUrl ? ` — ${run.htmlUrl}` : ""}`, }); } } readable++; } catch (err) { console.error(`[automation] couldn't read ${row.type} integration ${row.id}:`, err instanceof Error ? err.message : err); held.add(`${row.type}:${row.id}`); } } return { items, held, readable }; } // ─── The scheduled pass ───────────────────────────────────────────────────── async function loadState(): Promise { const raw = await getInternalFlag(STATE_FLAG); if (raw === null) return null; try { return JSON.parse(raw); } catch { return {}; } } /** * Reports each template/repo whose latest run failed, once, and again when it succeeds. State-based like the * health monitor: a template that keeps failing every night alerts on the first failure, not every night. * * The very first pass only records what is already failing, without announcing it — on a fresh install (or * the first time this feature runs) that would otherwise be a wall of alerts about failures that are months old. */ export async function runAutomationCheck(now: number = Date.now()): Promise<{ added: number; resolved: number; baseline: boolean }> { const { items, held: readHeld, readable } = await collectAutomation(); const stored = await loadState(); // Nothing could be read at all (or nothing is configured): don't spend the "first pass" on an empty picture. if (stored === null && readable === 0) return { added: 0, resolved: 0, baseline: false }; const { conditions, held: unknownHeld } = evaluateAutomation(items); const held = new Set([...readHeld, ...unknownHeld]); const { added, resolved, next } = diffConditions(stored ?? {}, conditions, held, await activeSubjects(new Date(now))); await setInternalFlag(STATE_FLAG, JSON.stringify(next)); if (stored === null) return { added: 0, resolved: 0, baseline: true }; // State is tracked even with the alert toggle off (the notify functions check it), so turning it back on isn't a flood. if (added.length > 0) await notifyAutomationFailed(added); if (resolved.length > 0) await notifyAutomationRecovered(resolved); return { added: added.length, resolved: resolved.length, baseline: false }; }