fix: improve the workflow ai

.github/workflows/lint.yml

@@ -12,5 +12,6 @@ jobs:
       - uses: actions/checkout@v4
       - uses: oven-sh/setup-bun@v2
       - run: bun install --frozen-lockfile
+      - run: bun run test
       - run: bun run lint
       - run: bun run format:check

migrations/0010_slippery_mattie_franklin.sql

@@ -0,0 +1,25 @@
+CREATE TABLE `ai_events` (
+	`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+	`kind` text NOT NULL,
+	`incident_id` integer,
+	`monitor_id` integer,
+	`model` text,
+	`outcome` text NOT NULL,
+	`reason` text,
+	`latency_ms` integer,
+	`prompt_tokens` integer,
+	`completion_tokens` integer,
+	`context_preview` text,
+	`output_preview` text,
+	`created_at` integer NOT NULL,
+	FOREIGN KEY (`incident_id`) REFERENCES `incidents`(`id`) ON UPDATE no action ON DELETE set null,
+	FOREIGN KEY (`monitor_id`) REFERENCES `monitors`(`id`) ON UPDATE no action ON DELETE set null
+);
+--> statement-breakpoint
+CREATE INDEX `ai_events_created_at_idx` ON `ai_events` (`created_at`);--> statement-breakpoint
+ALTER TABLE `ai_settings` ADD `autopilot_enabled` integer DEFAULT false NOT NULL;--> statement-breakpoint
+ALTER TABLE `ai_settings` ADD `autopilot_followup_minutes` integer DEFAULT 15 NOT NULL;--> statement-breakpoint
+ALTER TABLE `ai_settings` ADD `autopilot_max_updates` integer DEFAULT 6 NOT NULL;--> statement-breakpoint
+ALTER TABLE `ai_settings` ADD `autopilot_advance_status` integer DEFAULT false NOT NULL;--> statement-breakpoint
+ALTER TABLE `ai_settings` ADD `autopilot_degraded_incidents` integer DEFAULT false NOT NULL;--> statement-breakpoint
+ALTER TABLE `incidents` ADD `kind` text DEFAULT 'down' NOT NULL;

migrations/meta/_journal.json

@@ -71,6 +71,13 @@
       "when": 1788062139636,
       "tag": "0009_shocking_butterfly",
       "breakpoints": true
+    },
+    {
+      "idx": 10,
+      "version": "6",
+      "when": 1788183346274,
+      "tag": "0010_slippery_mattie_franklin",
+      "breakpoints": true
     }
   ]
 }

src/client/api/settings.ts

@@ -5,6 +5,11 @@ export type AiSettings = {
 	enabled: boolean;
 	baseUrl: string | null;
 	model: string | null;
+	autopilotEnabled: boolean;
+	autopilotFollowupMinutes: number;
+	autopilotMaxUpdates: number;
+	autopilotAdvanceStatus: boolean;
+	autopilotDegradedIncidents: boolean;
 	apiKeySet: boolean;
 	apiKeyPreview: string | null;
 	createdAt: string | null;
@@ -16,6 +21,35 @@ export type AiSettingsInput = {
 	baseUrl: string | null;
 	model: string | null;
 	apiKey?: string | null;
+	autopilotEnabled: boolean;
+	autopilotFollowupMinutes: number;
+	autopilotMaxUpdates: number;
+	autopilotAdvanceStatus: boolean;
+	autopilotDegradedIncidents: boolean;
+};
+
+export type AiEvent = {
+	id: number;
+	kind: string;
+	incidentId: number | null;
+	monitorId: number | null;
+	model: string | null;
+	outcome: string;
+	reason: string | null;
+	latencyMs: number | null;
+	promptTokens: number | null;
+	completionTokens: number | null;
+	contextPreview: string | null;
+	outputPreview: string | null;
+	createdAt: string;
+};
+
+export type AiEventSummary = {
+	total: number;
+	ok: number;
+	averageLatencyMs: number | null;
+	promptTokens: number;
+	completionTokens: number;
 };
 
 export function getAiSettings(signal?: AbortSignal) {
@@ -32,3 +66,10 @@ export function updateAiSettings(input: AiSettingsInput) {
 export function testAiSettings() {
 	return postJson<{ ok: true; message: string }>('/api/settings/ai/test');
 }
+
+export function getAiEvents(signal?: AbortSignal) {
+	return getJson<{ events: AiEvent[]; summary: AiEventSummary }>('/api/settings/ai/events?limit=50', {
+		signal,
+		credentials: 'same-origin',
+	});
+}

src/client/components/settings/AiActivityPanel.tsx

@@ -0,0 +1,66 @@
+import { Badge } from '@/components/ui/badge';
+import { Empty, EmptyDescription, EmptyTitle } from '@/components/ui/empty';
+import { useAiEventsQuery } from '../../queries/settings';
+
+function formatTokens(value: number) {
+	return new Intl.NumberFormat(undefined, { notation: value >= 10_000 ? 'compact' : 'standard' }).format(value);
+}
+
+export function AiActivityPanel() {
+	const query = useAiEventsQuery();
+	if (query.isPending) return <div className="table-empty">Loading AI activity…</div>;
+	if (query.isError) {
+		return (
+			<Empty variant="error" className="m-6">
+				<EmptyTitle>Unable to load AI activity</EmptyTitle>
+			</Empty>
+		);
+	}
+	const { events, summary } = query.data;
+	const tokenTotal = summary.promptTokens + summary.completionTokens;
+	return (
+		<div className="ai-activity-panel">
+			<div className="ai-activity-stats">
+				<div>
+					<strong>{summary.total ? Math.round((summary.ok / summary.total) * 100) : 0}%</strong>
+					<span>Success rate</span>
+				</div>
+				<div>
+					<strong>{summary.total}</strong>
+					<span>Calls · 7 days</span>
+				</div>
+				<div>
+					<strong>{summary.averageLatencyMs === null ? '—' : `${summary.averageLatencyMs} ms`}</strong>
+					<span>Average latency</span>
+				</div>
+				<div>
+					<strong>{formatTokens(tokenTotal)}</strong>
+					<span>Total tokens</span>
+				</div>
+			</div>
+			{events.length === 0 ? (
+				<Empty className="channel-empty">
+					<EmptyTitle>No AI activity yet</EmptyTitle>
+					<EmptyDescription>Attempts will appear here after AI generation or autopilot runs.</EmptyDescription>
+				</Empty>
+			) : (
+				<div className="ai-activity-history" aria-label="AI activity history">
+					{events.map((event) => (
+						<div className="ai-activity-row" key={event.id}>
+							<Badge variant={event.outcome === 'ok' ? 'online' : event.outcome.startsWith('skipped') ? 'pending' : 'offline'}>
+								{event.outcome.replaceAll('_', ' ')}
+							</Badge>
+							<strong>{event.kind.replaceAll('_', ' ')}</strong>
+							<span>{new Date(event.createdAt).toLocaleString()}</span>
+							<span>{event.reason ?? event.model ?? '—'}</span>
+							<small>
+								{event.latencyMs === null ? '—' : `${event.latencyMs} ms`} ·{' '}
+								{formatTokens((event.promptTokens ?? 0) + (event.completionTokens ?? 0))} tokens
+							</small>
+						</div>
+					))}
+				</div>
+			)}
+		</div>
+	);
+}

src/client/pages/SettingsPage.tsx

@@ -1,5 +1,5 @@
 import { type FormEvent, useState } from 'react';
-import { ArrowLeft, BellRing, Sparkles, Wrench } from 'lucide-react';
+import { Activity, ArrowLeft, BellRing, Sparkles, Wrench } from 'lucide-react';
 import { Button } from '@/components/ui/button';
 import { Card } from '@/components/ui/card';
 import { Empty, EmptyTitle } from '@/components/ui/empty';
@@ -7,6 +7,7 @@ import { Input } from '@/components/ui/input';
 import { Switch } from '@/components/ui/switch';
 import type { AiSettings } from '../api/settings';
 import { AppHeader } from '../components/AppHeader';
+import { AiActivityPanel } from '../components/settings/AiActivityPanel';
 import { MaintenanceWindowsPanel } from '../components/settings/MaintenanceWindowsPanel';
 import { NotificationChannelsPanel } from '../components/settings/NotificationChannelsPanel';
 import { navigate } from '../lib/router';
@@ -17,6 +18,11 @@ function AiSettingsForm({ settings }: { settings: AiSettings }) {
 	const [apiKey, setApiKey] = useState('');
 	const [model, setModel] = useState(settings.model ?? 'gpt-4o-mini');
 	const [enabled, setEnabled] = useState(settings.enabled);
+	const [autopilotEnabled, setAutopilotEnabled] = useState(settings.autopilotEnabled);
+	const [followupMinutes, setFollowupMinutes] = useState(settings.autopilotFollowupMinutes);
+	const [maxUpdates, setMaxUpdates] = useState(settings.autopilotMaxUpdates);
+	const [advanceStatus, setAdvanceStatus] = useState(settings.autopilotAdvanceStatus);
+	const [degradedIncidents, setDegradedIncidents] = useState(settings.autopilotDegradedIncidents);
 	const updateMutation = useUpdateAiSettingsMutation();
 	const testMutation = useTestAiSettingsMutation();
 
@@ -27,6 +33,11 @@ function AiSettingsForm({ settings }: { settings: AiSettings }) {
 			baseUrl: baseUrl.trim() || null,
 			model: model.trim() || null,
 			apiKey: apiKey.trim() || null,
+			autopilotEnabled,
+			autopilotFollowupMinutes: followupMinutes,
+			autopilotMaxUpdates: maxUpdates,
+			autopilotAdvanceStatus: advanceStatus,
+			autopilotDegradedIncidents: degradedIncidents,
 		});
 	}
 
@@ -65,6 +76,54 @@ function AiSettingsForm({ settings }: { settings: AiSettings }) {
 					<small>Generate one sanitized public update when an incident opens.</small>
 				</label>
 			</div>
+			<fieldset className="autopilot-settings">
+				<legend>Autopilot</legend>
+				<div className="settings-toggle">
+					<Switch id="autopilot-enabled" checked={autopilotEnabled} onCheckedChange={setAutopilotEnabled} />
+					<label htmlFor="autopilot-enabled">
+						<strong>Enable incident autopilot</strong>
+						<small>Write sanitized opening, follow-up, and resolution updates without sending extra alerts.</small>
+					</label>
+				</div>
+				<div className="autopilot-number-fields">
+					<label className="field" htmlFor="autopilot-cadence">
+						<span>Initial follow-up cadence (minutes)</span>
+						<Input
+							id="autopilot-cadence"
+							type="number"
+							min={5}
+							max={240}
+							value={followupMinutes}
+							onChange={(event) => setFollowupMinutes(Number(event.target.value))}
+						/>
+					</label>
+					<label className="field" htmlFor="autopilot-max-updates">
+						<span>Maximum automatic updates</span>
+						<Input
+							id="autopilot-max-updates"
+							type="number"
+							min={1}
+							max={20}
+							value={maxUpdates}
+							onChange={(event) => setMaxUpdates(Number(event.target.value))}
+						/>
+					</label>
+				</div>
+				<div className="settings-toggle">
+					<Switch id="autopilot-advance-status" checked={advanceStatus} onCheckedChange={setAdvanceStatus} />
+					<label htmlFor="autopilot-advance-status">
+						<strong>Advance incident status</strong>
+						<small>Use objective check patterns to move between investigating, identified, and monitoring.</small>
+					</label>
+				</div>
+				<div className="settings-toggle">
+					<Switch id="autopilot-degraded" checked={degradedIncidents} onCheckedChange={setDegradedIncidents} />
+					<label htmlFor="autopilot-degraded">
+						<strong>Open degraded incidents</strong>
+						<small>Publish performance degradation incidents. Keep disabled to avoid public noise.</small>
+					</label>
+				</div>
+			</fieldset>
 			<div className="settings-actions">
 				<Button
 					variant="unstyled"
@@ -102,6 +161,20 @@ export function SettingsPage() {
 					<h1>Notifications, AI &amp; maintenance</h1>
 					<p>Configure incident alerts, visitor-friendly updates, and planned downtime from one place.</p>
 				</section>
+				<Card asChild>
+					<section className="settings-card">
+						<div className="settings-card-intro">
+							<span>
+								<Activity />
+							</span>
+							<div>
+								<h2>AI activity</h2>
+								<p>Audit model calls, sanitizer rejections, latency, and token usage from the last seven days.</p>
+							</div>
+						</div>
+						<AiActivityPanel />
+					</section>
+				</Card>
 				<Card asChild>
 					<section className="settings-card">
 						<div className="settings-card-intro">

src/client/queries/settings.ts

@@ -1,10 +1,11 @@
 import { useMutation, useQuery } from '@tanstack/react-query';
-import { getAiSettings, testAiSettings, updateAiSettings, type AiSettingsInput } from '../api/settings';
+import { getAiEvents, getAiSettings, testAiSettings, updateAiSettings, type AiSettingsInput } from '../api/settings';
 import { queryClient } from '../lib/query-client';
 
 export const settingsKeys = {
 	all: ['settings'] as const,
 	ai: () => [...settingsKeys.all, 'ai'] as const,
+	aiEvents: () => [...settingsKeys.all, 'ai-events'] as const,
 };
 
 export function useAiSettingsQuery() {
@@ -24,3 +25,10 @@ export function useUpdateAiSettingsMutation() {
 export function useTestAiSettingsMutation() {
 	return useMutation({ mutationFn: testAiSettings });
 }
+
+export function useAiEventsQuery() {
+	return useQuery({
+		queryKey: settingsKeys.aiEvents(),
+		queryFn: ({ signal }) => getAiEvents(signal),
+	});
+}

src/client/styles.css

@@ -1626,6 +1626,81 @@ button {
 .channel-history-state {
 	color: var(--muted);
 }
+.autopilot-settings {
+	display: grid;
+	gap: 16px;
+	margin: 4px 0;
+	padding: 20px;
+	border: 1px solid #e6e6e6;
+	border-radius: 8px;
+	background: #fafcfb;
+}
+.autopilot-settings legend {
+	padding: 0 7px;
+	font-size: 13px;
+	font-weight: 600;
+}
+.autopilot-number-fields {
+	display: grid;
+	grid-template-columns: repeat(2, minmax(0, 1fr));
+	gap: 16px;
+}
+.ai-activity-panel {
+	padding: 22px 26px 26px;
+}
+.ai-activity-stats {
+	display: grid;
+	grid-template-columns: repeat(4, minmax(0, 1fr));
+	gap: 10px;
+}
+.ai-activity-stats > div {
+	display: grid;
+	gap: 3px;
+	padding: 14px;
+	border: 1px solid #e6e6e6;
+	border-radius: 7px;
+	background: #fafafa;
+}
+.ai-activity-stats strong {
+	font-size: 19px;
+}
+.ai-activity-stats span,
+.ai-activity-row span,
+.ai-activity-row small {
+	font-size: 11px;
+	color: var(--muted);
+}
+.ai-activity-history {
+	display: grid;
+	gap: 1px;
+	margin-top: 18px;
+	border: 1px solid #e6e6e6;
+	border-radius: 6px;
+	overflow: hidden;
+	background: #e6e6e6;
+}
+.ai-activity-row {
+	display: grid;
+	grid-template-columns: 100px minmax(110px, 0.8fr) minmax(145px, 1fr) minmax(130px, 1fr) auto;
+	align-items: center;
+	gap: 12px;
+	padding: 10px 12px;
+	font-size: 11px;
+	background: #fff;
+}
+.ai-activity-row strong {
+	font-weight: 600;
+	text-transform: capitalize;
+}
+@media (max-width: 720px) {
+	.ai-activity-stats,
+	.autopilot-number-fields {
+		grid-template-columns: 1fr 1fr;
+	}
+	.ai-activity-row {
+		grid-template-columns: 1fr 1fr;
+	}
+}
 .channel-history-state {
 	width: 100%;
 	margin-top: 16px;

src/worker/ai/autopilot-prompts.ts

@@ -0,0 +1,33 @@
+export const AUTOPILOT_STATUS_GUIDANCE = {
+	investigating: 'State that automated monitoring continues to observe the disruption. Do not imply that the cause is known.',
+	identified:
+		'State only that monitoring has consistently observed the same failure pattern. Do not claim a root cause or human diagnosis.',
+	monitoring:
+		'State that service responses have recently improved and automated monitoring is checking that recovery remains stable. Do not claim a fix was applied.',
+} as const;
+
+const PUBLIC_RULES = [
+	'Write for non-technical visitors in calm, factual language.',
+	'Never include URLs, hostnames, domains, IP addresses, ports, paths, HTTP status codes, error codes, or stack traces.',
+	'Never invent a root cause, a human action, or an estimated recovery time.',
+	'Output only the requested fields.',
+].join('\n');
+
+export const INCIDENT_OPEN_SYSTEM_PROMPT = [
+	'Create a concise public incident title and opening update.',
+	'Use exactly this format: TITLE: one short title\nBODY: two short sentences under 240 characters total.',
+	PUBLIC_RULES,
+].join('\n');
+
+export const INCIDENT_FOLLOWUP_SYSTEM_PROMPT = [
+	'Write one short public follow-up update under 240 characters.',
+	'Describe only the observed visitor impact and current automated-monitoring state.',
+	PUBLIC_RULES,
+].join('\n');
+
+export const INCIDENT_RESOLVE_SYSTEM_PROMPT = [
+	'Write one short public resolution update under 240 characters.',
+	'Say that service has recovered, mention the supplied approximate duration, and say monitoring confirms normal operation.',
+	'Do not claim that a fix was applied.',
+	PUBLIC_RULES,
+].join('\n');

src/worker/ai/client.ts

@@ -6,14 +6,29 @@ export type CompletionSettings = {
 
 type CompletionBody = {
 	choices?: Array<{ message?: { content?: unknown } }>;
+	usage?: { prompt_tokens?: number; completion_tokens?: number };
 };
 
-export async function requestCompletion(
+export type CompletionResult = {
+	content: string | null;
+	latencyMs: number;
+	promptTokens: number | null;
+	completionTokens: number | null;
+	failure: string | null;
+};
+
+function failureMessage(error: unknown): string {
+	if (error instanceof DOMException && error.name === 'TimeoutError') return 'timeout';
+	return (error instanceof Error ? error.message : String(error)).slice(0, 200);
+}
+
+export async function requestCompletionDetailed(
 	settings: CompletionSettings,
 	system: string,
 	user: string,
 	maxTokens = 160,
-): Promise<string | null> {
+): Promise<CompletionResult> {
+	const startedAt = Date.now();
 	try {
 		const response = await fetch(`${settings.baseUrl}/chat/completions`, {
 			method: 'POST',
@@ -35,23 +50,52 @@ export async function requestCompletion(
 		if (!response.ok) {
 			await response.body?.cancel();
 			console.warn(JSON.stringify({ message: 'AI completion returned an error', status: response.status }));
-			return null;
+			return {
+				content: null,
+				latencyMs: Date.now() - startedAt,
+				promptTokens: null,
+				completionTokens: null,
+				failure: `http_${response.status}`,
+			};
 		}
 
 		const body = (await response.json()) as CompletionBody;
 		const content = body.choices?.[0]?.message?.content;
 		if (typeof content !== 'string') {
 			console.warn(JSON.stringify({ message: 'AI completion returned malformed content' }));
-			return null;
+			return {
+				content: null,
+				latencyMs: Date.now() - startedAt,
+				promptTokens: body.usage?.prompt_tokens ?? null,
+				completionTokens: body.usage?.completion_tokens ?? null,
+				failure: 'malformed',
+			};
 		}
-		return content;
+		return {
+			content,
+			latencyMs: Date.now() - startedAt,
+			promptTokens: body.usage?.prompt_tokens ?? null,
+			completionTokens: body.usage?.completion_tokens ?? null,
+			failure: null,
+		};
 	} catch (error) {
-		console.warn(
-			JSON.stringify({
-				message: 'AI completion failed',
-				error: error instanceof Error ? error.message : String(error),
-			}),
-		);
-		return null;
+		const failure = failureMessage(error);
+		console.warn(JSON.stringify({ message: 'AI completion failed', error: failure }));
+		return {
+			content: null,
+			latencyMs: Date.now() - startedAt,
+			promptTokens: null,
+			completionTokens: null,
+			failure,
+		};
 	}
 }
+
+export async function requestCompletion(
+	settings: CompletionSettings,
+	system: string,
+	user: string,
+	maxTokens = 160,
+): Promise<string | null> {
+	return (await requestCompletionDetailed(settings, system, user, maxTokens)).content;
+}

src/worker/ai/events.ts

@@ -0,0 +1,55 @@
+import { getDb } from '../db/client';
+import { aiEvents } from '../db/schema';
+
+export type AiEventKind = 'incident_open' | 'incident_followup' | 'incident_resolve' | 'degraded_open' | 'manual_draft' | 'settings_test';
+export type AiEventOutcome = 'ok' | 'rejected' | 'failed' | 'skipped_budget' | 'skipped_standdown';
+
+export type AiEventInput = {
+	kind: AiEventKind;
+	incidentId?: number | null;
+	monitorId?: number | null;
+	model?: string | null;
+	outcome: AiEventOutcome;
+	reason?: string | null;
+	latencyMs?: number | null;
+	promptTokens?: number | null;
+	completionTokens?: number | null;
+	contextPreview?: string | null;
+	outputPreview?: string | null;
+};
+
+function preview(value: string | null | undefined) {
+	if (!value) return null;
+	return value
+		.replace(/https?:\/\/\S+/gi, '[redacted-url]')
+		.replace(/\b(?:\d{1,3}\.){3}\d{1,3}\b/g, '[redacted-ip]')
+		.replace(/\b(?:[a-z0-9-]+\.)+[a-z]{2,}\b/gi, '[redacted-host]')
+		.replace(/\b(?:sk|pk|rk)-[a-z0-9_-]+\b/gi, '[redacted-key]')
+		.replace(/\bBearer\s+\S+/gi, 'Bearer [redacted]')
+		.replace(/\s+/g, ' ')
+		.trim()
+		.slice(0, 240);
+}
+
+export async function recordAiEvent(env: Env, input: AiEventInput): Promise<void> {
+	try {
+		await getDb(env)
+			.insert(aiEvents)
+			.values({
+				kind: input.kind,
+				incidentId: input.incidentId ?? null,
+				monitorId: input.monitorId ?? null,
+				model: input.model ?? null,
+				outcome: input.outcome,
+				reason: preview(input.reason)?.slice(0, 200) ?? null,
+				latencyMs: input.latencyMs ?? null,
+				promptTokens: input.promptTokens ?? null,
+				completionTokens: input.completionTokens ?? null,
+				contextPreview: preview(input.contextPreview),
+				outputPreview: preview(input.outputPreview),
+				createdAt: new Date(),
+			});
+	} catch (error) {
+		console.warn(JSON.stringify({ message: 'AI event recording failed', error: error instanceof Error ? error.message : String(error) }));
+	}
+}

src/worker/ai/fallback-message.ts

@@ -4,6 +4,9 @@ export const INCIDENT_REASSURANCE =
 
 export const RECOVERY_UPDATE_BODY = 'The service has recovered and is responding normally again.';
 export const DEGRADED_MESSAGE = 'This service is responding more slowly than usual and some pages may take longer to load.';
+export const DEGRADED_RECOVERY_UPDATE_BODY = 'Response times have returned to normal and the service is operating normally.';
+export const DEGRADED_SUPERSEDED_UPDATE_BODY =
+	'The performance issue developed into a service disruption and is tracked in a new incident.';
 
 /** Plain-language, non-technical description of the impact, used when no AI message is available. */
 export function describeFailure(statusCode: number | null): string {

src/worker/ai/incident-context.ts

@@ -2,6 +2,7 @@ import { and, desc, eq, gte, sql } from 'drizzle-orm';
 import type { CheckResult, Monitor } from '../checks/run-check';
 import type { Database } from '../db/client';
 import { checks, incidentMonitors, incidents } from '../db/schema';
+import { classifyFailure, humanizeDuration, humanizeInterval } from '../lib/humanize';
 
 const HOUR_MS = 60 * 60 * 1000;
 const DAY_MS = 24 * HOUR_MS;
@@ -14,55 +15,6 @@ const FLAP_WINDOW = 10;
  * for the model only - it may name HTTP codes and transport errors here because
  * the prompt is responsible for translating them into visitor-facing language.
  */
-function classifyFailure(monitor: Monitor, result: CheckResult): string {
-	if (result.statusCode === null) {
-		const error = (result.error ?? '').toLowerCase();
-		if (/tim(?:e|ed)\s?out|timeout|deadline|aborted/.test(error)) {
-			return 'Request timed out - no response came back before the timeout limit';
-		}
-		if (/getaddrinfo|enotfound|dns|name not resolved|could not resolve|eai_again/.test(error)) {
-			return 'DNS lookup failed - the hostname could not be resolved to an address';
-		}
-		if (/certificate|cert(?:\s|_)|self[- ]signed|tls|ssl|handshake|err_cert/.test(error)) {
-			return 'TLS/SSL failure - the certificate is invalid, expired, or untrusted';
-		}
-		if (/econnrefused|connection refused|refused to connect/.test(error)) {
-			return 'Connection refused - nothing is accepting connections at that address';
-		}
-		if (/econnreset|connection reset|socket hang up|premature close/.test(error)) {
-			return 'Connection dropped before any response was returned';
-		}
-		if (/ehostunreach|enetunreach|network is unreachable|no route to host/.test(error)) {
-			return 'Network unreachable - the host could not be contacted at all';
-		}
-		return 'No HTTP response was received from the endpoint';
-	}
-
-	const code = result.statusCode;
-	if (code >= 500) return `Server error - the endpoint answered with HTTP ${code}`;
-	if (code === 429) return 'The endpoint is rate limiting - HTTP 429 Too Many Requests';
-	if (code === 401 || code === 403) return `The endpoint rejected the health check as unauthorized - HTTP ${code}`;
-	if (code === 404) return 'The health-check path returned HTTP 404 Not Found';
-	if (code >= 400) return `Client error - the endpoint answered with HTTP ${code}`;
-	if (code >= 300) return `Unexpected redirect - the endpoint answered with HTTP ${code}`;
-	return `The endpoint answered with HTTP ${code}, but HTTP ${monitor.expectedStatus} was expected`;
-}
-
-function humanizeInterval(seconds: number): string {
-	if (seconds >= 3600 && seconds % 3600 === 0) return `${seconds / 3600} h`;
-	if (seconds >= 60 && seconds % 60 === 0) return `${seconds / 60} min`;
-	return `${seconds} s`;
-}
-
-function humanizeDuration(ms: number): string {
-	const minutes = Math.round(ms / 60_000);
-	if (minutes < 1) return 'under a minute';
-	if (minutes < 60) return `${minutes} min`;
-	const hours = Math.floor(minutes / 60);
-	const rest = minutes % 60;
-	return rest ? `${hours} h ${rest} min` : `${hours} h`;
-}
-
 function median(values: number[]): number {
 	const sorted = [...values].sort((a, b) => a - b);
 	const mid = Math.floor(sorted.length / 2);

src/worker/ai/incident-draft.ts

@@ -1,7 +1,8 @@
 import { eq } from 'drizzle-orm';
 import { getDb } from '../db/client';
 import { aiSettings } from '../db/schema';
-import { requestCompletion } from './client';
+import { requestCompletionDetailed } from './client';
+import { recordAiEvent } from './events';
 import { sanitizePublicText } from './sanitize';
 
 export type IncidentStatus = 'investigating' | 'identified' | 'monitoring' | 'resolved';
@@ -63,20 +64,54 @@ export async function draftIncidentUpdate(env: Env, input: DraftInput): Promise<
 	]
 		.filter(Boolean)
 		.join('\n\n');
-	const completion = await requestCompletion(
+	const result = await requestCompletionDetailed(
 		{ baseUrl: settings.baseUrl, apiKey: settings.apiKey, model: settings.model },
 		INCIDENT_DRAFT_SYSTEM_PROMPT,
 		context,
 		320,
 	);
-	if (!completion) throw new IncidentDraftError('AI could not generate a safe update. Edit the note or write the update manually.', 422);
+	const completion = result.content;
+	if (!completion) {
+		await recordAiEvent(env, {
+			kind: 'manual_draft',
+			model: settings.model,
+			outcome: 'failed',
+			reason: result.failure,
+			latencyMs: result.latencyMs,
+			promptTokens: result.promptTokens,
+			completionTokens: result.completionTokens,
+			contextPreview: context,
+		});
+		throw new IncidentDraftError('AI could not generate a safe update. Edit the note or write the update manually.', 422);
+	}
 
 	const titleMatch = completion.match(/(?:^|\n)TITLE:\s*(.+?)(?=\nBODY:|$)/is);
 	const bodyMatch = completion.match(/(?:^|\n)BODY:\s*([\s\S]+)$/i);
 	const title = input.withTitle ? sanitizePublicText(titleMatch?.[1] ?? '', 120)?.replace(/[.!?]+$/, '') : null;
 	const body = sanitizePublicText(bodyMatch?.[1] ?? '', 400);
 	if ((input.withTitle && !title) || !body) {
+		await recordAiEvent(env, {
+			kind: 'manual_draft',
+			model: settings.model,
+			outcome: 'rejected',
+			reason: 'sanitizer_rejected',
+			latencyMs: result.latencyMs,
+			promptTokens: result.promptTokens,
+			completionTokens: result.completionTokens,
+			contextPreview: context,
+			outputPreview: completion,
+		});
 		throw new IncidentDraftError('AI could not generate a safe update. Edit the note or write the update manually.', 422);
 	}
+	await recordAiEvent(env, {
+		kind: 'manual_draft',
+		model: settings.model,
+		outcome: 'ok',
+		latencyMs: result.latencyMs,
+		promptTokens: result.promptTokens,
+		completionTokens: result.completionTokens,
+		contextPreview: context,
+		outputPreview: completion,
+	});
 	return { title: title ?? null, body };
 }

src/worker/ai/incident-message.ts

@@ -2,7 +2,8 @@ import { eq } from 'drizzle-orm';
 import type { CheckResult, Monitor } from '../checks/run-check';
 import { getDb } from '../db/client';
 import { aiSettings } from '../db/schema';
-import { requestCompletion } from './client';
+import { requestCompletionDetailed } from './client';
+import { recordAiEvent } from './events';
 import { buildIncidentContext } from './incident-context';
 import { sanitizePublicText } from './sanitize';
 
@@ -32,14 +33,28 @@ export async function generateIncidentMessage(env: Env, input: { monitor: Monito
 		const [settings] = await db.select().from(aiSettings).where(eq(aiSettings.id, 1)).limit(1);
 		if (!settings?.enabled || !settings.baseUrl || !settings.apiKey || !settings.model) return null;
 
-		const content = await requestCompletion(
+		const context = await buildIncidentContext(db, input.monitor, input.result);
+		const result = await requestCompletionDetailed(
 			{ baseUrl: settings.baseUrl, apiKey: settings.apiKey, model: settings.model },
 			INCIDENT_MESSAGE_SYSTEM_PROMPT,
-			await buildIncidentContext(db, input.monitor, input.result),
+			context,
 		);
-		if (!content) return null;
-		const message = sanitizeIncidentMessage(content);
-		if (!message) return null;
+		const message = result.content ? sanitizeIncidentMessage(result.content) : null;
+		if (!message) {
+			await recordAiEvent(env, {
+				kind: 'incident_open',
+				monitorId: input.monitor.id,
+				model: settings.model,
+				outcome: result.failure ? 'failed' : 'rejected',
+				reason: result.failure ?? 'sanitizer_rejected',
+				latencyMs: result.latencyMs,
+				promptTokens: result.promptTokens,
+				completionTokens: result.completionTokens,
+				contextPreview: context,
+				outputPreview: result.content,
+			});
+			return null;
+		}
 
 		await env.DB.prepare(
 			`INSERT INTO incident_updates (incident_id, status, body, source, created_at)
@@ -53,6 +68,17 @@ export async function generateIncidentMessage(env: Env, input: { monitor: Monito
 		)
 			.bind(message, Date.now(), input.monitor.id)
 			.run();
+		await recordAiEvent(env, {
+			kind: 'incident_open',
+			monitorId: input.monitor.id,
+			model: settings.model,
+			outcome: 'ok',
+			latencyMs: result.latencyMs,
+			promptTokens: result.promptTokens,
+			completionTokens: result.completionTokens,
+			contextPreview: context,
+			outputPreview: result.content,
+		});
 		return message;
 	} catch (error) {
 		console.warn(

src/worker/ai/sanitize.ts

@@ -1,4 +1,6 @@
-export function sanitizePublicText(value: string, maxLength: number): string | null {
+export type SanitizeReason = 'contains_url' | 'contains_ip' | 'contains_http_status' | 'empty';
+
+export function sanitizePublicTextWithReason(value: string, maxLength: number): { text: string | null; reason: SanitizeReason | null } {
 	const message = value
 		.replace(/\r/g, '')
 		.trim()
@@ -7,9 +9,15 @@ export function sanitizePublicText(value: string, maxLength: number): string | n
 		.replace(/\s+/g, ' ')
 		.trim()
 		.slice(0, maxLength);
-	if (!message) return null;
-	if (/https?:\/\//i.test(message)) return null;
-	if (/\b(?:\d{1,3}\.){3}\d{1,3}\b/.test(message)) return null;
-	if (/\bHTTP[\s/]?\d{3}\b/i.test(message) || /\b[45]\d{2}\s+(?:error|status|response)\b/i.test(message)) return null;
-	return message;
+	if (!message) return { text: null, reason: 'empty' };
+	if (/https?:\/\//i.test(message)) return { text: null, reason: 'contains_url' };
+	if (/\b(?:\d{1,3}\.){3}\d{1,3}\b/.test(message)) return { text: null, reason: 'contains_ip' };
+	if (/\bHTTP[\s/]?\d{3}\b/i.test(message) || /\b[45]\d{2}\s+(?:error|status|response)\b/i.test(message)) {
+		return { text: null, reason: 'contains_http_status' };
+	}
+	return { text: message, reason: null };
+}
+
+export function sanitizePublicText(value: string, maxLength: number): string | null {
+	return sanitizePublicTextWithReason(value, maxLength).text;
 }

src/worker/autopilot/cadence.ts

@@ -0,0 +1,39 @@
+export type AutopilotIncidentStatus = 'investigating' | 'identified' | 'monitoring' | 'resolved';
+
+export type StatusSignal = {
+	consecutiveFailures: number;
+	failureSignatureStable: boolean;
+	consecutiveOk: number;
+	latestOk: boolean;
+	regressionUsed: boolean;
+};
+
+export type ImpactSignal = {
+	kind: 'down' | 'degraded';
+	affectedMonitors: number;
+	totalMonitors: number;
+	recentChecks: boolean[];
+};
+
+export function nextFollowupDueAt(lastUpdateAt: number, autoUpdateCount: number, cadenceMinutes: number): number {
+	const exponent = Math.max(0, autoUpdateCount - 1);
+	const multiplier = Math.min(8, 2 ** exponent);
+	return lastUpdateAt + cadenceMinutes * multiplier * 60_000;
+}
+
+export function advanceStatus(current: AutopilotIncidentStatus, signal: StatusSignal): AutopilotIncidentStatus {
+	if (current === 'resolved') return current;
+	if (current === 'monitoring' && !signal.latestOk) return signal.regressionUsed ? current : 'identified';
+	if ((current === 'investigating' || current === 'identified') && signal.consecutiveOk >= 2 && !signal.regressionUsed) return 'monitoring';
+	if (current === 'investigating' && signal.consecutiveFailures >= 3 && signal.failureSignatureStable) return 'identified';
+	return current;
+}
+
+export function computeImpact(signal: ImpactSignal): 'minor' | 'major' | 'critical' {
+	if (signal.kind === 'degraded') return 'minor';
+	if (signal.affectedMonitors >= 2 && signal.totalMonitors > 0 && signal.affectedMonitors / signal.totalMonitors >= 0.5) return 'critical';
+	if (signal.affectedMonitors >= 3) return 'critical';
+	if (signal.recentChecks.length >= 10 && signal.recentChecks.slice(0, 10).every((ok) => !ok)) return 'major';
+	if (signal.recentChecks.some(Boolean)) return 'minor';
+	return 'major';
+}

src/worker/autopilot/guards.ts

@@ -0,0 +1,19 @@
+export type AutopilotCandidate = {
+	source: string;
+	resolvedAt: Date | null;
+	hasManualUpdate: boolean;
+	alertsEnabled: boolean;
+};
+
+export function autopilotEligible(candidate: AutopilotCandidate): boolean {
+	return candidate.source === 'auto' && candidate.resolvedAt === null && !candidate.hasManualUpdate && candidate.alertsEnabled;
+}
+
+export const AUTOPILOT_WRITE_GUARD = `
+	i.source = 'auto'
+	AND i.resolved_at IS NULL
+	AND i.updated_at = ?
+	AND NOT EXISTS (
+		SELECT 1 FROM incident_updates manual
+		WHERE manual.incident_id = i.id AND manual.source = 'manual'
+	)`;

src/worker/autopilot/index.ts

@@ -0,0 +1,379 @@
+import { and, eq, isNull, sql } from 'drizzle-orm';
+import { requestCompletionDetailed, type CompletionResult } from '../ai/client';
+import { DEGRADED_RECOVERY_UPDATE_BODY, RECOVERY_UPDATE_BODY } from '../ai/fallback-message';
+import { buildIncidentContext } from '../ai/incident-context';
+import { recordAiEvent, type AiEventKind } from '../ai/events';
+import {
+	AUTOPILOT_STATUS_GUIDANCE,
+	INCIDENT_FOLLOWUP_SYSTEM_PROMPT,
+	INCIDENT_OPEN_SYSTEM_PROMPT,
+	INCIDENT_RESOLVE_SYSTEM_PROMPT,
+} from '../ai/autopilot-prompts';
+import { sanitizePublicTextWithReason } from '../ai/sanitize';
+import type { CheckResult, Monitor } from '../checks/run-check';
+import { getDb } from '../db/client';
+import { aiSettings, incidentUpdates, incidents } from '../db/schema';
+import { humanizeDuration } from '../lib/humanize';
+import { advanceStatus, computeImpact, nextFollowupDueAt, type AutopilotIncidentStatus } from './cadence';
+import { loadIncidentSignal, findLatestAutoIncidentForMonitor } from './signal';
+
+export const MAX_AI_CALLS_PER_RUN = 12;
+export const MAX_FOLLOWUP_CALLS_PER_RUN = 6;
+export const AUTOPILOT_CONCURRENCY = 4;
+export const AUTOPILOT_DEADLINE_MS = 45_000;
+
+export type AiBudget = { remaining: number; deadline?: number };
+export type AutopilotEvent = {
+	monitor: Monitor;
+	result: CheckResult;
+	transition: 'opened' | 'resolved' | null;
+	latencyTransition: 'degraded' | 'recovered' | null;
+	checkedAt: Date;
+};
+export type AutopilotSummary = { calls: number; written: number; rejected: number; failed: number; skipped: number };
+
+type Settings = typeof aiSettings.$inferSelect;
+type Task = { kind: AiEventKind; incidentId: number; monitorId: number; run: () => Promise<boolean> };
+
+function completionEvent(result: CompletionResult) {
+	return {
+		latencyMs: result.latencyMs,
+		promptTokens: result.promptTokens,
+		completionTokens: result.completionTokens,
+		outputPreview: result.content,
+	};
+}
+
+async function complete(settings: Settings, system: string, user: string, maxTokens = 180) {
+	return requestCompletionDetailed(
+		{ baseUrl: settings.baseUrl!, apiKey: settings.apiKey!, model: settings.model! },
+		system,
+		user,
+		maxTokens,
+	);
+}
+
+async function writeCasUpdate(
+	env: Env,
+	incident: typeof incidents.$inferSelect,
+	status: string,
+	body: string,
+	now: number,
+): Promise<boolean> {
+	const results = await env.DB.batch([
+		env.DB.prepare(
+			`INSERT INTO incident_updates (incident_id, status, body, source, created_at)
+			 SELECT i.id, ?, ?, 'ai', ? FROM incidents i
+			 WHERE i.id = ? AND i.source = 'auto' AND i.resolved_at IS NULL AND i.updated_at = ?
+			   AND NOT EXISTS (SELECT 1 FROM incident_updates m WHERE m.incident_id = i.id AND m.source = 'manual')`,
+		).bind(status, body, now, incident.id, incident.updatedAt.getTime()),
+		env.DB.prepare(
+			`UPDATE incidents SET status = ?, updated_at = ?
+			 WHERE id = ? AND source = 'auto' AND resolved_at IS NULL AND updated_at = ?
+			   AND NOT EXISTS (SELECT 1 FROM incident_updates m WHERE m.incident_id = incidents.id AND m.source = 'manual')`,
+		).bind(status, now, incident.id, incident.updatedAt.getTime()),
+	]);
+	return Number(results[0].meta.changes ?? 0) === 1 && Number(results[1].meta.changes ?? 0) === 1;
+}
+
+async function processOpening(env: Env, settings: Settings, event: AutopilotEvent, incidentId: number, kind: 'down' | 'degraded') {
+	const db = getDb(env);
+	const [incident] = await db.select().from(incidents).where(eq(incidents.id, incidentId)).limit(1);
+	if (!incident || incident.resolvedAt || incident.source !== 'auto') return false;
+	const signal = await loadIncidentSignal(db, incident.id);
+	if (!signal) return false;
+	const context = await buildIncidentContext(db, event.monitor, event.result);
+	const result = await complete(settings, INCIDENT_OPEN_SYSTEM_PROMPT, context, 220);
+	if (!result.content) {
+		await recordAiEvent(env, {
+			kind: kind === 'degraded' ? 'degraded_open' : 'incident_open',
+			incidentId,
+			monitorId: event.monitor.id,
+			model: settings.model,
+			outcome: 'failed',
+			reason: result.failure,
+			contextPreview: context,
+			...completionEvent(result),
+		});
+		return false;
+	}
+	const titleRaw = result.content.match(/(?:^|\n)TITLE:\s*(.+?)(?=\nBODY:|$)/is)?.[1] ?? '';
+	const bodyRaw = result.content.match(/(?:^|\n)BODY:\s*([\s\S]+)$/i)?.[1] ?? '';
+	const title = sanitizePublicTextWithReason(titleRaw, 120);
+	const body = sanitizePublicTextWithReason(bodyRaw, 280);
+	const impact = computeImpact({
+		kind,
+		affectedMonitors: signal.affectedMonitors,
+		totalMonitors: signal.totalMonitors,
+		recentChecks: signal.recentChecks.map((check) => check.ok),
+	});
+	const now = Date.now();
+	const statements: D1PreparedStatement[] = [];
+	if (body.text) {
+		statements.push(
+			env.DB.prepare(
+				`INSERT INTO incident_updates (incident_id, status, body, source, created_at)
+				 SELECT i.id, i.status, ?, 'ai', ? FROM incidents i
+				 WHERE i.id = ? AND i.source = 'auto' AND i.resolved_at IS NULL AND i.updated_at = ?
+				   AND NOT EXISTS (SELECT 1 FROM incident_updates u WHERE u.incident_id = i.id)`,
+			).bind(body.text, now, incident.id, incident.updatedAt.getTime()),
+		);
+	}
+	statements.push(
+		env.DB.prepare(
+			`UPDATE incidents SET title = coalesce(?, title), impact = ?, updated_at = ?
+			 WHERE id = ? AND source = 'auto' AND resolved_at IS NULL AND updated_at = ? AND title IS NULL
+			   AND NOT EXISTS (SELECT 1 FROM incident_updates m WHERE m.incident_id = incidents.id AND m.source = 'manual')`,
+		).bind(title.text, impact, now, incident.id, incident.updatedAt.getTime()),
+	);
+	const writeResults = await env.DB.batch(statements);
+	const wrote = writeResults.some((writeResult) => Number(writeResult.meta.changes ?? 0) > 0);
+	await recordAiEvent(env, {
+		kind: kind === 'degraded' ? 'degraded_open' : 'incident_open',
+		incidentId,
+		monitorId: event.monitor.id,
+		model: settings.model,
+		outcome: title.text || body.text ? (wrote ? 'ok' : 'skipped_standdown') : 'rejected',
+		reason:
+			[title.reason && `title:${title.reason}`, body.reason && `body:${body.reason}`].filter(Boolean).join(',') ||
+			(wrote ? null : 'cas_conflict_or_manual_update'),
+		contextPreview: context,
+		...completionEvent(result),
+	});
+	return wrote;
+}
+
+async function processResolution(env: Env, settings: Settings, event: AutopilotEvent, incidentId: number, fallbackBody: string) {
+	const db = getDb(env);
+	const [incident] = await db.select().from(incidents).where(eq(incidents.id, incidentId)).limit(1);
+	if (!incident?.resolvedAt) return false;
+	const context = `Service name: ${event.monitor.name}\nObserved recovery: automated checks are healthy\nIncident duration: ${humanizeDuration(incident.durationMs ?? incident.resolvedAt.getTime() - incident.startedAt.getTime())}`;
+	const result = await complete(settings, INCIDENT_RESOLVE_SYSTEM_PROMPT, context);
+	const sanitized = result.content ? sanitizePublicTextWithReason(result.content, 280) : { text: null, reason: null };
+	let wrote = false;
+	if (sanitized.text) {
+		const write = await env.DB.prepare(
+			`UPDATE incident_updates SET body = ?, source = 'ai'
+			 WHERE id = (
+				SELECT iu.id FROM incident_updates iu
+				WHERE iu.incident_id = ? AND iu.status = 'resolved' AND iu.source = 'system'
+				  AND iu.body = ?
+				  AND NOT EXISTS (
+					SELECT 1 FROM incident_updates manual
+					WHERE manual.incident_id = iu.incident_id AND manual.source = 'manual'
+				  )
+				ORDER BY iu.id DESC LIMIT 1
+			)`,
+		)
+			.bind(sanitized.text, incidentId, fallbackBody)
+			.run();
+		wrote = Number(write.meta.changes ?? 0) === 1;
+	}
+	await recordAiEvent(env, {
+		kind: 'incident_resolve',
+		incidentId,
+		monitorId: event.monitor.id,
+		model: settings.model,
+		outcome: sanitized.text ? (wrote ? 'ok' : 'skipped_standdown') : result.failure ? 'failed' : 'rejected',
+		reason: result.failure ?? sanitized.reason ?? (wrote ? null : 'manual_update_or_already_rewritten'),
+		contextPreview: context,
+		...completionEvent(result),
+	});
+	return wrote;
+}
+
+async function processFollowup(env: Env, settings: Settings, incident: typeof incidents.$inferSelect) {
+	const db = getDb(env);
+	const signal = await loadIncidentSignal(db, incident.id);
+	if (!signal) return false;
+	const status = settings.autopilotAdvanceStatus
+		? advanceStatus(incident.status as AutopilotIncidentStatus, signal)
+		: (incident.status as AutopilotIncidentStatus);
+	const guidance = AUTOPILOT_STATUS_GUIDANCE[status === 'resolved' ? 'monitoring' : status];
+	const context = [
+		`Service name: ${signal.monitor.name}`,
+		`Incident age: ${humanizeDuration(Date.now() - incident.startedAt.getTime())}`,
+		`Current state: ${status}`,
+		`Recent checks: ${signal.recentChecks.filter((check) => check.ok).length} healthy of ${signal.recentChecks.length}`,
+		`Writing guidance: ${guidance}`,
+	].join('\n');
+	const result = await complete(settings, INCIDENT_FOLLOWUP_SYSTEM_PROMPT, context);
+	const sanitized = result.content ? sanitizePublicTextWithReason(result.content, 280) : { text: null, reason: null };
+	const wrote = sanitized.text ? await writeCasUpdate(env, incident, status, sanitized.text, Date.now()) : false;
+	await recordAiEvent(env, {
+		kind: 'incident_followup',
+		incidentId: incident.id,
+		monitorId: signal.monitor.id,
+		model: settings.model,
+		outcome: sanitized.text ? (wrote ? 'ok' : 'skipped_standdown') : result.failure ? 'failed' : 'rejected',
+		reason: result.failure ?? sanitized.reason ?? (wrote ? null : 'cas_conflict_or_manual_update'),
+		contextPreview: context,
+		...completionEvent(result),
+	});
+	return wrote;
+}
+
+async function loadFollowupTasks(env: Env, settings: Settings, excluded: Set<number>): Promise<Task[]> {
+	const db = getDb(env);
+	const rows = await db
+		.select({
+			incident: incidents,
+			lastUpdateAt: sql<number>`coalesce(max(${incidentUpdates.createdAt}), ${incidents.startedAt})`,
+			autoUpdateCount: sql<number>`coalesce(sum(case when ${incidentUpdates.source} in ('ai','system') then 1 else 0 end), 0)`,
+			hasManual: sql<number>`coalesce(max(case when ${incidentUpdates.source} = 'manual' then 1 else 0 end), 0)`,
+		})
+		.from(incidents)
+		.leftJoin(incidentUpdates, eq(incidentUpdates.incidentId, incidents.id))
+		.where(and(eq(incidents.source, 'auto'), isNull(incidents.resolvedAt)))
+		.groupBy(incidents.id)
+		.orderBy(sql`coalesce(max(${incidentUpdates.createdAt}), ${incidents.startedAt}) asc`);
+	const tasks: Task[] = [];
+	for (const row of rows) {
+		if (excluded.has(row.incident.id)) continue;
+		if (Number(row.hasManual) > 0) {
+			const [alreadyLogged] = await db
+				.select({ id: sql<number>`id` })
+				.from(sql`ai_events`)
+				.where(sql`incident_id = ${row.incident.id} and kind = 'incident_followup' and outcome = 'skipped_standdown'`)
+				.limit(1);
+			if (alreadyLogged) continue;
+			const [monitor] = await db
+				.select({ id: sql<number>`monitor_id` })
+				.from(sql`incident_monitors`)
+				.where(sql`incident_id = ${row.incident.id}`)
+				.limit(1);
+			await recordAiEvent(env, {
+				kind: 'incident_followup',
+				incidentId: row.incident.id,
+				monitorId: monitor?.id ?? null,
+				model: settings.model,
+				outcome: 'skipped_standdown',
+				reason: 'manual_update_present',
+			});
+			continue;
+		}
+		const count = Number(row.autoUpdateCount);
+		if (count >= settings.autopilotMaxUpdates) continue;
+		const last = row.lastUpdateAt instanceof Date ? row.lastUpdateAt.getTime() : Number(row.lastUpdateAt);
+		if (Date.now() < nextFollowupDueAt(last, count, settings.autopilotFollowupMinutes)) continue;
+		const signal = await loadIncidentSignal(db, row.incident.id);
+		if (!signal?.monitor.alertsEnabled) continue;
+		tasks.push({
+			kind: 'incident_followup',
+			incidentId: row.incident.id,
+			monitorId: signal.monitor.id,
+			run: () => processFollowup(env, settings, row.incident),
+		});
+		if (tasks.length >= MAX_FOLLOWUP_CALLS_PER_RUN) break;
+	}
+	return tasks;
+}
+
+export async function runAutopilot(
+	env: Env,
+	input: { events?: AutopilotEvent[]; budget?: AiBudget; skipSweep?: boolean } = {},
+): Promise<AutopilotSummary> {
+	const summary: AutopilotSummary = { calls: 0, written: 0, rejected: 0, failed: 0, skipped: 0 };
+	const db = getDb(env);
+	const [settings] = await db.select().from(aiSettings).where(eq(aiSettings.id, 1)).limit(1);
+	if (!settings?.enabled || !settings.autopilotEnabled || !settings.baseUrl || !settings.apiKey || !settings.model) return summary;
+	const budget = input.budget ?? { remaining: MAX_AI_CALLS_PER_RUN, deadline: Date.now() + AUTOPILOT_DEADLINE_MS };
+	budget.deadline ??= Date.now() + AUTOPILOT_DEADLINE_MS;
+	const tasks: Task[] = [];
+	const excluded = new Set<number>();
+	for (const event of input.events ?? []) {
+		if (!event.monitor.alertsEnabled) continue;
+		if (event.transition === 'opened') {
+			const incident = await findLatestAutoIncidentForMonitor(db, event.monitor.id, { resolved: false, kind: 'down' });
+			if (incident) {
+				excluded.add(incident.id);
+				tasks.push({
+					kind: 'incident_open',
+					incidentId: incident.id,
+					monitorId: event.monitor.id,
+					run: () => processOpening(env, settings, event, incident.id, 'down'),
+				});
+			}
+		}
+		if (event.transition === 'resolved') {
+			const incident = await findLatestAutoIncidentForMonitor(db, event.monitor.id, { resolved: true, kind: 'down' });
+			if (incident) {
+				tasks.push({
+					kind: 'incident_resolve',
+					incidentId: incident.id,
+					monitorId: event.monitor.id,
+					run: () => processResolution(env, settings, event, incident.id, RECOVERY_UPDATE_BODY),
+				});
+			}
+		}
+		if (event.latencyTransition === 'degraded' && settings.autopilotDegradedIncidents) {
+			const incident = await findLatestAutoIncidentForMonitor(db, event.monitor.id, { resolved: false, kind: 'degraded' });
+			if (incident) {
+				excluded.add(incident.id);
+				tasks.push({
+					kind: 'degraded_open',
+					incidentId: incident.id,
+					monitorId: event.monitor.id,
+					run: () => processOpening(env, settings, event, incident.id, 'degraded'),
+				});
+			}
+		}
+		if (event.transition !== 'opened' && event.latencyTransition === 'recovered' && settings.autopilotDegradedIncidents) {
+			const incident = await findLatestAutoIncidentForMonitor(db, event.monitor.id, { resolved: true, kind: 'degraded' });
+			if (incident) {
+				tasks.push({
+					kind: 'incident_resolve',
+					incidentId: incident.id,
+					monitorId: event.monitor.id,
+					run: () => processResolution(env, settings, event, incident.id, DEGRADED_RECOVERY_UPDATE_BODY),
+				});
+			}
+		}
+	}
+	if (!input.skipSweep) tasks.push(...(await loadFollowupTasks(env, settings, excluded)));
+	const priority: Record<AiEventKind, number> = {
+		incident_open: 0,
+		incident_resolve: 1,
+		degraded_open: 2,
+		incident_followup: 3,
+		manual_draft: 4,
+		settings_test: 5,
+	};
+	tasks.sort((left, right) => priority[left.kind] - priority[right.kind]);
+
+	for (let offset = 0; offset < tasks.length; offset += AUTOPILOT_CONCURRENCY) {
+		const batch = tasks.slice(offset, offset + AUTOPILOT_CONCURRENCY);
+		await Promise.all(
+			batch.map(async (task) => {
+				if (budget.remaining <= 0 || Date.now() >= budget.deadline!) {
+					summary.skipped += 1;
+					await recordAiEvent(env, {
+						kind: task.kind,
+						incidentId: task.incidentId,
+						monitorId: task.monitorId,
+						model: settings.model,
+						outcome: 'skipped_budget',
+						reason: budget.remaining <= 0 ? 'per_run_limit' : 'deadline',
+					});
+					return;
+				}
+				budget.remaining -= 1;
+				summary.calls += 1;
+				try {
+					if (await task.run()) summary.written += 1;
+				} catch (error) {
+					summary.failed += 1;
+					console.warn(
+						JSON.stringify({
+							message: 'autopilot task failed',
+							kind: task.kind,
+							incidentId: task.incidentId,
+							error: error instanceof Error ? error.message : String(error),
+						}),
+					);
+				}
+			}),
+		);
+	}
+	return summary;
+}

src/worker/autopilot/signal.ts

@@ -0,0 +1,102 @@
+import { and, asc, desc, eq, isNotNull, isNull, sql } from 'drizzle-orm';
+import type { CheckResult, Monitor } from '../checks/run-check';
+import type { Database } from '../db/client';
+import { checks, incidentMonitors, incidents, incidentUpdates, monitors } from '../db/schema';
+import { classifyFailure } from '../lib/humanize';
+
+export type IncidentSignal = {
+	monitor: Monitor;
+	recentChecks: Array<{ ok: boolean; statusCode: number | null; latencyMs: number | null; error: string | null }>;
+	consecutiveFailures: number;
+	consecutiveOk: number;
+	failureSignatureStable: boolean;
+	latestOk: boolean;
+	regressionUsed: boolean;
+	affectedMonitors: number;
+	totalMonitors: number;
+};
+
+export async function findLatestAutoIncidentForMonitor(
+	db: Database,
+	monitorId: number,
+	options: { resolved: boolean; kind?: 'down' | 'degraded' },
+) {
+	const [incident] = await db
+		.select()
+		.from(incidents)
+		.innerJoin(incidentMonitors, eq(incidentMonitors.incidentId, incidents.id))
+		.where(
+			and(
+				eq(incidentMonitors.monitorId, monitorId),
+				eq(incidents.source, 'auto'),
+				options.kind ? eq(incidents.kind, options.kind) : undefined,
+				options.resolved ? isNotNull(incidents.resolvedAt) : isNull(incidents.resolvedAt),
+			),
+		)
+		.orderBy(desc(incidents.startedAt))
+		.limit(1);
+	return incident?.incidents ?? null;
+}
+
+export async function loadIncidentSignal(db: Database, incidentId: number): Promise<IncidentSignal | null> {
+	const [monitor] = await db
+		.select({ monitor: monitors })
+		.from(incidentMonitors)
+		.innerJoin(monitors, eq(monitors.id, incidentMonitors.monitorId))
+		.where(eq(incidentMonitors.incidentId, incidentId))
+		.orderBy(asc(incidentMonitors.monitorId))
+		.limit(1);
+	if (!monitor) return null;
+
+	const recentChecks = await db
+		.select({ ok: checks.ok, statusCode: checks.statusCode, latencyMs: checks.latencyMs, error: checks.error })
+		.from(checks)
+		.where(and(eq(checks.monitorId, monitor.monitor.id), eq(checks.maintenance, false)))
+		.orderBy(desc(checks.checkedAt))
+		.limit(10);
+	let consecutiveFailures = 0;
+	let consecutiveOk = 0;
+	for (const check of recentChecks) {
+		if (check.ok) break;
+		consecutiveFailures += 1;
+	}
+	for (const check of recentChecks) {
+		if (!check.ok) break;
+		consecutiveOk += 1;
+	}
+	const failureSignatures = recentChecks.slice(0, consecutiveFailures).map((check) =>
+		classifyFailure(monitor.monitor, {
+			ok: check.ok,
+			degraded: false,
+			statusCode: check.statusCode,
+			latencyMs: check.latencyMs ?? 0,
+			error: check.error,
+		} satisfies CheckResult),
+	);
+	const statusHistory = await db
+		.select({ status: incidentUpdates.status })
+		.from(incidentUpdates)
+		.where(eq(incidentUpdates.incidentId, incidentId))
+		.orderBy(incidentUpdates.createdAt, incidentUpdates.id);
+	const monitoringIndex = statusHistory.findIndex((update) => update.status === 'monitoring');
+	const regressionUsed = monitoringIndex >= 0 && statusHistory.slice(monitoringIndex + 1).some((update) => update.status === 'identified');
+	const [affected] = await db
+		.select({ count: sql<number>`count(*)` })
+		.from(incidentMonitors)
+		.where(eq(incidentMonitors.incidentId, incidentId));
+	const [total] = await db
+		.select({ count: sql<number>`count(*)` })
+		.from(monitors)
+		.where(eq(monitors.enabled, true));
+	return {
+		monitor: monitor.monitor,
+		recentChecks,
+		consecutiveFailures,
+		consecutiveOk,
+		failureSignatureStable: failureSignatures.length >= 3 && new Set(failureSignatures).size === 1,
+		latestOk: recentChecks[0]?.ok ?? false,
+		regressionUsed,
+		affectedMonitors: Number(affected?.count ?? 1),
+		totalMonitors: Number(total?.count ?? 1),
+	};
+}

src/worker/checks/persist-result.ts

@@ -1,7 +1,7 @@
 import { and, eq, inArray, isNull, sql } from 'drizzle-orm';
 import type { Database } from '../db/client';
 import { checks, incidentMonitors, incidents, incidentUpdates, monitors } from '../db/schema';
-import { RECOVERY_UPDATE_BODY } from '../ai/fallback-message';
+import { DEGRADED_RECOVERY_UPDATE_BODY, DEGRADED_SUPERSEDED_UPDATE_BODY, RECOVERY_UPDATE_BODY } from '../ai/fallback-message';
 import type { CheckResult, Monitor } from './run-check';
 
 /** Only opened and resolved transitions are allowed to send alerts. */
@@ -11,7 +11,14 @@ export type LatencyTransition = 'degraded' | 'recovered' | null;
 
 type BatchStatement = Parameters<Database['batch']>[0][number];
 
-export function buildResultStatements(db: Database, monitor: Monitor, result: CheckResult, checkedAt: Date, maintenance = false) {
+export function buildResultStatements(
+	db: Database,
+	monitor: Monitor,
+	result: CheckResult,
+	checkedAt: Date,
+	maintenance = false,
+	options: { degradedIncidents?: boolean } = {},
+) {
 	const statements: BatchStatement[] = [
 		db.insert(checks).values({
 			monitorId: monitor.id,
@@ -78,11 +85,14 @@ export function buildResultStatements(db: Database, monitor: Monitor, result: Ch
 
 	let transition: CheckTransition = null;
 	if (!wasDown && isDown) {
+		const localImpact =
+			result.statusCode === null || (result.statusCode !== null && result.statusCode >= 500) || nextFailures >= 10 ? 'major' : 'minor';
 		statements.push(
 			db.insert(incidents).values({
 				status: 'investigating',
-				impact: 'major',
+				impact: localImpact,
 				source: 'auto',
+				kind: 'down',
 				startedAt: checkedAt,
 				startStatusCode: result.statusCode,
 				startError: result.error,
@@ -91,13 +101,60 @@ export function buildResultStatements(db: Database, monitor: Monitor, result: Ch
 			}),
 			db.insert(incidentMonitors).values({ incidentId: sql`last_insert_rowid()`, monitorId: monitor.id }),
 		);
+		// Keep this after incident_monitors: that statement must consume the new down incident rowid first.
+		if (options.degradedIncidents) {
+			const degradedIds = db
+				.select({ id: incidentMonitors.incidentId })
+				.from(incidentMonitors)
+				.innerJoin(incidents, eq(incidents.id, incidentMonitors.incidentId))
+				.where(
+					and(
+						eq(incidentMonitors.monitorId, monitor.id),
+						eq(incidents.source, 'auto'),
+						eq(incidents.kind, 'degraded'),
+						isNull(incidents.resolvedAt),
+					),
+				);
+			statements.push(
+				db.insert(incidentUpdates).select(
+					db
+						.select({
+							id: sql<number | null>`null`.as('id'),
+							incidentId: incidents.id,
+							status: sql<string>`'resolved'`.as('status'),
+							body: sql<string>`${DEGRADED_SUPERSEDED_UPDATE_BODY}`.as('body'),
+							note: sql<string | null>`null`.as('note'),
+							source: sql<string>`'system'`.as('source'),
+							createdAt: sql<Date>`${checkedAt.getTime()}`.as('created_at'),
+						})
+						.from(incidents)
+						.where(inArray(incidents.id, degradedIds)),
+				),
+				db
+					.update(incidents)
+					.set({
+						status: 'resolved',
+						resolvedAt: checkedAt,
+						durationMs: sql`${checkedAt.getTime()} - ${incidents.startedAt}`,
+						updatedAt: checkedAt,
+					})
+					.where(inArray(incidents.id, degradedIds)),
+			);
+		}
 		transition = 'opened';
 	} else if (wasDown && result.ok) {
 		const openIncidentIds = db
 			.select({ id: incidentMonitors.incidentId })
 			.from(incidentMonitors)
 			.innerJoin(incidents, eq(incidents.id, incidentMonitors.incidentId))
-			.where(and(eq(incidentMonitors.monitorId, monitor.id), eq(incidents.source, 'auto'), isNull(incidents.resolvedAt)));
+			.where(
+				and(
+					eq(incidentMonitors.monitorId, monitor.id),
+					eq(incidents.source, 'auto'),
+					eq(incidents.kind, 'down'),
+					isNull(incidents.resolvedAt),
+				),
+			);
 		statements.push(
 			db.insert(incidentUpdates).select(
 				db
@@ -111,7 +168,14 @@ export function buildResultStatements(db: Database, monitor: Monitor, result: Ch
 						createdAt: sql<Date>`${checkedAt.getTime()}`.as('created_at'),
 					})
 					.from(incidents)
-					.where(and(eq(incidents.source, 'auto'), isNull(incidents.resolvedAt), inArray(incidents.id, openIncidentIds))),
+					.where(
+						and(
+							eq(incidents.source, 'auto'),
+							eq(incidents.kind, 'down'),
+							isNull(incidents.resolvedAt),
+							inArray(incidents.id, openIncidentIds),
+						),
+					),
 			),
 			db
 				.update(incidents)
@@ -121,11 +185,73 @@ export function buildResultStatements(db: Database, monitor: Monitor, result: Ch
 					durationMs: sql`${checkedAt.getTime()} - ${incidents.startedAt}`,
 					updatedAt: checkedAt,
 				})
-				.where(and(eq(incidents.source, 'auto'), isNull(incidents.resolvedAt), inArray(incidents.id, openIncidentIds))),
+				.where(
+					and(
+						eq(incidents.source, 'auto'),
+						eq(incidents.kind, 'down'),
+						isNull(incidents.resolvedAt),
+						inArray(incidents.id, openIncidentIds),
+					),
+				),
 		);
 		transition = 'resolved';
 	} else if (!wasDown && !result.ok) transition = 'pending';
 	else if (!wasDown && result.ok && previousFailures > 0) transition = 'cleared';
 
+	if (options.degradedIncidents && monitor.alertsEnabled && latencyTransition === 'degraded' && !isDown) {
+		statements.push(
+			db.insert(incidents).values({
+				status: 'investigating',
+				impact: 'minor',
+				source: 'auto',
+				kind: 'degraded',
+				startedAt: checkedAt,
+				startStatusCode: result.statusCode,
+				startError: result.error,
+				createdAt: checkedAt,
+				updatedAt: checkedAt,
+			}),
+			db.insert(incidentMonitors).values({ incidentId: sql`last_insert_rowid()`, monitorId: monitor.id }),
+		);
+	} else if (options.degradedIncidents && latencyTransition === 'recovered') {
+		const degradedIds = db
+			.select({ id: incidentMonitors.incidentId })
+			.from(incidentMonitors)
+			.innerJoin(incidents, eq(incidents.id, incidentMonitors.incidentId))
+			.where(
+				and(
+					eq(incidentMonitors.monitorId, monitor.id),
+					eq(incidents.source, 'auto'),
+					eq(incidents.kind, 'degraded'),
+					isNull(incidents.resolvedAt),
+				),
+			);
+		statements.push(
+			db.insert(incidentUpdates).select(
+				db
+					.select({
+						id: sql<number | null>`null`.as('id'),
+						incidentId: incidents.id,
+						status: sql<string>`'resolved'`.as('status'),
+						body: sql<string>`${DEGRADED_RECOVERY_UPDATE_BODY}`.as('body'),
+						note: sql<string | null>`null`.as('note'),
+						source: sql<string>`'system'`.as('source'),
+						createdAt: sql<Date>`${checkedAt.getTime()}`.as('created_at'),
+					})
+					.from(incidents)
+					.where(inArray(incidents.id, degradedIds)),
+			),
+			db
+				.update(incidents)
+				.set({
+					status: 'resolved',
+					resolvedAt: checkedAt,
+					durationMs: sql`${checkedAt.getTime()} - ${incidents.startedAt}`,
+					updatedAt: checkedAt,
+				})
+				.where(inArray(incidents.id, degradedIds)),
+		);
+	}
+
 	return { statements, transition, latencyTransition, consecutiveFailures: nextFailures };
 }

src/worker/checks/run-due-checks.ts

@@ -1,15 +1,16 @@
 import { and, eq, sql } from 'drizzle-orm';
-import { generateIncidentMessage } from '../ai/incident-message';
+import type { AutopilotEvent } from '../autopilot';
 import { getDb } from '../db/client';
-import { monitors } from '../db/schema';
+import { aiSettings, monitors } from '../db/schema';
 import { loadActiveMaintenance } from '../maintenance/windows';
+import { buildAlertEvent } from '../notifications/compose';
 import { dispatchNotification, MAX_NOTIFICATIONS_PER_RUN, type NotificationBudget } from '../notifications/dispatch';
-import { type AlertTransition, buildResultStatements } from './persist-result';
+import { buildResultStatements } from './persist-result';
 import { runCheck, runCheckWithRetries, type RetryBudget } from './run-check';
 
 const MAX_MONITORS_PER_RUN = 40;
-const MAX_AI_MESSAGES_PER_RUN = 10;
 const CONCURRENCY = 10;
+const MAX_BATCH_STATEMENTS = 100;
 export const MAX_RETRY_ATTEMPTS_PER_RUN = 60;
 const RETRY_DEADLINE_MS = 90_000;
 
@@ -20,6 +21,7 @@ export type DueCheckSummary = {
 	pending: number;
 	opened: number;
 	retries: number;
+	events: AutopilotEvent[];
 };
 
 export async function runDueChecks(env: Env, ctx?: Pick<ExecutionContext, 'waitUntil'>): Promise<DueCheckSummary> {
@@ -37,8 +39,11 @@ export async function runDueChecks(env: Env, ctx?: Pick<ExecutionContext, 'waitU
 		.orderBy(sql`${monitors.lastCheckedAt} ASC NULLS FIRST`)
 		.limit(MAX_MONITORS_PER_RUN);
 
-	if (due.length === 0) return { checked: 0, up: 0, down: 0, pending: 0, opened: 0, retries: 0 };
-	const activeMaintenance = await loadActiveMaintenance(db, new Date());
+	if (due.length === 0) return { checked: 0, up: 0, down: 0, pending: 0, opened: 0, retries: 0, events: [] };
+	const [activeMaintenance, [settings]] = await Promise.all([
+		loadActiveMaintenance(db, new Date()),
+		db.select().from(aiSettings).where(eq(aiSettings.id, 1)).limit(1),
+	]);
 	const budget: RetryBudget = { remaining: MAX_RETRY_ATTEMPTS_PER_RUN, deadline: Date.now() + RETRY_DEADLINE_MS };
 
 	const completed: Array<{
@@ -65,58 +70,31 @@ export async function runDueChecks(env: Env, ctx?: Pick<ExecutionContext, 'waitU
 		monitor,
 		result,
 		checkedAt,
-		...buildResultStatements(db, monitor, result, checkedAt, maintenance),
+		...buildResultStatements(db, monitor, result, checkedAt, maintenance, {
+			degradedIncidents: Boolean(settings?.autopilotEnabled && settings.autopilotDegradedIncidents),
+		}),
 	}));
-	const statements = persisted.flatMap((item) => item.statements);
+	let statementChunk: (typeof persisted)[number]['statements'] = [];
+	for (const item of persisted) {
+		if (statementChunk.length > 0 && statementChunk.length + item.statements.length > MAX_BATCH_STATEMENTS) {
+			await db.batch(statementChunk as [(typeof statementChunk)[number], ...typeof statementChunk]);
+			statementChunk = [];
+		}
+		statementChunk.push(...item.statements);
+	}
+	if (statementChunk.length > 0) await db.batch(statementChunk as [(typeof statementChunk)[number], ...typeof statementChunk]);
 
-	await db.batch(statements as [(typeof statements)[number], ...typeof statements]);
-
-	let aiMessagesQueued = 0;
 	const notificationBudget: NotificationBudget = { remaining: MAX_NOTIFICATIONS_PER_RUN };
 	const notifications = persisted.flatMap((item) => {
 		const work: Promise<unknown>[] = [];
 		if (item.transition === 'opened' || item.transition === 'resolved') {
-			const kind: AlertTransition = item.transition;
-			work.push(
-				dispatchNotification(
-					env,
-					{
-						monitor: { id: item.monitor.id, name: item.monitor.name, url: item.monitor.url },
-						kind: kind === 'opened' ? 'down' : 'recovered',
-						incidentId: null,
-						title: kind === 'opened' ? `${item.monitor.name} is down` : `${item.monitor.name} recovered`,
-						body: item.result.error,
-						statusCode: item.result.statusCode,
-						error: item.result.error,
-						at: item.checkedAt,
-					},
-					notificationBudget,
-				),
-			);
+			work.push(dispatchNotification(env, buildAlertEvent(item.monitor, item.result, item.transition, item.checkedAt), notificationBudget));
 		}
 		if (item.latencyTransition) {
-			const degraded = item.latencyTransition === 'degraded';
 			work.push(
-				dispatchNotification(
-					env,
-					{
-						monitor: { id: item.monitor.id, name: item.monitor.name, url: item.monitor.url },
-						kind: degraded ? 'degraded' : 'recovered_degraded',
-						incidentId: null,
-						title: degraded ? `${item.monitor.name} performance degraded` : `${item.monitor.name} performance recovered`,
-						body: degraded ? `Response time was ${item.result.latencyMs} ms.` : 'Response time returned to normal.',
-						statusCode: item.result.statusCode,
-						error: item.result.error,
-						at: item.checkedAt,
-					},
-					notificationBudget,
-				),
+				dispatchNotification(env, buildAlertEvent(item.monitor, item.result, item.latencyTransition, item.checkedAt), notificationBudget),
 			);
 		}
-		if (item.transition === 'opened' && item.monitor.alertsEnabled && aiMessagesQueued < MAX_AI_MESSAGES_PER_RUN) {
-			aiMessagesQueued += 1;
-			work.push(generateIncidentMessage(env, { monitor: item.monitor, result: item.result }));
-		}
 		return work;
 	});
 	if (notifications.length > 0) {
@@ -126,6 +104,15 @@ export async function runDueChecks(env: Env, ctx?: Pick<ExecutionContext, 'waitU
 	}
 
 	const up = completed.reduce((count, item) => count + Number(item.result.ok), 0);
+	const events: AutopilotEvent[] = persisted
+		.filter((item) => item.transition === 'opened' || item.transition === 'resolved' || item.latencyTransition !== null)
+		.map((item) => ({
+			monitor: item.monitor,
+			result: item.result,
+			transition: item.transition === 'opened' || item.transition === 'resolved' ? item.transition : null,
+			latencyTransition: item.latencyTransition,
+			checkedAt: item.checkedAt,
+		}));
 	return {
 		checked: completed.length,
 		up,
@@ -133,5 +120,6 @@ export async function runDueChecks(env: Env, ctx?: Pick<ExecutionContext, 'waitU
 		pending: persisted.reduce((count, item) => count + Number(item.transition === 'pending'), 0),
 		opened: persisted.reduce((count, item) => count + Number(item.transition === 'opened'), 0),
 		retries: completed.reduce((count, item) => count + item.result.attempts - 1, 0),
+		events,
 	};
 }

src/worker/db/schema.ts

@@ -13,6 +13,11 @@ export const aiSettings = sqliteTable('ai_settings', {
 	baseUrl: text('base_url'),
 	apiKey: text('api_key'),
 	model: text('model'),
+	autopilotEnabled: integer('autopilot_enabled', { mode: 'boolean' }).notNull().default(false),
+	autopilotFollowupMinutes: integer('autopilot_followup_minutes').notNull().default(15),
+	autopilotMaxUpdates: integer('autopilot_max_updates').notNull().default(6),
+	autopilotAdvanceStatus: integer('autopilot_advance_status', { mode: 'boolean' }).notNull().default(false),
+	autopilotDegradedIncidents: integer('autopilot_degraded_incidents', { mode: 'boolean' }).notNull().default(false),
 	createdAt: integer('created_at', { mode: 'timestamp_ms' }).notNull(),
 	updatedAt: integer('updated_at', { mode: 'timestamp_ms' }).notNull(),
 });
@@ -134,6 +139,7 @@ export const incidents = sqliteTable(
 		status: text('status').notNull().default('investigating'),
 		impact: text('impact').notNull().default('major'),
 		source: text('source').notNull().default('auto'),
+		kind: text('kind').notNull().default('down'),
 		startedAt: integer('started_at', { mode: 'timestamp_ms' }).notNull(),
 		resolvedAt: integer('resolved_at', { mode: 'timestamp_ms' }),
 		startStatusCode: integer('start_status_code'),
@@ -174,6 +180,26 @@ export const incidentUpdates = sqliteTable(
 	(table) => [index('incident_updates_incident_id_created_at_idx').on(table.incidentId, table.createdAt)],
 );
 
+export const aiEvents = sqliteTable(
+	'ai_events',
+	{
+		id: integer('id').primaryKey({ autoIncrement: true }),
+		kind: text('kind').notNull(),
+		incidentId: integer('incident_id').references(() => incidents.id, { onDelete: 'set null' }),
+		monitorId: integer('monitor_id').references(() => monitors.id, { onDelete: 'set null' }),
+		model: text('model'),
+		outcome: text('outcome').notNull(),
+		reason: text('reason'),
+		latencyMs: integer('latency_ms'),
+		promptTokens: integer('prompt_tokens'),
+		completionTokens: integer('completion_tokens'),
+		contextPreview: text('context_preview'),
+		outputPreview: text('output_preview'),
+		createdAt: integer('created_at', { mode: 'timestamp_ms' }).notNull(),
+	},
+	(table) => [index('ai_events_created_at_idx').on(table.createdAt)],
+);
+
 export const notificationChannels = sqliteTable(
 	'notification_channels',
 	{

src/worker/index.ts

@@ -1,5 +1,6 @@
 import { Hono } from 'hono';
 import { csrf } from 'hono/csrf';
+import { runAutopilot } from './autopilot';
 import { runDueChecks } from './checks/run-due-checks';
 import authRoutes from './routes/auth';
 import channelRoutes from './routes/channels';
@@ -53,12 +54,18 @@ export default {
 
 		await cleanupExpiredAuthRecords(env);
 		const result = await runDueChecks(env, ctx);
+		ctx.waitUntil(
+			runAutopilot(env, { events: result.events }).then((autopilot) => {
+				console.log(JSON.stringify({ message: 'autopilot run completed', ...autopilot }));
+			}),
+		);
 		console.log(
 			JSON.stringify({
 				message: 'scheduled run completed',
 				cron: controller.cron,
 				scheduledTime: controller.scheduledTime,
 				...result,
+				events: result.events.length,
 			}),
 		);
 	},

src/worker/lib/humanize.ts

@@ -0,0 +1,48 @@
+import type { CheckResult, Monitor } from '../checks/run-check';
+
+export function humanizeInterval(seconds: number): string {
+	if (seconds >= 3600 && seconds % 3600 === 0) return `${seconds / 3600} h`;
+	if (seconds >= 60 && seconds % 60 === 0) return `${seconds / 60} min`;
+	return `${seconds} s`;
+}
+
+export function humanizeDuration(ms: number): string {
+	const minutes = Math.round(ms / 60_000);
+	if (minutes < 1) return 'under a minute';
+	if (minutes < 60) return `${minutes} min`;
+	const hours = Math.floor(minutes / 60);
+	const rest = minutes % 60;
+	return rest ? `${hours} h ${rest} min` : `${hours} h`;
+}
+
+export function classifyFailure(monitor: Pick<Monitor, 'expectedStatus'>, result: CheckResult): string {
+	if (result.statusCode === null) {
+		const error = (result.error ?? '').toLowerCase();
+		if (/tim(?:e|ed)\s?out|timeout|deadline|aborted/.test(error))
+			return 'Request timed out - no response came back before the timeout limit';
+		if (/getaddrinfo|enotfound|dns|name not resolved|could not resolve|eai_again/.test(error)) {
+			return 'DNS lookup failed - the hostname could not be resolved to an address';
+		}
+		if (/certificate|cert(?:\s|_)|self[- ]signed|tls|ssl|handshake|err_cert/.test(error)) {
+			return 'TLS/SSL failure - the certificate is invalid, expired, or untrusted';
+		}
+		if (/econnrefused|connection refused|refused to connect/.test(error)) {
+			return 'Connection refused - nothing is accepting connections at that address';
+		}
+		if (/econnreset|connection reset|socket hang up|premature close/.test(error))
+			return 'Connection dropped before any response was returned';
+		if (/ehostunreach|enetunreach|network is unreachable|no route to host/.test(error)) {
+			return 'Network unreachable - the host could not be contacted at all';
+		}
+		return 'No HTTP response was received from the endpoint';
+	}
+
+	const code = result.statusCode;
+	if (code >= 500) return `Server error - the endpoint answered with HTTP ${code}`;
+	if (code === 429) return 'The endpoint is rate limiting - HTTP 429 Too Many Requests';
+	if (code === 401 || code === 403) return `The endpoint rejected the health check as unauthorized - HTTP ${code}`;
+	if (code === 404) return 'The health-check path returned HTTP 404 Not Found';
+	if (code >= 400) return `Client error - the endpoint answered with HTTP ${code}`;
+	if (code >= 300) return `Unexpected redirect - the endpoint answered with HTTP ${code}`;
+	return `The endpoint answered with HTTP ${code}, but HTTP ${monitor.expectedStatus} was expected`;
+}

src/worker/notifications/compose.ts

@@ -0,0 +1,37 @@
+import type { CheckResult, Monitor } from '../checks/run-check';
+import type { AlertTransition, LatencyTransition } from '../checks/persist-result';
+import type { NotificationEvent } from './providers';
+
+type MonitorIdentity = Pick<Monitor, 'id' | 'name' | 'url'>;
+
+export function buildAlertEvent(
+	monitor: MonitorIdentity,
+	result: CheckResult,
+	transition: AlertTransition | LatencyTransition,
+	checkedAt: Date,
+): NotificationEvent {
+	if (transition === 'opened' || transition === 'resolved') {
+		const opened = transition === 'opened';
+		return {
+			monitor,
+			kind: opened ? 'down' : 'recovered',
+			incidentId: null,
+			title: opened ? `${monitor.name} is down` : `${monitor.name} recovered`,
+			body: result.error,
+			statusCode: result.statusCode,
+			error: result.error,
+			at: checkedAt,
+		};
+	}
+	const degraded = transition === 'degraded';
+	return {
+		monitor,
+		kind: degraded ? 'degraded' : 'recovered_degraded',
+		incidentId: null,
+		title: degraded ? `${monitor.name} performance degraded` : `${monitor.name} performance recovered`,
+		body: degraded ? `Response time was ${result.latencyMs} ms.` : 'Response time returned to normal.',
+		statusCode: result.statusCode,
+		error: result.error,
+		at: checkedAt,
+	};
+}

src/worker/notifications/dispatch.ts

@@ -1,6 +1,7 @@
-import { and, desc, eq, inArray, isNotNull, isNull } from 'drizzle-orm';
+import { and, eq, inArray } from 'drizzle-orm';
+import { findLatestAutoIncidentForMonitor } from '../autopilot/signal';
 import { getDb } from '../db/client';
-import { incidents, incidentMonitors, notificationChannelMonitors, notificationChannels, notificationDeliveries } from '../db/schema';
+import { incidentMonitors, notificationChannelMonitors, notificationChannels, notificationDeliveries } from '../db/schema';
 import {
 	CHANNEL_TYPES,
 	formatChannel,
@@ -69,19 +70,10 @@ export async function dispatchNotification(env: Env, event: NotificationEvent, b
 	const db = getDb(env);
 	let effectiveEvent = event;
 	if (event.incidentId === null && event.monitor && (event.kind === 'down' || event.kind === 'recovered')) {
-		const [incident] = await db
-			.select({ id: incidents.id })
-			.from(incidents)
-			.innerJoin(incidentMonitors, eq(incidentMonitors.incidentId, incidents.id))
-			.where(
-				and(
-					eq(incidentMonitors.monitorId, event.monitor.id),
-					eq(incidents.source, 'auto'),
-					event.kind === 'down' ? isNull(incidents.resolvedAt) : isNotNull(incidents.resolvedAt),
-				),
-			)
-			.orderBy(desc(incidents.updatedAt))
-			.limit(1);
+		const incident = await findLatestAutoIncidentForMonitor(db, event.monitor.id, {
+			resolved: event.kind === 'recovered',
+			kind: 'down',
+		});
 		if (incident) effectiveEvent = { ...event, incidentId: incident.id };
 	}
 	let targetMonitorIds = event.monitor ? [event.monitor.id] : [];

src/worker/routes/incidents.ts

@@ -267,6 +267,9 @@ incidentRoutes.patch('/:id', async (context) => {
 		);
 	}
 	if (parsed.value.monitorIds) {
+		if (parsed.value.title === undefined && parsed.value.impact === undefined) {
+			statements.push(db.update(incidents).set({ updatedAt: new Date() }).where(eq(incidents.id, id)));
+		}
 		statements.push(db.delete(incidentMonitors).where(eq(incidentMonitors.incidentId, id)));
 		statements.push(...parsed.value.monitorIds.map((monitorId) => db.insert(incidentMonitors).values({ incidentId: id, monitorId })));
 	}

src/worker/routes/monitors.ts

@@ -1,15 +1,16 @@
 import { and, desc, eq, gte, isNull, or, sql } from 'drizzle-orm';
 import { Hono } from 'hono';
-import { generateIncidentMessage } from '../ai/incident-message';
+import { runAutopilot } from '../autopilot';
 import { buildResultStatements } from '../checks/persist-result';
 import { MAX_RETRY_COUNT, runCheckWithRetries } from '../checks/run-check';
 import { getDb } from '../db/client';
-import { checks, incidentMonitors, incidents, maintenanceWindowMonitors, monitors } from '../db/schema';
+import { aiSettings, checks, incidentMonitors, incidents, maintenanceWindowMonitors, monitors } from '../db/schema';
 import { requireAuth, type AuthVariables } from '../lib/require-auth';
 import { loadActiveMaintenance } from '../maintenance/windows';
 import { isSafeRemoteUrl } from '../lib/safe-url';
 import { readBodyLimited } from '../lib/read-body';
 import { dispatchNotification } from '../notifications/dispatch';
+import { buildAlertEvent } from '../notifications/compose';
 
 type MonitorMethod = 'GET' | 'HEAD' | 'POST';
 
@@ -600,42 +601,35 @@ monitorRoutes.post('/:id/check', async (context) => {
 	const result = await runCheckWithRetries(monitor);
 	const checkedAt = new Date();
 	const activeMaintenance = await loadActiveMaintenance(db, checkedAt);
+	const [settings] = await db.select().from(aiSettings).where(eq(aiSettings.id, 1)).limit(1);
 	const { statements, transition, latencyTransition } = buildResultStatements(
 		db,
 		monitor,
 		result,
 		checkedAt,
 		activeMaintenance.has(monitor.id),
+		{ degradedIncidents: Boolean(settings?.autopilotEnabled && settings.autopilotDegradedIncidents) },
 	);
 	await db.batch(statements as [(typeof statements)[number], ...typeof statements]);
 	if (transition === 'opened' || transition === 'resolved') {
-		await dispatchNotification(context.env, {
-			monitor: { id: monitor.id, name: monitor.name, url: monitor.url },
-			kind: transition === 'opened' ? 'down' : 'recovered',
-			incidentId: null,
-			title: transition === 'opened' ? `${monitor.name} is down` : `${monitor.name} recovered`,
-			body: result.error,
-			statusCode: result.statusCode,
-			error: result.error,
-			at: checkedAt,
-		});
-		if (transition === 'opened') {
-			await generateIncidentMessage(context.env, { monitor, result });
-		}
+		await dispatchNotification(context.env, buildAlertEvent(monitor, result, transition, checkedAt));
 	}
 	if (latencyTransition) {
-		const degraded = latencyTransition === 'degraded';
-		await dispatchNotification(context.env, {
-			monitor: { id: monitor.id, name: monitor.name, url: monitor.url },
-			kind: degraded ? 'degraded' : 'recovered_degraded',
-			incidentId: null,
-			title: degraded ? `${monitor.name} performance degraded` : `${monitor.name} performance recovered`,
-			body: degraded ? `Response time was ${result.latencyMs} ms.` : 'Response time returned to normal.',
-			statusCode: result.statusCode,
-			error: result.error,
-			at: checkedAt,
-		});
+		await dispatchNotification(context.env, buildAlertEvent(monitor, result, latencyTransition, checkedAt));
 	}
+	await runAutopilot(context.env, {
+		events: [
+			{
+				monitor,
+				result,
+				transition: transition === 'opened' || transition === 'resolved' ? transition : null,
+				latencyTransition,
+				checkedAt,
+			},
+		],
+		budget: { remaining: 1 },
+		skipSweep: true,
+	});
 	const [updated] = await db.select().from(monitors).where(eq(monitors.id, monitor.id)).limit(1);
 
 	return context.json({ result, transition, latencyTransition, monitor: updated });

src/worker/routes/settings.ts

@@ -1,10 +1,11 @@
-import { eq } from 'drizzle-orm';
+import { desc, eq, gte, sql } from 'drizzle-orm';
 import { Hono } from 'hono';
-import { requestCompletion } from '../ai/client';
+import { requestCompletionDetailed } from '../ai/client';
+import { recordAiEvent } from '../ai/events';
 import { SAMPLE_INCIDENT_CONTEXT } from '../ai/incident-context';
 import { INCIDENT_MESSAGE_SYSTEM_PROMPT, sanitizeIncidentMessage } from '../ai/incident-message';
 import { getDb } from '../db/client';
-import { aiSettings } from '../db/schema';
+import { aiEvents, aiSettings } from '../db/schema';
 import { requireAuth, type AuthVariables } from '../lib/require-auth';
 import { isSafeRemoteUrl } from '../lib/safe-url';
 
@@ -13,14 +14,30 @@ type AiInput = {
 	baseUrl: string | null;
 	model: string | null;
 	apiKey?: string;
+	autopilotEnabled: boolean;
+	autopilotFollowupMinutes: number;
+	autopilotMaxUpdates: number;
+	autopilotAdvanceStatus: boolean;
+	autopilotDegradedIncidents: boolean;
 };
 
+function clampInteger(value: unknown, fallback: number, minimum: number, maximum: number) {
+	return typeof value === 'number' && Number.isSafeInteger(value) ? Math.min(maximum, Math.max(minimum, value)) : fallback;
+}
+
 function parseAiInput(value: unknown): AiInput | string {
 	if (typeof value !== 'object' || value === null || Array.isArray(value)) {
 		return 'Invalid request body';
 	}
 	const body = value as Record<string, unknown>;
 	if (typeof body.enabled !== 'boolean') return 'enabled must be a boolean';
+	if (body.autopilotEnabled !== undefined && typeof body.autopilotEnabled !== 'boolean') return 'autopilotEnabled must be a boolean';
+	if (body.autopilotAdvanceStatus !== undefined && typeof body.autopilotAdvanceStatus !== 'boolean') {
+		return 'autopilotAdvanceStatus must be a boolean';
+	}
+	if (body.autopilotDegradedIncidents !== undefined && typeof body.autopilotDegradedIncidents !== 'boolean') {
+		return 'autopilotDegradedIncidents must be a boolean';
+	}
 
 	const rawBaseUrl = typeof body.baseUrl === 'string' ? body.baseUrl.trim() : body.baseUrl;
 	if (rawBaseUrl !== null && typeof rawBaseUrl !== 'string') return 'baseUrl must be a URL or null';
@@ -46,7 +63,17 @@ function parseAiInput(value: unknown): AiInput | string {
 		apiKey = body.apiKey.trim();
 	}
 
-	return { enabled: body.enabled, baseUrl, model, apiKey };
+	return {
+		enabled: body.enabled,
+		baseUrl,
+		model,
+		apiKey,
+		autopilotEnabled: body.autopilotEnabled ?? false,
+		autopilotFollowupMinutes: clampInteger(body.autopilotFollowupMinutes, 15, 5, 240),
+		autopilotMaxUpdates: clampInteger(body.autopilotMaxUpdates, 6, 1, 20),
+		autopilotAdvanceStatus: body.autopilotAdvanceStatus ?? false,
+		autopilotDegradedIncidents: body.autopilotDegradedIncidents ?? false,
+	};
 }
 
 function publicAiSettings(settings: typeof aiSettings.$inferSelect | undefined) {
@@ -55,6 +82,11 @@ function publicAiSettings(settings: typeof aiSettings.$inferSelect | undefined)
 		enabled: settings?.enabled ?? false,
 		baseUrl: settings?.baseUrl ?? null,
 		model: settings?.model ?? null,
+		autopilotEnabled: settings?.autopilotEnabled ?? false,
+		autopilotFollowupMinutes: settings?.autopilotFollowupMinutes ?? 15,
+		autopilotMaxUpdates: settings?.autopilotMaxUpdates ?? 6,
+		autopilotAdvanceStatus: settings?.autopilotAdvanceStatus ?? false,
+		autopilotDegradedIncidents: settings?.autopilotDegradedIncidents ?? false,
 		apiKeySet: Boolean(settings?.apiKey),
 		apiKeyPreview: settings?.apiKey ? `••••••${settings.apiKey.slice(-4)}` : null,
 		createdAt: settings?.createdAt ?? null,
@@ -88,12 +120,28 @@ settingsRoutes.put('/ai', async (context) => {
 	if (input.enabled && !apiKey) return context.json({ message: 'An API key is required when AI messages are enabled' }, 400);
 
 	const now = new Date();
+	const autopilot = {
+		autopilotEnabled: input.autopilotEnabled,
+		autopilotFollowupMinutes: input.autopilotFollowupMinutes,
+		autopilotMaxUpdates: input.autopilotMaxUpdates,
+		autopilotAdvanceStatus: input.autopilotAdvanceStatus,
+		autopilotDegradedIncidents: input.autopilotDegradedIncidents,
+	};
 	const [settings] = await db
 		.insert(aiSettings)
-		.values({ id: 1, enabled: input.enabled, baseUrl: input.baseUrl, apiKey, model: input.model, createdAt: now, updatedAt: now })
+		.values({
+			id: 1,
+			enabled: input.enabled,
+			baseUrl: input.baseUrl,
+			apiKey,
+			model: input.model,
+			...autopilot,
+			createdAt: now,
+			updatedAt: now,
+		})
 		.onConflictDoUpdate({
 			target: aiSettings.id,
-			set: { enabled: input.enabled, baseUrl: input.baseUrl, apiKey, model: input.model, updatedAt: now },
+			set: { enabled: input.enabled, baseUrl: input.baseUrl, apiKey, model: input.model, ...autopilot, updatedAt: now },
 		})
 		.returning();
 	return context.json({ settings: publicAiSettings(settings) });
@@ -104,14 +152,53 @@ settingsRoutes.post('/ai/test', async (context) => {
 	if (!settings?.baseUrl || !settings.apiKey || !settings.model) {
 		return context.json({ message: 'Save a base URL, API key, and model first' }, 400);
 	}
-	const content = await requestCompletion(
+	const result = await requestCompletionDetailed(
 		{ baseUrl: settings.baseUrl, apiKey: settings.apiKey, model: settings.model },
 		INCIDENT_MESSAGE_SYSTEM_PROMPT,
 		SAMPLE_INCIDENT_CONTEXT,
 	);
-	const message = content ? sanitizeIncidentMessage(content) : null;
+	const message = result.content ? sanitizeIncidentMessage(result.content) : null;
+	await recordAiEvent(context.env, {
+		kind: 'settings_test',
+		model: settings.model,
+		outcome: message ? 'ok' : result.failure ? 'failed' : 'rejected',
+		reason: result.failure ?? (message ? null : 'sanitizer_rejected'),
+		latencyMs: result.latencyMs,
+		promptTokens: result.promptTokens,
+		completionTokens: result.completionTokens,
+		contextPreview: SAMPLE_INCIDENT_CONTEXT,
+		outputPreview: result.content,
+	});
 	if (!message) return context.json({ message: 'AI message generation failed' }, 502);
 	return context.json({ ok: true, message });
 });
 
+settingsRoutes.get('/ai/events', async (context) => {
+	const rawLimit = Number(context.req.query('limit') ?? 50);
+	const limit = Number.isSafeInteger(rawLimit) ? Math.min(100, Math.max(1, rawLimit)) : 50;
+	const db = getDb(context.env);
+	const events = await db.select().from(aiEvents).orderBy(desc(aiEvents.createdAt)).limit(limit);
+	const since = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000);
+	const [summary] = await db
+		.select({
+			total: sql<number>`count(*)`,
+			ok: sql<number>`coalesce(sum(case when ${aiEvents.outcome} = 'ok' then 1 else 0 end), 0)`,
+			averageLatencyMs: sql<number | null>`round(avg(${aiEvents.latencyMs}))`,
+			promptTokens: sql<number>`coalesce(sum(${aiEvents.promptTokens}), 0)`,
+			completionTokens: sql<number>`coalesce(sum(${aiEvents.completionTokens}), 0)`,
+		})
+		.from(aiEvents)
+		.where(gte(aiEvents.createdAt, since));
+	return context.json({
+		events: events.map((event) => ({ ...event, createdAt: event.createdAt.toISOString() })),
+		summary: {
+			total: Number(summary?.total ?? 0),
+			ok: Number(summary?.ok ?? 0),
+			averageLatencyMs: summary?.averageLatencyMs === null ? null : Number(summary?.averageLatencyMs ?? 0),
+			promptTokens: Number(summary?.promptTokens ?? 0),
+			completionTokens: Number(summary?.completionTokens ?? 0),
+		},
+	});
+});
+
 export default settingsRoutes;

src/worker/routes/status.ts

@@ -23,6 +23,7 @@ type PublicIncident = {
 	status: string;
 	impact: string;
 	source: string;
+	kind: string;
 	startedAt: Date;
 	resolvedAt: Date | null;
 	durationMs: number | null;
@@ -103,6 +104,7 @@ const incidentSelection = {
 	status: incidents.status,
 	impact: incidents.impact,
 	source: incidents.source,
+	kind: incidents.kind,
 	startedAt: incidents.startedAt,
 	resolvedAt: incidents.resolvedAt,
 	durationMs: incidents.durationMs,
@@ -177,10 +179,11 @@ statusRoutes.get('/', async (context) => {
 		if (buckets) buckets.push(row);
 		else bucketsByMonitor.set(row.monitorId, [row]);
 	}
-	const activeIncidentByMonitor = new Map<number, PublicIncident>();
+	const downIncidentByMonitor = new Map<number, PublicIncident>();
+	const degradedIncidentByMonitor = new Map<number, PublicIncident>();
 	for (const incident of activeIncidentRows) {
-		for (const service of incidentServices.get(incident.id) ?? [])
-			if (!activeIncidentByMonitor.has(service.id)) activeIncidentByMonitor.set(service.id, incident);
+		const target = incident.kind === 'degraded' ? degradedIncidentByMonitor : downIncidentByMonitor;
+		for (const service of incidentServices.get(incident.id) ?? []) if (!target.has(service.id)) target.set(service.id, incident);
 	}
 	const services = monitorRows.map((monitor) => {
 		const buckets = bucketsByMonitor.get(monitor.id) ?? [];
@@ -194,7 +197,8 @@ statusRoutes.get('/', async (context) => {
 				uptimePct: roundUptime(bucket.upChecks, bucket.totalChecks),
 			};
 		});
-		const incident = activeIncidentByMonitor.get(monitor.id);
+		const downIncident = downIncidentByMonitor.get(monitor.id);
+		const degradedIncident = degradedIncidentByMonitor.get(monitor.id);
 		const maintenance = activeMaintenance.get(monitor.id);
 		return {
 			id: monitor.id,
@@ -202,11 +206,13 @@ statusRoutes.get('/', async (context) => {
 			status: maintenance ? ('maintenance' as const) : serviceStatus(monitor.lastOk, monitor.lastDegraded),
 			message:
 				!maintenance && monitor.lastOk === false
-					? incident
-						? (latestUpdates.get(incident.id)?.body ?? deterministicIncidentMessage(incident.startStatusCode))
+					? downIncident
+						? (latestUpdates.get(downIncident.id)?.body ?? deterministicIncidentMessage(downIncident.startStatusCode))
 						: deterministicIncidentMessage(monitor.lastStatusCode)
 					: !maintenance && monitor.lastOk === true && monitor.lastDegraded
-						? DEGRADED_MESSAGE
+						? degradedIncident
+							? (latestUpdates.get(degradedIncident.id)?.body ?? DEGRADED_MESSAGE)
+							: DEGRADED_MESSAGE
 						: null,
 			maintenance: maintenance ? { name: maintenance.name, endsAt: maintenance.endsAt.toISOString() } : null,
 			lastCheckedAt: monitor.lastCheckedAt?.toISOString() ?? null,

src/worker/scheduled/cleanup.ts

@@ -1,11 +1,12 @@
 import { lt } from 'drizzle-orm';
 import { getDb } from '../db/client';
-import { checks, loginAttempts, monitorDailyStats, notificationDeliveries, sessions } from '../db/schema';
+import { aiEvents, checks, loginAttempts, monitorDailyStats, notificationDeliveries, sessions } from '../db/schema';
 
 const LOGIN_ATTEMPT_RETENTION_MS = 60 * 60 * 1000;
 const CHECK_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
 const DAILY_STATS_RETENTION_MS = 400 * 24 * 60 * 60 * 1000;
 const DELIVERY_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
+export const AI_EVENT_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
 
 export async function cleanupExpiredAuthRecords(env: Env) {
 	const db = getDb(env);
@@ -17,5 +18,6 @@ export async function cleanupExpiredAuthRecords(env: Env) {
 		db.delete(checks).where(lt(checks.checkedAt, new Date(now.getTime() - CHECK_RETENTION_MS))),
 		db.delete(monitorDailyStats).where(lt(monitorDailyStats.day, new Date(now.getTime() - DAILY_STATS_RETENTION_MS))),
 		db.delete(notificationDeliveries).where(lt(notificationDeliveries.createdAt, new Date(now.getTime() - DELIVERY_RETENTION_MS))),
+		db.delete(aiEvents).where(lt(aiEvents.createdAt, new Date(now.getTime() - AI_EVENT_RETENTION_MS))),
 	]);
 }

test/ai.spec.ts

@@ -36,6 +36,7 @@ const failedResult = {
 
 async function resetDatabase() {
 	await env.DB.batch([
+		env.DB.prepare('DELETE FROM ai_events'),
 		env.DB.prepare('DELETE FROM checks'),
 		env.DB.prepare('DELETE FROM incident_updates'),
 		env.DB.prepare('DELETE FROM incident_monitors'),

test/autopilot.spec.ts

@@ -0,0 +1,81 @@
+import { applyD1Migrations, type D1Migration } from 'cloudflare:test';
+import { env } from 'cloudflare:workers';
+import { beforeAll, describe, expect, it } from 'vitest';
+import { advanceStatus, computeImpact, nextFollowupDueAt } from '../src/worker/autopilot/cadence';
+import { recordAiEvent } from '../src/worker/ai/events';
+import { sanitizePublicTextWithReason } from '../src/worker/ai/sanitize';
+import { AI_EVENT_RETENTION_MS, cleanupExpiredAuthRecords } from '../src/worker/scheduled/cleanup';
+
+describe('incident autopilot', () => {
+	beforeAll(async () => {
+		const testEnv = env as Env & { TEST_MIGRATIONS: D1Migration[] };
+		await applyD1Migrations(testEnv.DB, testEnv.TEST_MIGRATIONS);
+	});
+
+	it('uses exponential cadence capped at eight times the configured interval', () => {
+		const start = 1_000_000;
+		expect(nextFollowupDueAt(start, 1, 15)).toBe(start + 15 * 60_000);
+		expect(nextFollowupDueAt(start, 2, 15)).toBe(start + 30 * 60_000);
+		expect(nextFollowupDueAt(start, 4, 15)).toBe(start + 120 * 60_000);
+		expect(nextFollowupDueAt(start, 9, 15)).toBe(start + 120 * 60_000);
+	});
+
+	it('advances only from objective signals', () => {
+		expect(
+			advanceStatus('investigating', {
+				consecutiveFailures: 3,
+				failureSignatureStable: true,
+				consecutiveOk: 0,
+				latestOk: false,
+				regressionUsed: false,
+			}),
+		).toBe('identified');
+		expect(
+			advanceStatus('identified', {
+				consecutiveFailures: 0,
+				failureSignatureStable: false,
+				consecutiveOk: 2,
+				latestOk: true,
+				regressionUsed: false,
+			}),
+		).toBe('monitoring');
+	});
+
+	it('computes impact deterministically', () => {
+		expect(computeImpact({ kind: 'degraded', affectedMonitors: 5, totalMonitors: 5, recentChecks: [] })).toBe('minor');
+		expect(computeImpact({ kind: 'down', affectedMonitors: 2, totalMonitors: 4, recentChecks: [false] })).toBe('critical');
+		expect(computeImpact({ kind: 'down', affectedMonitors: 1, totalMonitors: 4, recentChecks: [false, true] })).toBe('minor');
+	});
+
+	it('returns actionable sanitizer rejection reasons', () => {
+		expect(sanitizePublicTextWithReason('See https://private.example', 200)).toEqual({ text: null, reason: 'contains_url' });
+		expect(sanitizePublicTextWithReason('HTTP 503 is occurring', 200)).toEqual({ text: null, reason: 'contains_http_status' });
+		expect(sanitizePublicTextWithReason('A calm public update.', 200)).toEqual({ text: 'A calm public update.', reason: null });
+	});
+
+	it('removes AI audit rows after the retention period', async () => {
+		await env.DB.prepare('DELETE FROM ai_events').run();
+		await env.DB.prepare("INSERT INTO ai_events (kind, outcome, created_at) VALUES ('settings_test', 'ok', ?)")
+			.bind(Date.now() - AI_EVENT_RETENTION_MS - 1)
+			.run();
+		await cleanupExpiredAuthRecords(env);
+		expect((await env.DB.prepare('SELECT count(*) AS count FROM ai_events').first<{ count: number }>())?.count).toBe(0);
+	});
+
+	it('redacts sensitive values from AI audit previews', async () => {
+		await env.DB.prepare('DELETE FROM ai_events').run();
+		await recordAiEvent(env, {
+			kind: 'settings_test',
+			outcome: 'rejected',
+			reason: 'Failed at https://private.example.com with sk-secret-4f2a',
+			contextPreview: 'Probe https://api.example.com from 10.0.0.1 using Bearer private-token',
+			outputPreview: 'Visit status.example.com with sk-output-secret',
+		});
+		const text = JSON.stringify(await env.DB.prepare('SELECT reason, context_preview, output_preview FROM ai_events').first());
+		expect(text).not.toContain('private.example.com');
+		expect(text).not.toContain('api.example.com');
+		expect(text).not.toContain('10.0.0.1');
+		expect(text).not.toContain('private-token');
+		expect(text).not.toContain('sk-secret-4f2a');
+	});
+});

test/channels.spec.ts

@@ -34,6 +34,7 @@ describe('notification channels', () => {
 	});
 	beforeEach(async () => {
 		await env.DB.batch([
+			env.DB.prepare('DELETE FROM ai_events'),
 			env.DB.prepare('DELETE FROM notification_deliveries'),
 			env.DB.prepare('DELETE FROM notification_channel_monitors'),
 			env.DB.prepare('DELETE FROM notification_channels'),

test/checks.spec.ts

@@ -1,10 +1,12 @@
 import { applyD1Migrations, type D1Migration } from 'cloudflare:test';
 import { env } from 'cloudflare:workers';
 import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import { runAutopilot } from '../src/worker/autopilot';
 import { MAX_RETRY_ATTEMPTS_PER_RUN, runDueChecks } from '../src/worker/checks/run-due-checks';
 
 async function clearMonitoringTables() {
 	await env.DB.batch([
+		env.DB.prepare('DELETE FROM ai_events'),
 		env.DB.prepare('DELETE FROM maintenance_window_monitors'),
 		env.DB.prepare('DELETE FROM maintenance_windows'),
 		env.DB.prepare('DELETE FROM checks'),
@@ -100,7 +102,7 @@ describe('scheduled monitor checks', () => {
 		);
 
 		const summary = await runDueChecks(env);
-		expect(summary).toEqual({ checked: 1, up: 1, down: 0, pending: 0, opened: 0, retries: 0 });
+		expect(summary).toEqual({ checked: 1, up: 1, down: 0, pending: 0, opened: 0, retries: 0, events: [] });
 
 		const monitor = await env.DB.prepare('SELECT last_ok, last_status_code, last_latency_ms, last_checked_at FROM monitors WHERE id = ?')
 			.bind(id)
@@ -213,7 +215,7 @@ describe('scheduled monitor checks', () => {
 		vi.stubGlobal('fetch', fetchMock);
 
 		const summary = await runDueChecks(env);
-		expect(summary).toEqual({ checked: 1, up: 0, down: 1, pending: 1, opened: 0, retries: 0 });
+		expect(summary).toEqual({ checked: 1, up: 0, down: 1, pending: 1, opened: 0, retries: 0, events: [] });
 
 		const monitor = await env.DB.prepare('SELECT last_ok, consecutive_failures, last_status_code, last_error FROM monitors WHERE id = ?')
 			.bind(id)
@@ -338,7 +340,7 @@ describe('scheduled monitor checks', () => {
 				 VALUES ('Legacy webhook', 'webhook', '{"url":"https://hooks.example.test/events"}', 1, 1, ?, ?)`,
 			).bind(now, now),
 			env.DB.prepare(
-				"INSERT INTO ai_settings (id, enabled, base_url, api_key, model, created_at, updated_at) VALUES (1, 1, 'https://ai.example.test/v1', 'secret', 'test-model', ?, ?)",
+				"INSERT INTO ai_settings (id, enabled, base_url, api_key, model, autopilot_enabled, created_at, updated_at) VALUES (1, 1, 'https://ai.example.test/v1', 'secret', 'test-model', 1, ?, ?)",
 			).bind(now, now),
 		]);
 		const webhookBodies: Array<Record<string, unknown>> = [];
@@ -357,7 +359,8 @@ describe('scheduled monitor checks', () => {
 						choices: [
 							{
 								message: {
-									content: 'Some visitors may be unable to use the service. The team has been alerted and restoration work is underway.',
+									content:
+										'TITLE: Service disruption\nBODY: Some visitors may be unable to use the service. Automated monitoring continues while service is restored.',
 								},
 							},
 						],
@@ -373,7 +376,8 @@ describe('scheduled monitor checks', () => {
 		expect((await env.DB.prepare('SELECT COUNT(*) AS count FROM incident_updates').first<{ count: number }>())?.count).toBe(0);
 
 		await env.DB.prepare('UPDATE monitors SET last_checked_at = NULL WHERE id = ?').bind(id).run();
-		await runDueChecks(env);
+		const confirmed = await runDueChecks(env);
+		await runAutopilot(env, { events: confirmed.events });
 		expect(webhookBodies).toHaveLength(1);
 		expect(webhookBodies[0]).toMatchObject({ event: 'down' });
 		expect(aiCalls).toBe(1);
@@ -453,7 +457,7 @@ describe('scheduled monitor checks', () => {
 		vi.stubGlobal('fetch', fetchMock);
 
 		const summary = await runDueChecks(env);
-		expect(summary).toEqual({ checked: 0, up: 0, down: 0, pending: 0, opened: 0, retries: 0 });
+		expect(summary).toEqual({ checked: 0, up: 0, down: 0, pending: 0, opened: 0, retries: 0, events: [] });
 		expect(fetchMock).not.toHaveBeenCalled();
 	});
 });

test/incident-draft.spec.ts

@@ -6,6 +6,7 @@ import { hashPassword } from '../src/worker/lib/password';
 const PASSWORD = 'correct-horse-battery-staple';
 async function reset() {
 	await env.DB.batch([
+		env.DB.prepare('DELETE FROM ai_events'),
 		env.DB.prepare('DELETE FROM incident_updates'),
 		env.DB.prepare('DELETE FROM incident_monitors'),
 		env.DB.prepare('DELETE FROM incidents'),

test/incidents.spec.ts

@@ -6,6 +6,7 @@ import { hashPassword } from '../src/worker/lib/password';
 const PASSWORD = 'correct-horse-battery-staple';
 async function reset() {
 	await env.DB.batch([
+		env.DB.prepare('DELETE FROM ai_events'),
 		env.DB.prepare('DELETE FROM notification_deliveries'),
 		env.DB.prepare('DELETE FROM notification_channel_monitors'),
 		env.DB.prepare('DELETE FROM notification_channels'),
@@ -123,4 +124,23 @@ describe('incident lifecycle API', () => {
 		expect(await env.DB.prepare('SELECT id FROM incidents WHERE id = ?').bind(incidentId).first()).toBeTruthy();
 		expect(await env.DB.prepare('SELECT * FROM incident_monitors WHERE incident_id = ?').bind(incidentId).first()).toBeNull();
 	});
+	it('bumps updated_at when only monitor assignments change', async () => {
+		const firstId = await monitor('API');
+		const secondId = await monitor('Web');
+		const auth = await cookie();
+		const created = await request('/api/incidents', 'POST', auth, {
+			title: 'API issue',
+			impact: 'major',
+			status: 'investigating',
+			body: 'Some requests are failing.',
+			monitorIds: [firstId],
+		});
+		const incidentId = (await created.json<{ incident: { id: number } }>()).incident.id;
+		await env.DB.prepare('UPDATE incidents SET updated_at = 1 WHERE id = ?').bind(incidentId).run();
+		expect((await request(`/api/incidents/${incidentId}`, 'PATCH', auth, { monitorIds: [secondId] })).status).toBe(200);
+		const row = await env.DB.prepare('SELECT updated_at AS updatedAt FROM incidents WHERE id = ?')
+			.bind(incidentId)
+			.first<{ updatedAt: number }>();
+		expect(row?.updatedAt).toBeGreaterThan(1);
+	});
 });

test/maintenance.spec.ts

@@ -21,6 +21,7 @@ function windowRow(overrides: Partial<MaintenanceWindowRow> = {}): MaintenanceWi
 
 async function resetDatabase() {
 	await env.DB.batch([
+		env.DB.prepare('DELETE FROM ai_events'),
 		env.DB.prepare('DELETE FROM maintenance_window_monitors'),
 		env.DB.prepare('DELETE FROM maintenance_windows'),
 		env.DB.prepare('DELETE FROM checks'),

test/monitors.spec.ts

@@ -16,6 +16,7 @@ const VALID_MONITOR = {
 
 async function seedAdmin() {
 	await env.DB.batch([
+		env.DB.prepare('DELETE FROM ai_events'),
 		env.DB.prepare('DELETE FROM checks'),
 		env.DB.prepare('DELETE FROM incident_updates'),
 		env.DB.prepare('DELETE FROM incident_monitors'),

test/rollup.spec.ts

@@ -10,6 +10,7 @@ describe('daily monitor rollups', () => {
 	});
 	beforeEach(async () => {
 		await env.DB.batch([
+			env.DB.prepare('DELETE FROM ai_events'),
 			env.DB.prepare('DELETE FROM maintenance_window_monitors'),
 			env.DB.prepare('DELETE FROM maintenance_windows'),
 			env.DB.prepare('DELETE FROM monitor_daily_stats'),

test/status.spec.ts

@@ -37,6 +37,7 @@ type IncidentHistoryResponse = {
 
 async function resetDatabase() {
 	await env.DB.batch([
+		env.DB.prepare('DELETE FROM ai_events'),
 		env.DB.prepare('DELETE FROM maintenance_window_monitors'),
 		env.DB.prepare('DELETE FROM maintenance_windows'),
 		env.DB.prepare('DELETE FROM checks'),
@@ -165,6 +166,52 @@ describe('public status API', () => {
 		expect(body.services[0]).toMatchObject({ status: 'degraded', message: DEGRADED_MESSAGE });
 	});
 
+	it('uses an open degraded incident update as the public latency message', async () => {
+		const monitorId = await insertMonitor({ name: 'Slow API', lastOk: true, lastDegraded: true, lastLatencyMs: 2200 });
+		const now = Date.now();
+		const inserted = await env.DB.prepare(
+			"INSERT INTO incidents (status, impact, source, kind, started_at, created_at, updated_at) VALUES ('investigating', 'minor', 'auto', 'degraded', ?, ?, ?)",
+		)
+			.bind(now, now, now)
+			.run();
+		const incidentId = Number(inserted.meta.last_row_id);
+		await env.DB.batch([
+			env.DB.prepare('INSERT INTO incident_monitors (incident_id, monitor_id) VALUES (?, ?)').bind(incidentId, monitorId),
+			env.DB.prepare(
+				"INSERT INTO incident_updates (incident_id, status, body, source, created_at) VALUES (?, 'investigating', 'Performance remains slower than usual.', 'ai', ?)",
+			).bind(incidentId, now),
+		]);
+		const body = await (await statusFetch()).json<PublicStatusResponse>();
+		expect(body.services[0].message).toBe('Performance remains slower than usual.');
+	});
+
+	it('never shadows a down incident message with a degraded incident', async () => {
+		const monitorId = await insertMonitor({ name: 'API', lastOk: false, lastDegraded: false, lastStatusCode: 503 });
+		const now = Date.now();
+		const degraded = await env.DB.prepare(
+			"INSERT INTO incidents (status, impact, source, kind, started_at, created_at, updated_at) VALUES ('investigating', 'minor', 'auto', 'degraded', ?, ?, ?)",
+		)
+			.bind(now + 1, now + 1, now + 1)
+			.run();
+		const down = await env.DB.prepare(
+			"INSERT INTO incidents (status, impact, source, kind, started_at, start_status_code, created_at, updated_at) VALUES ('investigating', 'major', 'auto', 'down', ?, 503, ?, ?)",
+		)
+			.bind(now, now, now)
+			.run();
+		await env.DB.batch([
+			env.DB.prepare('INSERT INTO incident_monitors (incident_id, monitor_id) VALUES (?, ?)').bind(degraded.meta.last_row_id, monitorId),
+			env.DB.prepare('INSERT INTO incident_monitors (incident_id, monitor_id) VALUES (?, ?)').bind(down.meta.last_row_id, monitorId),
+			env.DB.prepare(
+				"INSERT INTO incident_updates (incident_id, status, body, source, created_at) VALUES (?, 'investigating', 'Degraded copy.', 'ai', ?)",
+			).bind(degraded.meta.last_row_id, now),
+			env.DB.prepare(
+				"INSERT INTO incident_updates (incident_id, status, body, source, created_at) VALUES (?, 'investigating', 'Down copy.', 'ai', ?)",
+			).bind(down.meta.last_row_id, now),
+		]);
+		const body = await (await statusFetch()).json<PublicStatusResponse>();
+		expect(body.services[0]).toMatchObject({ status: 'down', message: 'Down copy.' });
+	});
+
 	it('excludes disabled monitors and reports degraded health for a partial outage', async () => {
 		await insertMonitor({ name: 'Healthy', lastOk: true });
 		await insertMonitor({ name: 'Unavailable', lastOk: false });