diff --git a/src/client/queries/settings.ts b/src/client/queries/settings.ts
index d0687f3..fa53deb 100644
--- a/src/client/queries/settings.ts
+++ b/src/client/queries/settings.ts
@@ -1,10 +1,11 @@
import { useMutation, useQuery } from '@tanstack/react-query';
-import { getAiSettings, testAiSettings, updateAiSettings, type AiSettingsInput } from '../api/settings';
+import { getAiEvents, getAiSettings, testAiSettings, updateAiSettings, type AiSettingsInput } from '../api/settings';
import { queryClient } from '../lib/query-client';
export const settingsKeys = {
all: ['settings'] as const,
ai: () => [...settingsKeys.all, 'ai'] as const,
+ aiEvents: () => [...settingsKeys.all, 'ai-events'] as const,
};
export function useAiSettingsQuery() {
@@ -24,3 +25,10 @@ export function useUpdateAiSettingsMutation() {
export function useTestAiSettingsMutation() {
return useMutation({ mutationFn: testAiSettings });
}
+
+export function useAiEventsQuery() {
+ return useQuery({
+ queryKey: settingsKeys.aiEvents(),
+ queryFn: ({ signal }) => getAiEvents(signal),
+ });
+}
diff --git a/src/client/styles.css b/src/client/styles.css
index 998f0e5..371331d 100644
--- a/src/client/styles.css
+++ b/src/client/styles.css
@@ -1626,6 +1626,81 @@ button {
.channel-history-state {
color: var(--muted);
}
+.autopilot-settings {
+ display: grid;
+ gap: 16px;
+ margin: 4px 0;
+ padding: 20px;
+ border: 1px solid #e6e6e6;
+ border-radius: 8px;
+ background: #fafcfb;
+}
+.autopilot-settings legend {
+ padding: 0 7px;
+ font-size: 13px;
+ font-weight: 600;
+}
+.autopilot-number-fields {
+ display: grid;
+ grid-template-columns: repeat(2, minmax(0, 1fr));
+ gap: 16px;
+}
+.ai-activity-panel {
+ padding: 22px 26px 26px;
+}
+.ai-activity-stats {
+ display: grid;
+ grid-template-columns: repeat(4, minmax(0, 1fr));
+ gap: 10px;
+}
+.ai-activity-stats > div {
+ display: grid;
+ gap: 3px;
+ padding: 14px;
+ border: 1px solid #e6e6e6;
+ border-radius: 7px;
+ background: #fafafa;
+}
+.ai-activity-stats strong {
+ font-size: 19px;
+}
+.ai-activity-stats span,
+.ai-activity-row span,
+.ai-activity-row small {
+ font-size: 11px;
+ color: var(--muted);
+}
+.ai-activity-history {
+ display: grid;
+ gap: 1px;
+ margin-top: 18px;
+ border: 1px solid #e6e6e6;
+ border-radius: 6px;
+ overflow: hidden;
+ background: #e6e6e6;
+}
+.ai-activity-row {
+ display: grid;
+ grid-template-columns: 100px minmax(110px, 0.8fr) minmax(145px, 1fr) minmax(130px, 1fr) auto;
+ align-items: center;
+ gap: 12px;
+ padding: 10px 12px;
+ font-size: 11px;
+ background: #fff;
+}
+.ai-activity-row strong {
+ font-weight: 600;
+ text-transform: capitalize;
+}
+@media (max-width: 720px) {
+ .ai-activity-stats,
+ .autopilot-number-fields {
+ grid-template-columns: 1fr 1fr;
+ }
+ .ai-activity-row {
+ grid-template-columns: 1fr 1fr;
+ }
+}
.channel-history-state {
width: 100%;
margin-top: 16px;
diff --git a/src/worker/ai/autopilot-prompts.ts b/src/worker/ai/autopilot-prompts.ts
new file mode 100644
index 0000000..63b723c
--- /dev/null
+++ b/src/worker/ai/autopilot-prompts.ts
@@ -0,0 +1,33 @@
+export const AUTOPILOT_STATUS_GUIDANCE = {
+ investigating: 'State that automated monitoring continues to observe the disruption. Do not imply that the cause is known.',
+ identified:
+ 'State only that monitoring has consistently observed the same failure pattern. Do not claim a root cause or human diagnosis.',
+ monitoring:
+ 'State that service responses have recently improved and automated monitoring is checking that recovery remains stable. Do not claim a fix was applied.',
+} as const;
+
+const PUBLIC_RULES = [
+ 'Write for non-technical visitors in calm, factual language.',
+ 'Never include URLs, hostnames, domains, IP addresses, ports, paths, HTTP status codes, error codes, or stack traces.',
+ 'Never invent a root cause, a human action, or an estimated recovery time.',
+ 'Output only the requested fields.',
+].join('\n');
+
+export const INCIDENT_OPEN_SYSTEM_PROMPT = [
+ 'Create a concise public incident title and opening update.',
+ 'Use exactly this format: TITLE: one short title\nBODY: two short sentences under 240 characters total.',
+ PUBLIC_RULES,
+].join('\n');
+
+export const INCIDENT_FOLLOWUP_SYSTEM_PROMPT = [
+ 'Write one short public follow-up update under 240 characters.',
+ 'Describe only the observed visitor impact and current automated-monitoring state.',
+ PUBLIC_RULES,
+].join('\n');
+
+export const INCIDENT_RESOLVE_SYSTEM_PROMPT = [
+ 'Write one short public resolution update under 240 characters.',
+ 'Say that service has recovered, mention the supplied approximate duration, and say monitoring confirms normal operation.',
+ 'Do not claim that a fix was applied.',
+ PUBLIC_RULES,
+].join('\n');
diff --git a/src/worker/ai/client.ts b/src/worker/ai/client.ts
index 07e3ff5..fa54b10 100644
--- a/src/worker/ai/client.ts
+++ b/src/worker/ai/client.ts
@@ -6,14 +6,29 @@ export type CompletionSettings = {
type CompletionBody = {
choices?: Array<{ message?: { content?: unknown } }>;
+ usage?: { prompt_tokens?: number; completion_tokens?: number };
};
-export async function requestCompletion(
+export type CompletionResult = {
+ content: string | null;
+ latencyMs: number;
+ promptTokens: number | null;
+ completionTokens: number | null;
+ failure: string | null;
+};
+
+function failureMessage(error: unknown): string {
+ if (error instanceof DOMException && error.name === 'TimeoutError') return 'timeout';
+ return (error instanceof Error ? error.message : String(error)).slice(0, 200);
+}
+
+export async function requestCompletionDetailed(
settings: CompletionSettings,
system: string,
user: string,
maxTokens = 160,
-): Promise
{
+): Promise {
+ const startedAt = Date.now();
try {
const response = await fetch(`${settings.baseUrl}/chat/completions`, {
method: 'POST',
@@ -35,23 +50,52 @@ export async function requestCompletion(
if (!response.ok) {
await response.body?.cancel();
console.warn(JSON.stringify({ message: 'AI completion returned an error', status: response.status }));
- return null;
+ return {
+ content: null,
+ latencyMs: Date.now() - startedAt,
+ promptTokens: null,
+ completionTokens: null,
+ failure: `http_${response.status}`,
+ };
}
const body = (await response.json()) as CompletionBody;
const content = body.choices?.[0]?.message?.content;
if (typeof content !== 'string') {
console.warn(JSON.stringify({ message: 'AI completion returned malformed content' }));
- return null;
+ return {
+ content: null,
+ latencyMs: Date.now() - startedAt,
+ promptTokens: body.usage?.prompt_tokens ?? null,
+ completionTokens: body.usage?.completion_tokens ?? null,
+ failure: 'malformed',
+ };
}
- return content;
+ return {
+ content,
+ latencyMs: Date.now() - startedAt,
+ promptTokens: body.usage?.prompt_tokens ?? null,
+ completionTokens: body.usage?.completion_tokens ?? null,
+ failure: null,
+ };
} catch (error) {
- console.warn(
- JSON.stringify({
- message: 'AI completion failed',
- error: error instanceof Error ? error.message : String(error),
- }),
- );
- return null;
+ const failure = failureMessage(error);
+ console.warn(JSON.stringify({ message: 'AI completion failed', error: failure }));
+ return {
+ content: null,
+ latencyMs: Date.now() - startedAt,
+ promptTokens: null,
+ completionTokens: null,
+ failure,
+ };
}
}
+
+export async function requestCompletion(
+ settings: CompletionSettings,
+ system: string,
+ user: string,
+ maxTokens = 160,
+): Promise {
+ return (await requestCompletionDetailed(settings, system, user, maxTokens)).content;
+}
diff --git a/src/worker/ai/events.ts b/src/worker/ai/events.ts
new file mode 100644
index 0000000..49b817d
--- /dev/null
+++ b/src/worker/ai/events.ts
@@ -0,0 +1,55 @@
+import { getDb } from '../db/client';
+import { aiEvents } from '../db/schema';
+
+export type AiEventKind = 'incident_open' | 'incident_followup' | 'incident_resolve' | 'degraded_open' | 'manual_draft' | 'settings_test';
+export type AiEventOutcome = 'ok' | 'rejected' | 'failed' | 'skipped_budget' | 'skipped_standdown';
+
+export type AiEventInput = {
+ kind: AiEventKind;
+ incidentId?: number | null;
+ monitorId?: number | null;
+ model?: string | null;
+ outcome: AiEventOutcome;
+ reason?: string | null;
+ latencyMs?: number | null;
+ promptTokens?: number | null;
+ completionTokens?: number | null;
+ contextPreview?: string | null;
+ outputPreview?: string | null;
+};
+
+function preview(value: string | null | undefined) {
+ if (!value) return null;
+ return value
+ .replace(/https?:\/\/\S+/gi, '[redacted-url]')
+ .replace(/\b(?:\d{1,3}\.){3}\d{1,3}\b/g, '[redacted-ip]')
+ .replace(/\b(?:[a-z0-9-]+\.)+[a-z]{2,}\b/gi, '[redacted-host]')
+ .replace(/\b(?:sk|pk|rk)-[a-z0-9_-]+\b/gi, '[redacted-key]')
+ .replace(/\bBearer\s+\S+/gi, 'Bearer [redacted]')
+ .replace(/\s+/g, ' ')
+ .trim()
+ .slice(0, 240);
+}
+
+export async function recordAiEvent(env: Env, input: AiEventInput): Promise {
+ try {
+ await getDb(env)
+ .insert(aiEvents)
+ .values({
+ kind: input.kind,
+ incidentId: input.incidentId ?? null,
+ monitorId: input.monitorId ?? null,
+ model: input.model ?? null,
+ outcome: input.outcome,
+ reason: preview(input.reason)?.slice(0, 200) ?? null,
+ latencyMs: input.latencyMs ?? null,
+ promptTokens: input.promptTokens ?? null,
+ completionTokens: input.completionTokens ?? null,
+ contextPreview: preview(input.contextPreview),
+ outputPreview: preview(input.outputPreview),
+ createdAt: new Date(),
+ });
+ } catch (error) {
+ console.warn(JSON.stringify({ message: 'AI event recording failed', error: error instanceof Error ? error.message : String(error) }));
+ }
+}
diff --git a/src/worker/ai/fallback-message.ts b/src/worker/ai/fallback-message.ts
index 9aa8913..329aef8 100644
--- a/src/worker/ai/fallback-message.ts
+++ b/src/worker/ai/fallback-message.ts
@@ -4,6 +4,9 @@ export const INCIDENT_REASSURANCE =
export const RECOVERY_UPDATE_BODY = 'The service has recovered and is responding normally again.';
export const DEGRADED_MESSAGE = 'This service is responding more slowly than usual and some pages may take longer to load.';
+export const DEGRADED_RECOVERY_UPDATE_BODY = 'Response times have returned to normal and the service is operating normally.';
+export const DEGRADED_SUPERSEDED_UPDATE_BODY =
+ 'The performance issue developed into a service disruption and is tracked in a new incident.';
/** Plain-language, non-technical description of the impact, used when no AI message is available. */
export function describeFailure(statusCode: number | null): string {
diff --git a/src/worker/ai/incident-context.ts b/src/worker/ai/incident-context.ts
index 8db0d3f..3dd5be1 100644
--- a/src/worker/ai/incident-context.ts
+++ b/src/worker/ai/incident-context.ts
@@ -2,6 +2,7 @@ import { and, desc, eq, gte, sql } from 'drizzle-orm';
import type { CheckResult, Monitor } from '../checks/run-check';
import type { Database } from '../db/client';
import { checks, incidentMonitors, incidents } from '../db/schema';
+import { classifyFailure, humanizeDuration, humanizeInterval } from '../lib/humanize';
const HOUR_MS = 60 * 60 * 1000;
const DAY_MS = 24 * HOUR_MS;
@@ -14,55 +15,6 @@ const FLAP_WINDOW = 10;
* for the model only - it may name HTTP codes and transport errors here because
* the prompt is responsible for translating them into visitor-facing language.
*/
-function classifyFailure(monitor: Monitor, result: CheckResult): string {
- if (result.statusCode === null) {
- const error = (result.error ?? '').toLowerCase();
- if (/tim(?:e|ed)\s?out|timeout|deadline|aborted/.test(error)) {
- return 'Request timed out - no response came back before the timeout limit';
- }
- if (/getaddrinfo|enotfound|dns|name not resolved|could not resolve|eai_again/.test(error)) {
- return 'DNS lookup failed - the hostname could not be resolved to an address';
- }
- if (/certificate|cert(?:\s|_)|self[- ]signed|tls|ssl|handshake|err_cert/.test(error)) {
- return 'TLS/SSL failure - the certificate is invalid, expired, or untrusted';
- }
- if (/econnrefused|connection refused|refused to connect/.test(error)) {
- return 'Connection refused - nothing is accepting connections at that address';
- }
- if (/econnreset|connection reset|socket hang up|premature close/.test(error)) {
- return 'Connection dropped before any response was returned';
- }
- if (/ehostunreach|enetunreach|network is unreachable|no route to host/.test(error)) {
- return 'Network unreachable - the host could not be contacted at all';
- }
- return 'No HTTP response was received from the endpoint';
- }
-
- const code = result.statusCode;
- if (code >= 500) return `Server error - the endpoint answered with HTTP ${code}`;
- if (code === 429) return 'The endpoint is rate limiting - HTTP 429 Too Many Requests';
- if (code === 401 || code === 403) return `The endpoint rejected the health check as unauthorized - HTTP ${code}`;
- if (code === 404) return 'The health-check path returned HTTP 404 Not Found';
- if (code >= 400) return `Client error - the endpoint answered with HTTP ${code}`;
- if (code >= 300) return `Unexpected redirect - the endpoint answered with HTTP ${code}`;
- return `The endpoint answered with HTTP ${code}, but HTTP ${monitor.expectedStatus} was expected`;
-}
-
-function humanizeInterval(seconds: number): string {
- if (seconds >= 3600 && seconds % 3600 === 0) return `${seconds / 3600} h`;
- if (seconds >= 60 && seconds % 60 === 0) return `${seconds / 60} min`;
- return `${seconds} s`;
-}
-
-function humanizeDuration(ms: number): string {
- const minutes = Math.round(ms / 60_000);
- if (minutes < 1) return 'under a minute';
- if (minutes < 60) return `${minutes} min`;
- const hours = Math.floor(minutes / 60);
- const rest = minutes % 60;
- return rest ? `${hours} h ${rest} min` : `${hours} h`;
-}
-
function median(values: number[]): number {
const sorted = [...values].sort((a, b) => a - b);
const mid = Math.floor(sorted.length / 2);
diff --git a/src/worker/ai/incident-draft.ts b/src/worker/ai/incident-draft.ts
index d7b3f3f..f387561 100644
--- a/src/worker/ai/incident-draft.ts
+++ b/src/worker/ai/incident-draft.ts
@@ -1,7 +1,8 @@
import { eq } from 'drizzle-orm';
import { getDb } from '../db/client';
import { aiSettings } from '../db/schema';
-import { requestCompletion } from './client';
+import { requestCompletionDetailed } from './client';
+import { recordAiEvent } from './events';
import { sanitizePublicText } from './sanitize';
export type IncidentStatus = 'investigating' | 'identified' | 'monitoring' | 'resolved';
@@ -63,20 +64,54 @@ export async function draftIncidentUpdate(env: Env, input: DraftInput): Promise<
]
.filter(Boolean)
.join('\n\n');
- const completion = await requestCompletion(
+ const result = await requestCompletionDetailed(
{ baseUrl: settings.baseUrl, apiKey: settings.apiKey, model: settings.model },
INCIDENT_DRAFT_SYSTEM_PROMPT,
context,
320,
);
- if (!completion) throw new IncidentDraftError('AI could not generate a safe update. Edit the note or write the update manually.', 422);
+ const completion = result.content;
+ if (!completion) {
+ await recordAiEvent(env, {
+ kind: 'manual_draft',
+ model: settings.model,
+ outcome: 'failed',
+ reason: result.failure,
+ latencyMs: result.latencyMs,
+ promptTokens: result.promptTokens,
+ completionTokens: result.completionTokens,
+ contextPreview: context,
+ });
+ throw new IncidentDraftError('AI could not generate a safe update. Edit the note or write the update manually.', 422);
+ }
const titleMatch = completion.match(/(?:^|\n)TITLE:\s*(.+?)(?=\nBODY:|$)/is);
const bodyMatch = completion.match(/(?:^|\n)BODY:\s*([\s\S]+)$/i);
const title = input.withTitle ? sanitizePublicText(titleMatch?.[1] ?? '', 120)?.replace(/[.!?]+$/, '') : null;
const body = sanitizePublicText(bodyMatch?.[1] ?? '', 400);
if ((input.withTitle && !title) || !body) {
+ await recordAiEvent(env, {
+ kind: 'manual_draft',
+ model: settings.model,
+ outcome: 'rejected',
+ reason: 'sanitizer_rejected',
+ latencyMs: result.latencyMs,
+ promptTokens: result.promptTokens,
+ completionTokens: result.completionTokens,
+ contextPreview: context,
+ outputPreview: completion,
+ });
throw new IncidentDraftError('AI could not generate a safe update. Edit the note or write the update manually.', 422);
}
+ await recordAiEvent(env, {
+ kind: 'manual_draft',
+ model: settings.model,
+ outcome: 'ok',
+ latencyMs: result.latencyMs,
+ promptTokens: result.promptTokens,
+ completionTokens: result.completionTokens,
+ contextPreview: context,
+ outputPreview: completion,
+ });
return { title: title ?? null, body };
}
diff --git a/src/worker/ai/incident-message.ts b/src/worker/ai/incident-message.ts
index 690ea1a..98cd374 100644
--- a/src/worker/ai/incident-message.ts
+++ b/src/worker/ai/incident-message.ts
@@ -2,7 +2,8 @@ import { eq } from 'drizzle-orm';
import type { CheckResult, Monitor } from '../checks/run-check';
import { getDb } from '../db/client';
import { aiSettings } from '../db/schema';
-import { requestCompletion } from './client';
+import { requestCompletionDetailed } from './client';
+import { recordAiEvent } from './events';
import { buildIncidentContext } from './incident-context';
import { sanitizePublicText } from './sanitize';
@@ -32,14 +33,28 @@ export async function generateIncidentMessage(env: Env, input: { monitor: Monito
const [settings] = await db.select().from(aiSettings).where(eq(aiSettings.id, 1)).limit(1);
if (!settings?.enabled || !settings.baseUrl || !settings.apiKey || !settings.model) return null;
- const content = await requestCompletion(
+ const context = await buildIncidentContext(db, input.monitor, input.result);
+ const result = await requestCompletionDetailed(
{ baseUrl: settings.baseUrl, apiKey: settings.apiKey, model: settings.model },
INCIDENT_MESSAGE_SYSTEM_PROMPT,
- await buildIncidentContext(db, input.monitor, input.result),
+ context,
);
- if (!content) return null;
- const message = sanitizeIncidentMessage(content);
- if (!message) return null;
+ const message = result.content ? sanitizeIncidentMessage(result.content) : null;
+ if (!message) {
+ await recordAiEvent(env, {
+ kind: 'incident_open',
+ monitorId: input.monitor.id,
+ model: settings.model,
+ outcome: result.failure ? 'failed' : 'rejected',
+ reason: result.failure ?? 'sanitizer_rejected',
+ latencyMs: result.latencyMs,
+ promptTokens: result.promptTokens,
+ completionTokens: result.completionTokens,
+ contextPreview: context,
+ outputPreview: result.content,
+ });
+ return null;
+ }
await env.DB.prepare(
`INSERT INTO incident_updates (incident_id, status, body, source, created_at)
@@ -53,6 +68,17 @@ export async function generateIncidentMessage(env: Env, input: { monitor: Monito
)
.bind(message, Date.now(), input.monitor.id)
.run();
+ await recordAiEvent(env, {
+ kind: 'incident_open',
+ monitorId: input.monitor.id,
+ model: settings.model,
+ outcome: 'ok',
+ latencyMs: result.latencyMs,
+ promptTokens: result.promptTokens,
+ completionTokens: result.completionTokens,
+ contextPreview: context,
+ outputPreview: result.content,
+ });
return message;
} catch (error) {
console.warn(
diff --git a/src/worker/ai/sanitize.ts b/src/worker/ai/sanitize.ts
index c34d617..51f615e 100644
--- a/src/worker/ai/sanitize.ts
+++ b/src/worker/ai/sanitize.ts
@@ -1,4 +1,6 @@
-export function sanitizePublicText(value: string, maxLength: number): string | null {
+export type SanitizeReason = 'contains_url' | 'contains_ip' | 'contains_http_status' | 'empty';
+
+export function sanitizePublicTextWithReason(value: string, maxLength: number): { text: string | null; reason: SanitizeReason | null } {
const message = value
.replace(/\r/g, '')
.trim()
@@ -7,9 +9,15 @@ export function sanitizePublicText(value: string, maxLength: number): string | n
.replace(/\s+/g, ' ')
.trim()
.slice(0, maxLength);
- if (!message) return null;
- if (/https?:\/\//i.test(message)) return null;
- if (/\b(?:\d{1,3}\.){3}\d{1,3}\b/.test(message)) return null;
- if (/\bHTTP[\s/]?\d{3}\b/i.test(message) || /\b[45]\d{2}\s+(?:error|status|response)\b/i.test(message)) return null;
- return message;
+ if (!message) return { text: null, reason: 'empty' };
+ if (/https?:\/\//i.test(message)) return { text: null, reason: 'contains_url' };
+ if (/\b(?:\d{1,3}\.){3}\d{1,3}\b/.test(message)) return { text: null, reason: 'contains_ip' };
+ if (/\bHTTP[\s/]?\d{3}\b/i.test(message) || /\b[45]\d{2}\s+(?:error|status|response)\b/i.test(message)) {
+ return { text: null, reason: 'contains_http_status' };
+ }
+ return { text: message, reason: null };
+}
+
+export function sanitizePublicText(value: string, maxLength: number): string | null {
+ return sanitizePublicTextWithReason(value, maxLength).text;
}
diff --git a/src/worker/autopilot/cadence.ts b/src/worker/autopilot/cadence.ts
new file mode 100644
index 0000000..1c14b7c
--- /dev/null
+++ b/src/worker/autopilot/cadence.ts
@@ -0,0 +1,39 @@
+export type AutopilotIncidentStatus = 'investigating' | 'identified' | 'monitoring' | 'resolved';
+
+export type StatusSignal = {
+ consecutiveFailures: number;
+ failureSignatureStable: boolean;
+ consecutiveOk: number;
+ latestOk: boolean;
+ regressionUsed: boolean;
+};
+
+export type ImpactSignal = {
+ kind: 'down' | 'degraded';
+ affectedMonitors: number;
+ totalMonitors: number;
+ recentChecks: boolean[];
+};
+
+export function nextFollowupDueAt(lastUpdateAt: number, autoUpdateCount: number, cadenceMinutes: number): number {
+ const exponent = Math.max(0, autoUpdateCount - 1);
+ const multiplier = Math.min(8, 2 ** exponent);
+ return lastUpdateAt + cadenceMinutes * multiplier * 60_000;
+}
+
+export function advanceStatus(current: AutopilotIncidentStatus, signal: StatusSignal): AutopilotIncidentStatus {
+ if (current === 'resolved') return current;
+ if (current === 'monitoring' && !signal.latestOk) return signal.regressionUsed ? current : 'identified';
+ if ((current === 'investigating' || current === 'identified') && signal.consecutiveOk >= 2 && !signal.regressionUsed) return 'monitoring';
+ if (current === 'investigating' && signal.consecutiveFailures >= 3 && signal.failureSignatureStable) return 'identified';
+ return current;
+}
+
+export function computeImpact(signal: ImpactSignal): 'minor' | 'major' | 'critical' {
+ if (signal.kind === 'degraded') return 'minor';
+ if (signal.affectedMonitors >= 2 && signal.totalMonitors > 0 && signal.affectedMonitors / signal.totalMonitors >= 0.5) return 'critical';
+ if (signal.affectedMonitors >= 3) return 'critical';
+ if (signal.recentChecks.length >= 10 && signal.recentChecks.slice(0, 10).every((ok) => !ok)) return 'major';
+ if (signal.recentChecks.some(Boolean)) return 'minor';
+ return 'major';
+}
diff --git a/src/worker/autopilot/guards.ts b/src/worker/autopilot/guards.ts
new file mode 100644
index 0000000..1ff98db
--- /dev/null
+++ b/src/worker/autopilot/guards.ts
@@ -0,0 +1,19 @@
+export type AutopilotCandidate = {
+ source: string;
+ resolvedAt: Date | null;
+ hasManualUpdate: boolean;
+ alertsEnabled: boolean;
+};
+
+export function autopilotEligible(candidate: AutopilotCandidate): boolean {
+ return candidate.source === 'auto' && candidate.resolvedAt === null && !candidate.hasManualUpdate && candidate.alertsEnabled;
+}
+
+export const AUTOPILOT_WRITE_GUARD = `
+ i.source = 'auto'
+ AND i.resolved_at IS NULL
+ AND i.updated_at = ?
+ AND NOT EXISTS (
+ SELECT 1 FROM incident_updates manual
+ WHERE manual.incident_id = i.id AND manual.source = 'manual'
+ )`;
diff --git a/src/worker/autopilot/index.ts b/src/worker/autopilot/index.ts
new file mode 100644
index 0000000..8f43258
--- /dev/null
+++ b/src/worker/autopilot/index.ts
@@ -0,0 +1,379 @@
+import { and, eq, isNull, sql } from 'drizzle-orm';
+import { requestCompletionDetailed, type CompletionResult } from '../ai/client';
+import { DEGRADED_RECOVERY_UPDATE_BODY, RECOVERY_UPDATE_BODY } from '../ai/fallback-message';
+import { buildIncidentContext } from '../ai/incident-context';
+import { recordAiEvent, type AiEventKind } from '../ai/events';
+import {
+ AUTOPILOT_STATUS_GUIDANCE,
+ INCIDENT_FOLLOWUP_SYSTEM_PROMPT,
+ INCIDENT_OPEN_SYSTEM_PROMPT,
+ INCIDENT_RESOLVE_SYSTEM_PROMPT,
+} from '../ai/autopilot-prompts';
+import { sanitizePublicTextWithReason } from '../ai/sanitize';
+import type { CheckResult, Monitor } from '../checks/run-check';
+import { getDb } from '../db/client';
+import { aiSettings, incidentUpdates, incidents } from '../db/schema';
+import { humanizeDuration } from '../lib/humanize';
+import { advanceStatus, computeImpact, nextFollowupDueAt, type AutopilotIncidentStatus } from './cadence';
+import { loadIncidentSignal, findLatestAutoIncidentForMonitor } from './signal';
+
+export const MAX_AI_CALLS_PER_RUN = 12;
+export const MAX_FOLLOWUP_CALLS_PER_RUN = 6;
+export const AUTOPILOT_CONCURRENCY = 4;
+export const AUTOPILOT_DEADLINE_MS = 45_000;
+
+export type AiBudget = { remaining: number; deadline?: number };
+export type AutopilotEvent = {
+ monitor: Monitor;
+ result: CheckResult;
+ transition: 'opened' | 'resolved' | null;
+ latencyTransition: 'degraded' | 'recovered' | null;
+ checkedAt: Date;
+};
+export type AutopilotSummary = { calls: number; written: number; rejected: number; failed: number; skipped: number };
+
+type Settings = typeof aiSettings.$inferSelect;
+type Task = { kind: AiEventKind; incidentId: number; monitorId: number; run: () => Promise };
+
+function completionEvent(result: CompletionResult) {
+ return {
+ latencyMs: result.latencyMs,
+ promptTokens: result.promptTokens,
+ completionTokens: result.completionTokens,
+ outputPreview: result.content,
+ };
+}
+
+async function complete(settings: Settings, system: string, user: string, maxTokens = 180) {
+ return requestCompletionDetailed(
+ { baseUrl: settings.baseUrl!, apiKey: settings.apiKey!, model: settings.model! },
+ system,
+ user,
+ maxTokens,
+ );
+}
+
+async function writeCasUpdate(
+ env: Env,
+ incident: typeof incidents.$inferSelect,
+ status: string,
+ body: string,
+ now: number,
+): Promise {
+ const results = await env.DB.batch([
+ env.DB.prepare(
+ `INSERT INTO incident_updates (incident_id, status, body, source, created_at)
+ SELECT i.id, ?, ?, 'ai', ? FROM incidents i
+ WHERE i.id = ? AND i.source = 'auto' AND i.resolved_at IS NULL AND i.updated_at = ?
+ AND NOT EXISTS (SELECT 1 FROM incident_updates m WHERE m.incident_id = i.id AND m.source = 'manual')`,
+ ).bind(status, body, now, incident.id, incident.updatedAt.getTime()),
+ env.DB.prepare(
+ `UPDATE incidents SET status = ?, updated_at = ?
+ WHERE id = ? AND source = 'auto' AND resolved_at IS NULL AND updated_at = ?
+ AND NOT EXISTS (SELECT 1 FROM incident_updates m WHERE m.incident_id = incidents.id AND m.source = 'manual')`,
+ ).bind(status, now, incident.id, incident.updatedAt.getTime()),
+ ]);
+ return Number(results[0].meta.changes ?? 0) === 1 && Number(results[1].meta.changes ?? 0) === 1;
+}
+
+async function processOpening(env: Env, settings: Settings, event: AutopilotEvent, incidentId: number, kind: 'down' | 'degraded') {
+ const db = getDb(env);
+ const [incident] = await db.select().from(incidents).where(eq(incidents.id, incidentId)).limit(1);
+ if (!incident || incident.resolvedAt || incident.source !== 'auto') return false;
+ const signal = await loadIncidentSignal(db, incident.id);
+ if (!signal) return false;
+ const context = await buildIncidentContext(db, event.monitor, event.result);
+ const result = await complete(settings, INCIDENT_OPEN_SYSTEM_PROMPT, context, 220);
+ if (!result.content) {
+ await recordAiEvent(env, {
+ kind: kind === 'degraded' ? 'degraded_open' : 'incident_open',
+ incidentId,
+ monitorId: event.monitor.id,
+ model: settings.model,
+ outcome: 'failed',
+ reason: result.failure,
+ contextPreview: context,
+ ...completionEvent(result),
+ });
+ return false;
+ }
+ const titleRaw = result.content.match(/(?:^|\n)TITLE:\s*(.+?)(?=\nBODY:|$)/is)?.[1] ?? '';
+ const bodyRaw = result.content.match(/(?:^|\n)BODY:\s*([\s\S]+)$/i)?.[1] ?? '';
+ const title = sanitizePublicTextWithReason(titleRaw, 120);
+ const body = sanitizePublicTextWithReason(bodyRaw, 280);
+ const impact = computeImpact({
+ kind,
+ affectedMonitors: signal.affectedMonitors,
+ totalMonitors: signal.totalMonitors,
+ recentChecks: signal.recentChecks.map((check) => check.ok),
+ });
+ const now = Date.now();
+ const statements: D1PreparedStatement[] = [];
+ if (body.text) {
+ statements.push(
+ env.DB.prepare(
+ `INSERT INTO incident_updates (incident_id, status, body, source, created_at)
+ SELECT i.id, i.status, ?, 'ai', ? FROM incidents i
+ WHERE i.id = ? AND i.source = 'auto' AND i.resolved_at IS NULL AND i.updated_at = ?
+ AND NOT EXISTS (SELECT 1 FROM incident_updates u WHERE u.incident_id = i.id)`,
+ ).bind(body.text, now, incident.id, incident.updatedAt.getTime()),
+ );
+ }
+ statements.push(
+ env.DB.prepare(
+ `UPDATE incidents SET title = coalesce(?, title), impact = ?, updated_at = ?
+ WHERE id = ? AND source = 'auto' AND resolved_at IS NULL AND updated_at = ? AND title IS NULL
+ AND NOT EXISTS (SELECT 1 FROM incident_updates m WHERE m.incident_id = incidents.id AND m.source = 'manual')`,
+ ).bind(title.text, impact, now, incident.id, incident.updatedAt.getTime()),
+ );
+ const writeResults = await env.DB.batch(statements);
+ const wrote = writeResults.some((writeResult) => Number(writeResult.meta.changes ?? 0) > 0);
+ await recordAiEvent(env, {
+ kind: kind === 'degraded' ? 'degraded_open' : 'incident_open',
+ incidentId,
+ monitorId: event.monitor.id,
+ model: settings.model,
+ outcome: title.text || body.text ? (wrote ? 'ok' : 'skipped_standdown') : 'rejected',
+ reason:
+ [title.reason && `title:${title.reason}`, body.reason && `body:${body.reason}`].filter(Boolean).join(',') ||
+ (wrote ? null : 'cas_conflict_or_manual_update'),
+ contextPreview: context,
+ ...completionEvent(result),
+ });
+ return wrote;
+}
+
+async function processResolution(env: Env, settings: Settings, event: AutopilotEvent, incidentId: number, fallbackBody: string) {
+ const db = getDb(env);
+ const [incident] = await db.select().from(incidents).where(eq(incidents.id, incidentId)).limit(1);
+ if (!incident?.resolvedAt) return false;
+ const context = `Service name: ${event.monitor.name}\nObserved recovery: automated checks are healthy\nIncident duration: ${humanizeDuration(incident.durationMs ?? incident.resolvedAt.getTime() - incident.startedAt.getTime())}`;
+ const result = await complete(settings, INCIDENT_RESOLVE_SYSTEM_PROMPT, context);
+ const sanitized = result.content ? sanitizePublicTextWithReason(result.content, 280) : { text: null, reason: null };
+ let wrote = false;
+ if (sanitized.text) {
+ const write = await env.DB.prepare(
+ `UPDATE incident_updates SET body = ?, source = 'ai'
+ WHERE id = (
+ SELECT iu.id FROM incident_updates iu
+ WHERE iu.incident_id = ? AND iu.status = 'resolved' AND iu.source = 'system'
+ AND iu.body = ?
+ AND NOT EXISTS (
+ SELECT 1 FROM incident_updates manual
+ WHERE manual.incident_id = iu.incident_id AND manual.source = 'manual'
+ )
+ ORDER BY iu.id DESC LIMIT 1
+ )`,
+ )
+ .bind(sanitized.text, incidentId, fallbackBody)
+ .run();
+ wrote = Number(write.meta.changes ?? 0) === 1;
+ }
+ await recordAiEvent(env, {
+ kind: 'incident_resolve',
+ incidentId,
+ monitorId: event.monitor.id,
+ model: settings.model,
+ outcome: sanitized.text ? (wrote ? 'ok' : 'skipped_standdown') : result.failure ? 'failed' : 'rejected',
+ reason: result.failure ?? sanitized.reason ?? (wrote ? null : 'manual_update_or_already_rewritten'),
+ contextPreview: context,
+ ...completionEvent(result),
+ });
+ return wrote;
+}
+
+async function processFollowup(env: Env, settings: Settings, incident: typeof incidents.$inferSelect) {
+ const db = getDb(env);
+ const signal = await loadIncidentSignal(db, incident.id);
+ if (!signal) return false;
+ const status = settings.autopilotAdvanceStatus
+ ? advanceStatus(incident.status as AutopilotIncidentStatus, signal)
+ : (incident.status as AutopilotIncidentStatus);
+ const guidance = AUTOPILOT_STATUS_GUIDANCE[status === 'resolved' ? 'monitoring' : status];
+ const context = [
+ `Service name: ${signal.monitor.name}`,
+ `Incident age: ${humanizeDuration(Date.now() - incident.startedAt.getTime())}`,
+ `Current state: ${status}`,
+ `Recent checks: ${signal.recentChecks.filter((check) => check.ok).length} healthy of ${signal.recentChecks.length}`,
+ `Writing guidance: ${guidance}`,
+ ].join('\n');
+ const result = await complete(settings, INCIDENT_FOLLOWUP_SYSTEM_PROMPT, context);
+ const sanitized = result.content ? sanitizePublicTextWithReason(result.content, 280) : { text: null, reason: null };
+ const wrote = sanitized.text ? await writeCasUpdate(env, incident, status, sanitized.text, Date.now()) : false;
+ await recordAiEvent(env, {
+ kind: 'incident_followup',
+ incidentId: incident.id,
+ monitorId: signal.monitor.id,
+ model: settings.model,
+ outcome: sanitized.text ? (wrote ? 'ok' : 'skipped_standdown') : result.failure ? 'failed' : 'rejected',
+ reason: result.failure ?? sanitized.reason ?? (wrote ? null : 'cas_conflict_or_manual_update'),
+ contextPreview: context,
+ ...completionEvent(result),
+ });
+ return wrote;
+}
+
+async function loadFollowupTasks(env: Env, settings: Settings, excluded: Set): Promise {
+ const db = getDb(env);
+ const rows = await db
+ .select({
+ incident: incidents,
+ lastUpdateAt: sql`coalesce(max(${incidentUpdates.createdAt}), ${incidents.startedAt})`,
+ autoUpdateCount: sql`coalesce(sum(case when ${incidentUpdates.source} in ('ai','system') then 1 else 0 end), 0)`,
+ hasManual: sql`coalesce(max(case when ${incidentUpdates.source} = 'manual' then 1 else 0 end), 0)`,
+ })
+ .from(incidents)
+ .leftJoin(incidentUpdates, eq(incidentUpdates.incidentId, incidents.id))
+ .where(and(eq(incidents.source, 'auto'), isNull(incidents.resolvedAt)))
+ .groupBy(incidents.id)
+ .orderBy(sql`coalesce(max(${incidentUpdates.createdAt}), ${incidents.startedAt}) asc`);
+ const tasks: Task[] = [];
+ for (const row of rows) {
+ if (excluded.has(row.incident.id)) continue;
+ if (Number(row.hasManual) > 0) {
+ const [alreadyLogged] = await db
+ .select({ id: sql`id` })
+ .from(sql`ai_events`)
+ .where(sql`incident_id = ${row.incident.id} and kind = 'incident_followup' and outcome = 'skipped_standdown'`)
+ .limit(1);
+ if (alreadyLogged) continue;
+ const [monitor] = await db
+ .select({ id: sql`monitor_id` })
+ .from(sql`incident_monitors`)
+ .where(sql`incident_id = ${row.incident.id}`)
+ .limit(1);
+ await recordAiEvent(env, {
+ kind: 'incident_followup',
+ incidentId: row.incident.id,
+ monitorId: monitor?.id ?? null,
+ model: settings.model,
+ outcome: 'skipped_standdown',
+ reason: 'manual_update_present',
+ });
+ continue;
+ }
+ const count = Number(row.autoUpdateCount);
+ if (count >= settings.autopilotMaxUpdates) continue;
+ const last = row.lastUpdateAt instanceof Date ? row.lastUpdateAt.getTime() : Number(row.lastUpdateAt);
+ if (Date.now() < nextFollowupDueAt(last, count, settings.autopilotFollowupMinutes)) continue;
+ const signal = await loadIncidentSignal(db, row.incident.id);
+ if (!signal?.monitor.alertsEnabled) continue;
+ tasks.push({
+ kind: 'incident_followup',
+ incidentId: row.incident.id,
+ monitorId: signal.monitor.id,
+ run: () => processFollowup(env, settings, row.incident),
+ });
+ if (tasks.length >= MAX_FOLLOWUP_CALLS_PER_RUN) break;
+ }
+ return tasks;
+}
+
+export async function runAutopilot(
+ env: Env,
+ input: { events?: AutopilotEvent[]; budget?: AiBudget; skipSweep?: boolean } = {},
+): Promise {
+ const summary: AutopilotSummary = { calls: 0, written: 0, rejected: 0, failed: 0, skipped: 0 };
+ const db = getDb(env);
+ const [settings] = await db.select().from(aiSettings).where(eq(aiSettings.id, 1)).limit(1);
+ if (!settings?.enabled || !settings.autopilotEnabled || !settings.baseUrl || !settings.apiKey || !settings.model) return summary;
+ const budget = input.budget ?? { remaining: MAX_AI_CALLS_PER_RUN, deadline: Date.now() + AUTOPILOT_DEADLINE_MS };
+ budget.deadline ??= Date.now() + AUTOPILOT_DEADLINE_MS;
+ const tasks: Task[] = [];
+ const excluded = new Set();
+ for (const event of input.events ?? []) {
+ if (!event.monitor.alertsEnabled) continue;
+ if (event.transition === 'opened') {
+ const incident = await findLatestAutoIncidentForMonitor(db, event.monitor.id, { resolved: false, kind: 'down' });
+ if (incident) {
+ excluded.add(incident.id);
+ tasks.push({
+ kind: 'incident_open',
+ incidentId: incident.id,
+ monitorId: event.monitor.id,
+ run: () => processOpening(env, settings, event, incident.id, 'down'),
+ });
+ }
+ }
+ if (event.transition === 'resolved') {
+ const incident = await findLatestAutoIncidentForMonitor(db, event.monitor.id, { resolved: true, kind: 'down' });
+ if (incident) {
+ tasks.push({
+ kind: 'incident_resolve',
+ incidentId: incident.id,
+ monitorId: event.monitor.id,
+ run: () => processResolution(env, settings, event, incident.id, RECOVERY_UPDATE_BODY),
+ });
+ }
+ }
+ if (event.latencyTransition === 'degraded' && settings.autopilotDegradedIncidents) {
+ const incident = await findLatestAutoIncidentForMonitor(db, event.monitor.id, { resolved: false, kind: 'degraded' });
+ if (incident) {
+ excluded.add(incident.id);
+ tasks.push({
+ kind: 'degraded_open',
+ incidentId: incident.id,
+ monitorId: event.monitor.id,
+ run: () => processOpening(env, settings, event, incident.id, 'degraded'),
+ });
+ }
+ }
+ if (event.transition !== 'opened' && event.latencyTransition === 'recovered' && settings.autopilotDegradedIncidents) {
+ const incident = await findLatestAutoIncidentForMonitor(db, event.monitor.id, { resolved: true, kind: 'degraded' });
+ if (incident) {
+ tasks.push({
+ kind: 'incident_resolve',
+ incidentId: incident.id,
+ monitorId: event.monitor.id,
+ run: () => processResolution(env, settings, event, incident.id, DEGRADED_RECOVERY_UPDATE_BODY),
+ });
+ }
+ }
+ }
+ if (!input.skipSweep) tasks.push(...(await loadFollowupTasks(env, settings, excluded)));
+ const priority: Record = {
+ incident_open: 0,
+ incident_resolve: 1,
+ degraded_open: 2,
+ incident_followup: 3,
+ manual_draft: 4,
+ settings_test: 5,
+ };
+ tasks.sort((left, right) => priority[left.kind] - priority[right.kind]);
+
+ for (let offset = 0; offset < tasks.length; offset += AUTOPILOT_CONCURRENCY) {
+ const batch = tasks.slice(offset, offset + AUTOPILOT_CONCURRENCY);
+ await Promise.all(
+ batch.map(async (task) => {
+ if (budget.remaining <= 0 || Date.now() >= budget.deadline!) {
+ summary.skipped += 1;
+ await recordAiEvent(env, {
+ kind: task.kind,
+ incidentId: task.incidentId,
+ monitorId: task.monitorId,
+ model: settings.model,
+ outcome: 'skipped_budget',
+ reason: budget.remaining <= 0 ? 'per_run_limit' : 'deadline',
+ });
+ return;
+ }
+ budget.remaining -= 1;
+ summary.calls += 1;
+ try {
+ if (await task.run()) summary.written += 1;
+ } catch (error) {
+ summary.failed += 1;
+ console.warn(
+ JSON.stringify({
+ message: 'autopilot task failed',
+ kind: task.kind,
+ incidentId: task.incidentId,
+ error: error instanceof Error ? error.message : String(error),
+ }),
+ );
+ }
+ }),
+ );
+ }
+ return summary;
+}
diff --git a/src/worker/autopilot/signal.ts b/src/worker/autopilot/signal.ts
new file mode 100644
index 0000000..9b3d8bf
--- /dev/null
+++ b/src/worker/autopilot/signal.ts
@@ -0,0 +1,102 @@
+import { and, asc, desc, eq, isNotNull, isNull, sql } from 'drizzle-orm';
+import type { CheckResult, Monitor } from '../checks/run-check';
+import type { Database } from '../db/client';
+import { checks, incidentMonitors, incidents, incidentUpdates, monitors } from '../db/schema';
+import { classifyFailure } from '../lib/humanize';
+
+export type IncidentSignal = {
+ monitor: Monitor;
+ recentChecks: Array<{ ok: boolean; statusCode: number | null; latencyMs: number | null; error: string | null }>;
+ consecutiveFailures: number;
+ consecutiveOk: number;
+ failureSignatureStable: boolean;
+ latestOk: boolean;
+ regressionUsed: boolean;
+ affectedMonitors: number;
+ totalMonitors: number;
+};
+
+export async function findLatestAutoIncidentForMonitor(
+ db: Database,
+ monitorId: number,
+ options: { resolved: boolean; kind?: 'down' | 'degraded' },
+) {
+ const [incident] = await db
+ .select()
+ .from(incidents)
+ .innerJoin(incidentMonitors, eq(incidentMonitors.incidentId, incidents.id))
+ .where(
+ and(
+ eq(incidentMonitors.monitorId, monitorId),
+ eq(incidents.source, 'auto'),
+ options.kind ? eq(incidents.kind, options.kind) : undefined,
+ options.resolved ? isNotNull(incidents.resolvedAt) : isNull(incidents.resolvedAt),
+ ),
+ )
+ .orderBy(desc(incidents.startedAt))
+ .limit(1);
+ return incident?.incidents ?? null;
+}
+
+export async function loadIncidentSignal(db: Database, incidentId: number): Promise {
+ const [monitor] = await db
+ .select({ monitor: monitors })
+ .from(incidentMonitors)
+ .innerJoin(monitors, eq(monitors.id, incidentMonitors.monitorId))
+ .where(eq(incidentMonitors.incidentId, incidentId))
+ .orderBy(asc(incidentMonitors.monitorId))
+ .limit(1);
+ if (!monitor) return null;
+
+ const recentChecks = await db
+ .select({ ok: checks.ok, statusCode: checks.statusCode, latencyMs: checks.latencyMs, error: checks.error })
+ .from(checks)
+ .where(and(eq(checks.monitorId, monitor.monitor.id), eq(checks.maintenance, false)))
+ .orderBy(desc(checks.checkedAt))
+ .limit(10);
+ let consecutiveFailures = 0;
+ let consecutiveOk = 0;
+ for (const check of recentChecks) {
+ if (check.ok) break;
+ consecutiveFailures += 1;
+ }
+ for (const check of recentChecks) {
+ if (!check.ok) break;
+ consecutiveOk += 1;
+ }
+ const failureSignatures = recentChecks.slice(0, consecutiveFailures).map((check) =>
+ classifyFailure(monitor.monitor, {
+ ok: check.ok,
+ degraded: false,
+ statusCode: check.statusCode,
+ latencyMs: check.latencyMs ?? 0,
+ error: check.error,
+ } satisfies CheckResult),
+ );
+ const statusHistory = await db
+ .select({ status: incidentUpdates.status })
+ .from(incidentUpdates)
+ .where(eq(incidentUpdates.incidentId, incidentId))
+ .orderBy(incidentUpdates.createdAt, incidentUpdates.id);
+ const monitoringIndex = statusHistory.findIndex((update) => update.status === 'monitoring');
+ const regressionUsed = monitoringIndex >= 0 && statusHistory.slice(monitoringIndex + 1).some((update) => update.status === 'identified');
+ const [affected] = await db
+ .select({ count: sql`count(*)` })
+ .from(incidentMonitors)
+ .where(eq(incidentMonitors.incidentId, incidentId));
+ const [total] = await db
+ .select({ count: sql`count(*)` })
+ .from(monitors)
+ .where(eq(monitors.enabled, true));
+ return {
+ monitor: monitor.monitor,
+ recentChecks,
+ consecutiveFailures,
+ consecutiveOk,
+ failureSignatureStable: failureSignatures.length >= 3 && new Set(failureSignatures).size === 1,
+ latestOk: recentChecks[0]?.ok ?? false,
+ regressionUsed,
+ affectedMonitors: Number(affected?.count ?? 1),
+ totalMonitors: Number(total?.count ?? 1),
+ };
+}
diff --git a/src/worker/checks/persist-result.ts b/src/worker/checks/persist-result.ts
index 4bfec32..fdbac91 100644
--- a/src/worker/checks/persist-result.ts
+++ b/src/worker/checks/persist-result.ts
@@ -1,7 +1,7 @@
import { and, eq, inArray, isNull, sql } from 'drizzle-orm';
import type { Database } from '../db/client';
import { checks, incidentMonitors, incidents, incidentUpdates, monitors } from '../db/schema';
-import { RECOVERY_UPDATE_BODY } from '../ai/fallback-message';
+import { DEGRADED_RECOVERY_UPDATE_BODY, DEGRADED_SUPERSEDED_UPDATE_BODY, RECOVERY_UPDATE_BODY } from '../ai/fallback-message';
import type { CheckResult, Monitor } from './run-check';
/** Only opened and resolved transitions are allowed to send alerts. */
@@ -11,7 +11,14 @@ export type LatencyTransition = 'degraded' | 'recovered' | null;
type BatchStatement = Parameters[0][number];
-export function buildResultStatements(db: Database, monitor: Monitor, result: CheckResult, checkedAt: Date, maintenance = false) {
+export function buildResultStatements(
+ db: Database,
+ monitor: Monitor,
+ result: CheckResult,
+ checkedAt: Date,
+ maintenance = false,
+ options: { degradedIncidents?: boolean } = {},
+) {
const statements: BatchStatement[] = [
db.insert(checks).values({
monitorId: monitor.id,
@@ -78,11 +85,14 @@ export function buildResultStatements(db: Database, monitor: Monitor, result: Ch
let transition: CheckTransition = null;
if (!wasDown && isDown) {
+ const localImpact =
+ result.statusCode === null || (result.statusCode !== null && result.statusCode >= 500) || nextFailures >= 10 ? 'major' : 'minor';
statements.push(
db.insert(incidents).values({
status: 'investigating',
- impact: 'major',
+ impact: localImpact,
source: 'auto',
+ kind: 'down',
startedAt: checkedAt,
startStatusCode: result.statusCode,
startError: result.error,
@@ -91,13 +101,60 @@ export function buildResultStatements(db: Database, monitor: Monitor, result: Ch
}),
db.insert(incidentMonitors).values({ incidentId: sql`last_insert_rowid()`, monitorId: monitor.id }),
);
+ // Keep this after incident_monitors: that statement must consume the new down incident rowid first.
+ if (options.degradedIncidents) {
+ const degradedIds = db
+ .select({ id: incidentMonitors.incidentId })
+ .from(incidentMonitors)
+ .innerJoin(incidents, eq(incidents.id, incidentMonitors.incidentId))
+ .where(
+ and(
+ eq(incidentMonitors.monitorId, monitor.id),
+ eq(incidents.source, 'auto'),
+ eq(incidents.kind, 'degraded'),
+ isNull(incidents.resolvedAt),
+ ),
+ );
+ statements.push(
+ db.insert(incidentUpdates).select(
+ db
+ .select({
+ id: sql`null`.as('id'),
+ incidentId: incidents.id,
+ status: sql`'resolved'`.as('status'),
+ body: sql`${DEGRADED_SUPERSEDED_UPDATE_BODY}`.as('body'),
+ note: sql`null`.as('note'),
+ source: sql`'system'`.as('source'),
+ createdAt: sql`${checkedAt.getTime()}`.as('created_at'),
+ })
+ .from(incidents)
+ .where(inArray(incidents.id, degradedIds)),
+ ),
+ db
+ .update(incidents)
+ .set({
+ status: 'resolved',
+ resolvedAt: checkedAt,
+ durationMs: sql`${checkedAt.getTime()} - ${incidents.startedAt}`,
+ updatedAt: checkedAt,
+ })
+ .where(inArray(incidents.id, degradedIds)),
+ );
+ }
transition = 'opened';
} else if (wasDown && result.ok) {
const openIncidentIds = db
.select({ id: incidentMonitors.incidentId })
.from(incidentMonitors)
.innerJoin(incidents, eq(incidents.id, incidentMonitors.incidentId))
- .where(and(eq(incidentMonitors.monitorId, monitor.id), eq(incidents.source, 'auto'), isNull(incidents.resolvedAt)));
+ .where(
+ and(
+ eq(incidentMonitors.monitorId, monitor.id),
+ eq(incidents.source, 'auto'),
+ eq(incidents.kind, 'down'),
+ isNull(incidents.resolvedAt),
+ ),
+ );
statements.push(
db.insert(incidentUpdates).select(
db
@@ -111,7 +168,14 @@ export function buildResultStatements(db: Database, monitor: Monitor, result: Ch
createdAt: sql`${checkedAt.getTime()}`.as('created_at'),
})
.from(incidents)
- .where(and(eq(incidents.source, 'auto'), isNull(incidents.resolvedAt), inArray(incidents.id, openIncidentIds))),
+ .where(
+ and(
+ eq(incidents.source, 'auto'),
+ eq(incidents.kind, 'down'),
+ isNull(incidents.resolvedAt),
+ inArray(incidents.id, openIncidentIds),
+ ),
+ ),
),
db
.update(incidents)
@@ -121,11 +185,73 @@ export function buildResultStatements(db: Database, monitor: Monitor, result: Ch
durationMs: sql`${checkedAt.getTime()} - ${incidents.startedAt}`,
updatedAt: checkedAt,
})
- .where(and(eq(incidents.source, 'auto'), isNull(incidents.resolvedAt), inArray(incidents.id, openIncidentIds))),
+ .where(
+ and(
+ eq(incidents.source, 'auto'),
+ eq(incidents.kind, 'down'),
+ isNull(incidents.resolvedAt),
+ inArray(incidents.id, openIncidentIds),
+ ),
+ ),
);
transition = 'resolved';
} else if (!wasDown && !result.ok) transition = 'pending';
else if (!wasDown && result.ok && previousFailures > 0) transition = 'cleared';
+ if (options.degradedIncidents && monitor.alertsEnabled && latencyTransition === 'degraded' && !isDown) {
+ statements.push(
+ db.insert(incidents).values({
+ status: 'investigating',
+ impact: 'minor',
+ source: 'auto',
+ kind: 'degraded',
+ startedAt: checkedAt,
+ startStatusCode: result.statusCode,
+ startError: result.error,
+ createdAt: checkedAt,
+ updatedAt: checkedAt,
+ }),
+ db.insert(incidentMonitors).values({ incidentId: sql`last_insert_rowid()`, monitorId: monitor.id }),
+ );
+ } else if (options.degradedIncidents && latencyTransition === 'recovered') {
+ const degradedIds = db
+ .select({ id: incidentMonitors.incidentId })
+ .from(incidentMonitors)
+ .innerJoin(incidents, eq(incidents.id, incidentMonitors.incidentId))
+ .where(
+ and(
+ eq(incidentMonitors.monitorId, monitor.id),
+ eq(incidents.source, 'auto'),
+ eq(incidents.kind, 'degraded'),
+ isNull(incidents.resolvedAt),
+ ),
+ );
+ statements.push(
+ db.insert(incidentUpdates).select(
+ db
+ .select({
+ id: sql`null`.as('id'),
+ incidentId: incidents.id,
+ status: sql`'resolved'`.as('status'),
+ body: sql`${DEGRADED_RECOVERY_UPDATE_BODY}`.as('body'),
+ note: sql`null`.as('note'),
+ source: sql`'system'`.as('source'),
+ createdAt: sql`${checkedAt.getTime()}`.as('created_at'),
+ })
+ .from(incidents)
+ .where(inArray(incidents.id, degradedIds)),
+ ),
+ db
+ .update(incidents)
+ .set({
+ status: 'resolved',
+ resolvedAt: checkedAt,
+ durationMs: sql`${checkedAt.getTime()} - ${incidents.startedAt}`,
+ updatedAt: checkedAt,
+ })
+ .where(inArray(incidents.id, degradedIds)),
+ );
+ }
+
return { statements, transition, latencyTransition, consecutiveFailures: nextFailures };
}
diff --git a/src/worker/checks/run-due-checks.ts b/src/worker/checks/run-due-checks.ts
index 5f0550f..2d172e3 100644
--- a/src/worker/checks/run-due-checks.ts
+++ b/src/worker/checks/run-due-checks.ts
@@ -1,15 +1,16 @@
import { and, eq, sql } from 'drizzle-orm';
-import { generateIncidentMessage } from '../ai/incident-message';
+import type { AutopilotEvent } from '../autopilot';
import { getDb } from '../db/client';
-import { monitors } from '../db/schema';
+import { aiSettings, monitors } from '../db/schema';
import { loadActiveMaintenance } from '../maintenance/windows';
+import { buildAlertEvent } from '../notifications/compose';
import { dispatchNotification, MAX_NOTIFICATIONS_PER_RUN, type NotificationBudget } from '../notifications/dispatch';
-import { type AlertTransition, buildResultStatements } from './persist-result';
+import { buildResultStatements } from './persist-result';
import { runCheck, runCheckWithRetries, type RetryBudget } from './run-check';
const MAX_MONITORS_PER_RUN = 40;
-const MAX_AI_MESSAGES_PER_RUN = 10;
const CONCURRENCY = 10;
+const MAX_BATCH_STATEMENTS = 100;
export const MAX_RETRY_ATTEMPTS_PER_RUN = 60;
const RETRY_DEADLINE_MS = 90_000;
@@ -20,6 +21,7 @@ export type DueCheckSummary = {
pending: number;
opened: number;
retries: number;
+ events: AutopilotEvent[];
};
export async function runDueChecks(env: Env, ctx?: Pick): Promise {
@@ -37,8 +39,11 @@ export async function runDueChecks(env: Env, ctx?: Pick item.statements);
+ let statementChunk: (typeof persisted)[number]['statements'] = [];
+ for (const item of persisted) {
+ if (statementChunk.length > 0 && statementChunk.length + item.statements.length > MAX_BATCH_STATEMENTS) {
+ await db.batch(statementChunk as [(typeof statementChunk)[number], ...typeof statementChunk]);
+ statementChunk = [];
+ }
+ statementChunk.push(...item.statements);
+ }
+ if (statementChunk.length > 0) await db.batch(statementChunk as [(typeof statementChunk)[number], ...typeof statementChunk]);
- await db.batch(statements as [(typeof statements)[number], ...typeof statements]);
-
- let aiMessagesQueued = 0;
const notificationBudget: NotificationBudget = { remaining: MAX_NOTIFICATIONS_PER_RUN };
const notifications = persisted.flatMap((item) => {
const work: Promise[] = [];
if (item.transition === 'opened' || item.transition === 'resolved') {
- const kind: AlertTransition = item.transition;
- work.push(
- dispatchNotification(
- env,
- {
- monitor: { id: item.monitor.id, name: item.monitor.name, url: item.monitor.url },
- kind: kind === 'opened' ? 'down' : 'recovered',
- incidentId: null,
- title: kind === 'opened' ? `${item.monitor.name} is down` : `${item.monitor.name} recovered`,
- body: item.result.error,
- statusCode: item.result.statusCode,
- error: item.result.error,
- at: item.checkedAt,
- },
- notificationBudget,
- ),
- );
+ work.push(dispatchNotification(env, buildAlertEvent(item.monitor, item.result, item.transition, item.checkedAt), notificationBudget));
}
if (item.latencyTransition) {
- const degraded = item.latencyTransition === 'degraded';
work.push(
- dispatchNotification(
- env,
- {
- monitor: { id: item.monitor.id, name: item.monitor.name, url: item.monitor.url },
- kind: degraded ? 'degraded' : 'recovered_degraded',
- incidentId: null,
- title: degraded ? `${item.monitor.name} performance degraded` : `${item.monitor.name} performance recovered`,
- body: degraded ? `Response time was ${item.result.latencyMs} ms.` : 'Response time returned to normal.',
- statusCode: item.result.statusCode,
- error: item.result.error,
- at: item.checkedAt,
- },
- notificationBudget,
- ),
+ dispatchNotification(env, buildAlertEvent(item.monitor, item.result, item.latencyTransition, item.checkedAt), notificationBudget),
);
}
- if (item.transition === 'opened' && item.monitor.alertsEnabled && aiMessagesQueued < MAX_AI_MESSAGES_PER_RUN) {
- aiMessagesQueued += 1;
- work.push(generateIncidentMessage(env, { monitor: item.monitor, result: item.result }));
- }
return work;
});
if (notifications.length > 0) {
@@ -126,6 +104,15 @@ export async function runDueChecks(env: Env, ctx?: Pick count + Number(item.result.ok), 0);
+ const events: AutopilotEvent[] = persisted
+ .filter((item) => item.transition === 'opened' || item.transition === 'resolved' || item.latencyTransition !== null)
+ .map((item) => ({
+ monitor: item.monitor,
+ result: item.result,
+ transition: item.transition === 'opened' || item.transition === 'resolved' ? item.transition : null,
+ latencyTransition: item.latencyTransition,
+ checkedAt: item.checkedAt,
+ }));
return {
checked: completed.length,
up,
@@ -133,5 +120,6 @@ export async function runDueChecks(env: Env, ctx?: Pick count + Number(item.transition === 'pending'), 0),
opened: persisted.reduce((count, item) => count + Number(item.transition === 'opened'), 0),
retries: completed.reduce((count, item) => count + item.result.attempts - 1, 0),
+ events,
};
}
diff --git a/src/worker/db/schema.ts b/src/worker/db/schema.ts
index dd0e52a..89573e7 100644
--- a/src/worker/db/schema.ts
+++ b/src/worker/db/schema.ts
@@ -13,6 +13,11 @@ export const aiSettings = sqliteTable('ai_settings', {
baseUrl: text('base_url'),
apiKey: text('api_key'),
model: text('model'),
+ autopilotEnabled: integer('autopilot_enabled', { mode: 'boolean' }).notNull().default(false),
+ autopilotFollowupMinutes: integer('autopilot_followup_minutes').notNull().default(15),
+ autopilotMaxUpdates: integer('autopilot_max_updates').notNull().default(6),
+ autopilotAdvanceStatus: integer('autopilot_advance_status', { mode: 'boolean' }).notNull().default(false),
+ autopilotDegradedIncidents: integer('autopilot_degraded_incidents', { mode: 'boolean' }).notNull().default(false),
createdAt: integer('created_at', { mode: 'timestamp_ms' }).notNull(),
updatedAt: integer('updated_at', { mode: 'timestamp_ms' }).notNull(),
});
@@ -134,6 +139,7 @@ export const incidents = sqliteTable(
status: text('status').notNull().default('investigating'),
impact: text('impact').notNull().default('major'),
source: text('source').notNull().default('auto'),
+ kind: text('kind').notNull().default('down'),
startedAt: integer('started_at', { mode: 'timestamp_ms' }).notNull(),
resolvedAt: integer('resolved_at', { mode: 'timestamp_ms' }),
startStatusCode: integer('start_status_code'),
@@ -174,6 +180,26 @@ export const incidentUpdates = sqliteTable(
(table) => [index('incident_updates_incident_id_created_at_idx').on(table.incidentId, table.createdAt)],
);
+export const aiEvents = sqliteTable(
+ 'ai_events',
+ {
+ id: integer('id').primaryKey({ autoIncrement: true }),
+ kind: text('kind').notNull(),
+ incidentId: integer('incident_id').references(() => incidents.id, { onDelete: 'set null' }),
+ monitorId: integer('monitor_id').references(() => monitors.id, { onDelete: 'set null' }),
+ model: text('model'),
+ outcome: text('outcome').notNull(),
+ reason: text('reason'),
+ latencyMs: integer('latency_ms'),
+ promptTokens: integer('prompt_tokens'),
+ completionTokens: integer('completion_tokens'),
+ contextPreview: text('context_preview'),
+ outputPreview: text('output_preview'),
+ createdAt: integer('created_at', { mode: 'timestamp_ms' }).notNull(),
+ },
+ (table) => [index('ai_events_created_at_idx').on(table.createdAt)],
+);
+
export const notificationChannels = sqliteTable(
'notification_channels',
{
diff --git a/src/worker/index.ts b/src/worker/index.ts
index bbd3d86..7cf92e2 100644
--- a/src/worker/index.ts
+++ b/src/worker/index.ts
@@ -1,5 +1,6 @@
import { Hono } from 'hono';
import { csrf } from 'hono/csrf';
+import { runAutopilot } from './autopilot';
import { runDueChecks } from './checks/run-due-checks';
import authRoutes from './routes/auth';
import channelRoutes from './routes/channels';
@@ -53,12 +54,18 @@ export default {
await cleanupExpiredAuthRecords(env);
const result = await runDueChecks(env, ctx);
+ ctx.waitUntil(
+ runAutopilot(env, { events: result.events }).then((autopilot) => {
+ console.log(JSON.stringify({ message: 'autopilot run completed', ...autopilot }));
+ }),
+ );
console.log(
JSON.stringify({
message: 'scheduled run completed',
cron: controller.cron,
scheduledTime: controller.scheduledTime,
...result,
+ events: result.events.length,
}),
);
},
diff --git a/src/worker/lib/humanize.ts b/src/worker/lib/humanize.ts
new file mode 100644
index 0000000..09415aa
--- /dev/null
+++ b/src/worker/lib/humanize.ts
@@ -0,0 +1,48 @@
+import type { CheckResult, Monitor } from '../checks/run-check';
+
+export function humanizeInterval(seconds: number): string {
+ if (seconds >= 3600 && seconds % 3600 === 0) return `${seconds / 3600} h`;
+ if (seconds >= 60 && seconds % 60 === 0) return `${seconds / 60} min`;
+ return `${seconds} s`;
+}
+
+export function humanizeDuration(ms: number): string {
+ const minutes = Math.round(ms / 60_000);
+ if (minutes < 1) return 'under a minute';
+ if (minutes < 60) return `${minutes} min`;
+ const hours = Math.floor(minutes / 60);
+ const rest = minutes % 60;
+ return rest ? `${hours} h ${rest} min` : `${hours} h`;
+}
+
+export function classifyFailure(monitor: Pick, result: CheckResult): string {
+ if (result.statusCode === null) {
+ const error = (result.error ?? '').toLowerCase();
+ if (/tim(?:e|ed)\s?out|timeout|deadline|aborted/.test(error))
+ return 'Request timed out - no response came back before the timeout limit';
+ if (/getaddrinfo|enotfound|dns|name not resolved|could not resolve|eai_again/.test(error)) {
+ return 'DNS lookup failed - the hostname could not be resolved to an address';
+ }
+ if (/certificate|cert(?:\s|_)|self[- ]signed|tls|ssl|handshake|err_cert/.test(error)) {
+ return 'TLS/SSL failure - the certificate is invalid, expired, or untrusted';
+ }
+ if (/econnrefused|connection refused|refused to connect/.test(error)) {
+ return 'Connection refused - nothing is accepting connections at that address';
+ }
+ if (/econnreset|connection reset|socket hang up|premature close/.test(error))
+ return 'Connection dropped before any response was returned';
+ if (/ehostunreach|enetunreach|network is unreachable|no route to host/.test(error)) {
+ return 'Network unreachable - the host could not be contacted at all';
+ }
+ return 'No HTTP response was received from the endpoint';
+ }
+
+ const code = result.statusCode;
+ if (code >= 500) return `Server error - the endpoint answered with HTTP ${code}`;
+ if (code === 429) return 'The endpoint is rate limiting - HTTP 429 Too Many Requests';
+ if (code === 401 || code === 403) return `The endpoint rejected the health check as unauthorized - HTTP ${code}`;
+ if (code === 404) return 'The health-check path returned HTTP 404 Not Found';
+ if (code >= 400) return `Client error - the endpoint answered with HTTP ${code}`;
+ if (code >= 300) return `Unexpected redirect - the endpoint answered with HTTP ${code}`;
+ return `The endpoint answered with HTTP ${code}, but HTTP ${monitor.expectedStatus} was expected`;
+}
diff --git a/src/worker/notifications/compose.ts b/src/worker/notifications/compose.ts
new file mode 100644
index 0000000..c609576
--- /dev/null
+++ b/src/worker/notifications/compose.ts
@@ -0,0 +1,37 @@
+import type { CheckResult, Monitor } from '../checks/run-check';
+import type { AlertTransition, LatencyTransition } from '../checks/persist-result';
+import type { NotificationEvent } from './providers';
+
+type MonitorIdentity = Pick;
+
+export function buildAlertEvent(
+ monitor: MonitorIdentity,
+ result: CheckResult,
+ transition: AlertTransition | LatencyTransition,
+ checkedAt: Date,
+): NotificationEvent {
+ if (transition === 'opened' || transition === 'resolved') {
+ const opened = transition === 'opened';
+ return {
+ monitor,
+ kind: opened ? 'down' : 'recovered',
+ incidentId: null,
+ title: opened ? `${monitor.name} is down` : `${monitor.name} recovered`,
+ body: result.error,
+ statusCode: result.statusCode,
+ error: result.error,
+ at: checkedAt,
+ };
+ }
+ const degraded = transition === 'degraded';
+ return {
+ monitor,
+ kind: degraded ? 'degraded' : 'recovered_degraded',
+ incidentId: null,
+ title: degraded ? `${monitor.name} performance degraded` : `${monitor.name} performance recovered`,
+ body: degraded ? `Response time was ${result.latencyMs} ms.` : 'Response time returned to normal.',
+ statusCode: result.statusCode,
+ error: result.error,
+ at: checkedAt,
+ };
+}
diff --git a/src/worker/notifications/dispatch.ts b/src/worker/notifications/dispatch.ts
index d67e997..4855acb 100644
--- a/src/worker/notifications/dispatch.ts
+++ b/src/worker/notifications/dispatch.ts
@@ -1,6 +1,7 @@
-import { and, desc, eq, inArray, isNotNull, isNull } from 'drizzle-orm';
+import { and, eq, inArray } from 'drizzle-orm';
+import { findLatestAutoIncidentForMonitor } from '../autopilot/signal';
import { getDb } from '../db/client';
-import { incidents, incidentMonitors, notificationChannelMonitors, notificationChannels, notificationDeliveries } from '../db/schema';
+import { incidentMonitors, notificationChannelMonitors, notificationChannels, notificationDeliveries } from '../db/schema';
import {
CHANNEL_TYPES,
formatChannel,
@@ -69,19 +70,10 @@ export async function dispatchNotification(env: Env, event: NotificationEvent, b
const db = getDb(env);
let effectiveEvent = event;
if (event.incidentId === null && event.monitor && (event.kind === 'down' || event.kind === 'recovered')) {
- const [incident] = await db
- .select({ id: incidents.id })
- .from(incidents)
- .innerJoin(incidentMonitors, eq(incidentMonitors.incidentId, incidents.id))
- .where(
- and(
- eq(incidentMonitors.monitorId, event.monitor.id),
- eq(incidents.source, 'auto'),
- event.kind === 'down' ? isNull(incidents.resolvedAt) : isNotNull(incidents.resolvedAt),
- ),
- )
- .orderBy(desc(incidents.updatedAt))
- .limit(1);
+ const incident = await findLatestAutoIncidentForMonitor(db, event.monitor.id, {
+ resolved: event.kind === 'recovered',
+ kind: 'down',
+ });
if (incident) effectiveEvent = { ...event, incidentId: incident.id };
}
let targetMonitorIds = event.monitor ? [event.monitor.id] : [];
diff --git a/src/worker/routes/incidents.ts b/src/worker/routes/incidents.ts
index 6625aec..39f71ed 100644
--- a/src/worker/routes/incidents.ts
+++ b/src/worker/routes/incidents.ts
@@ -267,6 +267,9 @@ incidentRoutes.patch('/:id', async (context) => {
);
}
if (parsed.value.monitorIds) {
+ if (parsed.value.title === undefined && parsed.value.impact === undefined) {
+ statements.push(db.update(incidents).set({ updatedAt: new Date() }).where(eq(incidents.id, id)));
+ }
statements.push(db.delete(incidentMonitors).where(eq(incidentMonitors.incidentId, id)));
statements.push(...parsed.value.monitorIds.map((monitorId) => db.insert(incidentMonitors).values({ incidentId: id, monitorId })));
}
diff --git a/src/worker/routes/monitors.ts b/src/worker/routes/monitors.ts
index 312c19a..2ab6ed1 100644
--- a/src/worker/routes/monitors.ts
+++ b/src/worker/routes/monitors.ts
@@ -1,15 +1,16 @@
import { and, desc, eq, gte, isNull, or, sql } from 'drizzle-orm';
import { Hono } from 'hono';
-import { generateIncidentMessage } from '../ai/incident-message';
+import { runAutopilot } from '../autopilot';
import { buildResultStatements } from '../checks/persist-result';
import { MAX_RETRY_COUNT, runCheckWithRetries } from '../checks/run-check';
import { getDb } from '../db/client';
-import { checks, incidentMonitors, incidents, maintenanceWindowMonitors, monitors } from '../db/schema';
+import { aiSettings, checks, incidentMonitors, incidents, maintenanceWindowMonitors, monitors } from '../db/schema';
import { requireAuth, type AuthVariables } from '../lib/require-auth';
import { loadActiveMaintenance } from '../maintenance/windows';
import { isSafeRemoteUrl } from '../lib/safe-url';
import { readBodyLimited } from '../lib/read-body';
import { dispatchNotification } from '../notifications/dispatch';
+import { buildAlertEvent } from '../notifications/compose';
type MonitorMethod = 'GET' | 'HEAD' | 'POST';
@@ -600,42 +601,35 @@ monitorRoutes.post('/:id/check', async (context) => {
const result = await runCheckWithRetries(monitor);
const checkedAt = new Date();
const activeMaintenance = await loadActiveMaintenance(db, checkedAt);
+ const [settings] = await db.select().from(aiSettings).where(eq(aiSettings.id, 1)).limit(1);
const { statements, transition, latencyTransition } = buildResultStatements(
db,
monitor,
result,
checkedAt,
activeMaintenance.has(monitor.id),
+ { degradedIncidents: Boolean(settings?.autopilotEnabled && settings.autopilotDegradedIncidents) },
);
await db.batch(statements as [(typeof statements)[number], ...typeof statements]);
if (transition === 'opened' || transition === 'resolved') {
- await dispatchNotification(context.env, {
- monitor: { id: monitor.id, name: monitor.name, url: monitor.url },
- kind: transition === 'opened' ? 'down' : 'recovered',
- incidentId: null,
- title: transition === 'opened' ? `${monitor.name} is down` : `${monitor.name} recovered`,
- body: result.error,
- statusCode: result.statusCode,
- error: result.error,
- at: checkedAt,
- });
- if (transition === 'opened') {
- await generateIncidentMessage(context.env, { monitor, result });
- }
+ await dispatchNotification(context.env, buildAlertEvent(monitor, result, transition, checkedAt));
}
if (latencyTransition) {
- const degraded = latencyTransition === 'degraded';
- await dispatchNotification(context.env, {
- monitor: { id: monitor.id, name: monitor.name, url: monitor.url },
- kind: degraded ? 'degraded' : 'recovered_degraded',
- incidentId: null,
- title: degraded ? `${monitor.name} performance degraded` : `${monitor.name} performance recovered`,
- body: degraded ? `Response time was ${result.latencyMs} ms.` : 'Response time returned to normal.',
- statusCode: result.statusCode,
- error: result.error,
- at: checkedAt,
- });
+ await dispatchNotification(context.env, buildAlertEvent(monitor, result, latencyTransition, checkedAt));
}
+ await runAutopilot(context.env, {
+ events: [
+ {
+ monitor,
+ result,
+ transition: transition === 'opened' || transition === 'resolved' ? transition : null,
+ latencyTransition,
+ checkedAt,
+ },
+ ],
+ budget: { remaining: 1 },
+ skipSweep: true,
+ });
const [updated] = await db.select().from(monitors).where(eq(monitors.id, monitor.id)).limit(1);
return context.json({ result, transition, latencyTransition, monitor: updated });
diff --git a/src/worker/routes/settings.ts b/src/worker/routes/settings.ts
index a681bce..0aa9959 100644
--- a/src/worker/routes/settings.ts
+++ b/src/worker/routes/settings.ts
@@ -1,10 +1,11 @@
-import { eq } from 'drizzle-orm';
+import { desc, eq, gte, sql } from 'drizzle-orm';
import { Hono } from 'hono';
-import { requestCompletion } from '../ai/client';
+import { requestCompletionDetailed } from '../ai/client';
+import { recordAiEvent } from '../ai/events';
import { SAMPLE_INCIDENT_CONTEXT } from '../ai/incident-context';
import { INCIDENT_MESSAGE_SYSTEM_PROMPT, sanitizeIncidentMessage } from '../ai/incident-message';
import { getDb } from '../db/client';
-import { aiSettings } from '../db/schema';
+import { aiEvents, aiSettings } from '../db/schema';
import { requireAuth, type AuthVariables } from '../lib/require-auth';
import { isSafeRemoteUrl } from '../lib/safe-url';
@@ -13,14 +14,30 @@ type AiInput = {
baseUrl: string | null;
model: string | null;
apiKey?: string;
+ autopilotEnabled: boolean;
+ autopilotFollowupMinutes: number;
+ autopilotMaxUpdates: number;
+ autopilotAdvanceStatus: boolean;
+ autopilotDegradedIncidents: boolean;
};
+function clampInteger(value: unknown, fallback: number, minimum: number, maximum: number) {
+ return typeof value === 'number' && Number.isSafeInteger(value) ? Math.min(maximum, Math.max(minimum, value)) : fallback;
+}
+
function parseAiInput(value: unknown): AiInput | string {
if (typeof value !== 'object' || value === null || Array.isArray(value)) {
return 'Invalid request body';
}
const body = value as Record;
if (typeof body.enabled !== 'boolean') return 'enabled must be a boolean';
+ if (body.autopilotEnabled !== undefined && typeof body.autopilotEnabled !== 'boolean') return 'autopilotEnabled must be a boolean';
+ if (body.autopilotAdvanceStatus !== undefined && typeof body.autopilotAdvanceStatus !== 'boolean') {
+ return 'autopilotAdvanceStatus must be a boolean';
+ }
+ if (body.autopilotDegradedIncidents !== undefined && typeof body.autopilotDegradedIncidents !== 'boolean') {
+ return 'autopilotDegradedIncidents must be a boolean';
+ }
const rawBaseUrl = typeof body.baseUrl === 'string' ? body.baseUrl.trim() : body.baseUrl;
if (rawBaseUrl !== null && typeof rawBaseUrl !== 'string') return 'baseUrl must be a URL or null';
@@ -46,7 +63,17 @@ function parseAiInput(value: unknown): AiInput | string {
apiKey = body.apiKey.trim();
}
- return { enabled: body.enabled, baseUrl, model, apiKey };
+ return {
+ enabled: body.enabled,
+ baseUrl,
+ model,
+ apiKey,
+ autopilotEnabled: body.autopilotEnabled ?? false,
+ autopilotFollowupMinutes: clampInteger(body.autopilotFollowupMinutes, 15, 5, 240),
+ autopilotMaxUpdates: clampInteger(body.autopilotMaxUpdates, 6, 1, 20),
+ autopilotAdvanceStatus: body.autopilotAdvanceStatus ?? false,
+ autopilotDegradedIncidents: body.autopilotDegradedIncidents ?? false,
+ };
}
function publicAiSettings(settings: typeof aiSettings.$inferSelect | undefined) {
@@ -55,6 +82,11 @@ function publicAiSettings(settings: typeof aiSettings.$inferSelect | undefined)
enabled: settings?.enabled ?? false,
baseUrl: settings?.baseUrl ?? null,
model: settings?.model ?? null,
+ autopilotEnabled: settings?.autopilotEnabled ?? false,
+ autopilotFollowupMinutes: settings?.autopilotFollowupMinutes ?? 15,
+ autopilotMaxUpdates: settings?.autopilotMaxUpdates ?? 6,
+ autopilotAdvanceStatus: settings?.autopilotAdvanceStatus ?? false,
+ autopilotDegradedIncidents: settings?.autopilotDegradedIncidents ?? false,
apiKeySet: Boolean(settings?.apiKey),
apiKeyPreview: settings?.apiKey ? `••••••${settings.apiKey.slice(-4)}` : null,
createdAt: settings?.createdAt ?? null,
@@ -88,12 +120,28 @@ settingsRoutes.put('/ai', async (context) => {
if (input.enabled && !apiKey) return context.json({ message: 'An API key is required when AI messages are enabled' }, 400);
const now = new Date();
+ const autopilot = {
+ autopilotEnabled: input.autopilotEnabled,
+ autopilotFollowupMinutes: input.autopilotFollowupMinutes,
+ autopilotMaxUpdates: input.autopilotMaxUpdates,
+ autopilotAdvanceStatus: input.autopilotAdvanceStatus,
+ autopilotDegradedIncidents: input.autopilotDegradedIncidents,
+ };
const [settings] = await db
.insert(aiSettings)
- .values({ id: 1, enabled: input.enabled, baseUrl: input.baseUrl, apiKey, model: input.model, createdAt: now, updatedAt: now })
+ .values({
+ id: 1,
+ enabled: input.enabled,
+ baseUrl: input.baseUrl,
+ apiKey,
+ model: input.model,
+ ...autopilot,
+ createdAt: now,
+ updatedAt: now,
+ })
.onConflictDoUpdate({
target: aiSettings.id,
- set: { enabled: input.enabled, baseUrl: input.baseUrl, apiKey, model: input.model, updatedAt: now },
+ set: { enabled: input.enabled, baseUrl: input.baseUrl, apiKey, model: input.model, ...autopilot, updatedAt: now },
})
.returning();
return context.json({ settings: publicAiSettings(settings) });
@@ -104,14 +152,53 @@ settingsRoutes.post('/ai/test', async (context) => {
if (!settings?.baseUrl || !settings.apiKey || !settings.model) {
return context.json({ message: 'Save a base URL, API key, and model first' }, 400);
}
- const content = await requestCompletion(
+ const result = await requestCompletionDetailed(
{ baseUrl: settings.baseUrl, apiKey: settings.apiKey, model: settings.model },
INCIDENT_MESSAGE_SYSTEM_PROMPT,
SAMPLE_INCIDENT_CONTEXT,
);
- const message = content ? sanitizeIncidentMessage(content) : null;
+ const message = result.content ? sanitizeIncidentMessage(result.content) : null;
+ await recordAiEvent(context.env, {
+ kind: 'settings_test',
+ model: settings.model,
+ outcome: message ? 'ok' : result.failure ? 'failed' : 'rejected',
+ reason: result.failure ?? (message ? null : 'sanitizer_rejected'),
+ latencyMs: result.latencyMs,
+ promptTokens: result.promptTokens,
+ completionTokens: result.completionTokens,
+ contextPreview: SAMPLE_INCIDENT_CONTEXT,
+ outputPreview: result.content,
+ });
if (!message) return context.json({ message: 'AI message generation failed' }, 502);
return context.json({ ok: true, message });
});
+settingsRoutes.get('/ai/events', async (context) => {
+ const rawLimit = Number(context.req.query('limit') ?? 50);
+ const limit = Number.isSafeInteger(rawLimit) ? Math.min(100, Math.max(1, rawLimit)) : 50;
+ const db = getDb(context.env);
+ const events = await db.select().from(aiEvents).orderBy(desc(aiEvents.createdAt)).limit(limit);
+ const since = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000);
+ const [summary] = await db
+ .select({
+ total: sql`count(*)`,
+ ok: sql`coalesce(sum(case when ${aiEvents.outcome} = 'ok' then 1 else 0 end), 0)`,
+ averageLatencyMs: sql`round(avg(${aiEvents.latencyMs}))`,
+ promptTokens: sql`coalesce(sum(${aiEvents.promptTokens}), 0)`,
+ completionTokens: sql`coalesce(sum(${aiEvents.completionTokens}), 0)`,
+ })
+ .from(aiEvents)
+ .where(gte(aiEvents.createdAt, since));
+ return context.json({
+ events: events.map((event) => ({ ...event, createdAt: event.createdAt.toISOString() })),
+ summary: {
+ total: Number(summary?.total ?? 0),
+ ok: Number(summary?.ok ?? 0),
+ averageLatencyMs: summary?.averageLatencyMs === null ? null : Number(summary?.averageLatencyMs ?? 0),
+ promptTokens: Number(summary?.promptTokens ?? 0),
+ completionTokens: Number(summary?.completionTokens ?? 0),
+ },
+ });
+});
+
export default settingsRoutes;
diff --git a/src/worker/routes/status.ts b/src/worker/routes/status.ts
index e3d7592..54f8476 100644
--- a/src/worker/routes/status.ts
+++ b/src/worker/routes/status.ts
@@ -23,6 +23,7 @@ type PublicIncident = {
status: string;
impact: string;
source: string;
+ kind: string;
startedAt: Date;
resolvedAt: Date | null;
durationMs: number | null;
@@ -103,6 +104,7 @@ const incidentSelection = {
status: incidents.status,
impact: incidents.impact,
source: incidents.source,
+ kind: incidents.kind,
startedAt: incidents.startedAt,
resolvedAt: incidents.resolvedAt,
durationMs: incidents.durationMs,
@@ -177,10 +179,11 @@ statusRoutes.get('/', async (context) => {
if (buckets) buckets.push(row);
else bucketsByMonitor.set(row.monitorId, [row]);
}
- const activeIncidentByMonitor = new Map();
+ const downIncidentByMonitor = new Map();
+ const degradedIncidentByMonitor = new Map();
for (const incident of activeIncidentRows) {
- for (const service of incidentServices.get(incident.id) ?? [])
- if (!activeIncidentByMonitor.has(service.id)) activeIncidentByMonitor.set(service.id, incident);
+ const target = incident.kind === 'degraded' ? degradedIncidentByMonitor : downIncidentByMonitor;
+ for (const service of incidentServices.get(incident.id) ?? []) if (!target.has(service.id)) target.set(service.id, incident);
}
const services = monitorRows.map((monitor) => {
const buckets = bucketsByMonitor.get(monitor.id) ?? [];
@@ -194,7 +197,8 @@ statusRoutes.get('/', async (context) => {
uptimePct: roundUptime(bucket.upChecks, bucket.totalChecks),
};
});
- const incident = activeIncidentByMonitor.get(monitor.id);
+ const downIncident = downIncidentByMonitor.get(monitor.id);
+ const degradedIncident = degradedIncidentByMonitor.get(monitor.id);
const maintenance = activeMaintenance.get(monitor.id);
return {
id: monitor.id,
@@ -202,11 +206,13 @@ statusRoutes.get('/', async (context) => {
status: maintenance ? ('maintenance' as const) : serviceStatus(monitor.lastOk, monitor.lastDegraded),
message:
!maintenance && monitor.lastOk === false
- ? incident
- ? (latestUpdates.get(incident.id)?.body ?? deterministicIncidentMessage(incident.startStatusCode))
+ ? downIncident
+ ? (latestUpdates.get(downIncident.id)?.body ?? deterministicIncidentMessage(downIncident.startStatusCode))
: deterministicIncidentMessage(monitor.lastStatusCode)
: !maintenance && monitor.lastOk === true && monitor.lastDegraded
- ? DEGRADED_MESSAGE
+ ? degradedIncident
+ ? (latestUpdates.get(degradedIncident.id)?.body ?? DEGRADED_MESSAGE)
+ : DEGRADED_MESSAGE
: null,
maintenance: maintenance ? { name: maintenance.name, endsAt: maintenance.endsAt.toISOString() } : null,
lastCheckedAt: monitor.lastCheckedAt?.toISOString() ?? null,
diff --git a/src/worker/scheduled/cleanup.ts b/src/worker/scheduled/cleanup.ts
index 7a7909d..efe9493 100644
--- a/src/worker/scheduled/cleanup.ts
+++ b/src/worker/scheduled/cleanup.ts
@@ -1,11 +1,12 @@
import { lt } from 'drizzle-orm';
import { getDb } from '../db/client';
-import { checks, loginAttempts, monitorDailyStats, notificationDeliveries, sessions } from '../db/schema';
+import { aiEvents, checks, loginAttempts, monitorDailyStats, notificationDeliveries, sessions } from '../db/schema';
const LOGIN_ATTEMPT_RETENTION_MS = 60 * 60 * 1000;
const CHECK_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
const DAILY_STATS_RETENTION_MS = 400 * 24 * 60 * 60 * 1000;
const DELIVERY_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
+export const AI_EVENT_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
export async function cleanupExpiredAuthRecords(env: Env) {
const db = getDb(env);
@@ -17,5 +18,6 @@ export async function cleanupExpiredAuthRecords(env: Env) {
db.delete(checks).where(lt(checks.checkedAt, new Date(now.getTime() - CHECK_RETENTION_MS))),
db.delete(monitorDailyStats).where(lt(monitorDailyStats.day, new Date(now.getTime() - DAILY_STATS_RETENTION_MS))),
db.delete(notificationDeliveries).where(lt(notificationDeliveries.createdAt, new Date(now.getTime() - DELIVERY_RETENTION_MS))),
+ db.delete(aiEvents).where(lt(aiEvents.createdAt, new Date(now.getTime() - AI_EVENT_RETENTION_MS))),
]);
}
diff --git a/test/ai.spec.ts b/test/ai.spec.ts
index 05258f7..540dd97 100644
--- a/test/ai.spec.ts
+++ b/test/ai.spec.ts
@@ -36,6 +36,7 @@ const failedResult = {
async function resetDatabase() {
await env.DB.batch([
+ env.DB.prepare('DELETE FROM ai_events'),
env.DB.prepare('DELETE FROM checks'),
env.DB.prepare('DELETE FROM incident_updates'),
env.DB.prepare('DELETE FROM incident_monitors'),
diff --git a/test/autopilot.spec.ts b/test/autopilot.spec.ts
new file mode 100644
index 0000000..0428bd2
--- /dev/null
+++ b/test/autopilot.spec.ts
@@ -0,0 +1,81 @@
+import { applyD1Migrations, type D1Migration } from 'cloudflare:test';
+import { env } from 'cloudflare:workers';
+import { beforeAll, describe, expect, it } from 'vitest';
+import { advanceStatus, computeImpact, nextFollowupDueAt } from '../src/worker/autopilot/cadence';
+import { recordAiEvent } from '../src/worker/ai/events';
+import { sanitizePublicTextWithReason } from '../src/worker/ai/sanitize';
+import { AI_EVENT_RETENTION_MS, cleanupExpiredAuthRecords } from '../src/worker/scheduled/cleanup';
+
+describe('incident autopilot', () => {
+ beforeAll(async () => {
+ const testEnv = env as Env & { TEST_MIGRATIONS: D1Migration[] };
+ await applyD1Migrations(testEnv.DB, testEnv.TEST_MIGRATIONS);
+ });
+
+ it('uses exponential cadence capped at eight times the configured interval', () => {
+ const start = 1_000_000;
+ expect(nextFollowupDueAt(start, 1, 15)).toBe(start + 15 * 60_000);
+ expect(nextFollowupDueAt(start, 2, 15)).toBe(start + 30 * 60_000);
+ expect(nextFollowupDueAt(start, 4, 15)).toBe(start + 120 * 60_000);
+ expect(nextFollowupDueAt(start, 9, 15)).toBe(start + 120 * 60_000);
+ });
+
+ it('advances only from objective signals', () => {
+ expect(
+ advanceStatus('investigating', {
+ consecutiveFailures: 3,
+ failureSignatureStable: true,
+ consecutiveOk: 0,
+ latestOk: false,
+ regressionUsed: false,
+ }),
+ ).toBe('identified');
+ expect(
+ advanceStatus('identified', {
+ consecutiveFailures: 0,
+ failureSignatureStable: false,
+ consecutiveOk: 2,
+ latestOk: true,
+ regressionUsed: false,
+ }),
+ ).toBe('monitoring');
+ });
+
+ it('computes impact deterministically', () => {
+ expect(computeImpact({ kind: 'degraded', affectedMonitors: 5, totalMonitors: 5, recentChecks: [] })).toBe('minor');
+ expect(computeImpact({ kind: 'down', affectedMonitors: 2, totalMonitors: 4, recentChecks: [false] })).toBe('critical');
+ expect(computeImpact({ kind: 'down', affectedMonitors: 1, totalMonitors: 4, recentChecks: [false, true] })).toBe('minor');
+ });
+
+ it('returns actionable sanitizer rejection reasons', () => {
+ expect(sanitizePublicTextWithReason('See https://private.example', 200)).toEqual({ text: null, reason: 'contains_url' });
+ expect(sanitizePublicTextWithReason('HTTP 503 is occurring', 200)).toEqual({ text: null, reason: 'contains_http_status' });
+ expect(sanitizePublicTextWithReason('A calm public update.', 200)).toEqual({ text: 'A calm public update.', reason: null });
+ });
+
+ it('removes AI audit rows after the retention period', async () => {
+ await env.DB.prepare('DELETE FROM ai_events').run();
+ await env.DB.prepare("INSERT INTO ai_events (kind, outcome, created_at) VALUES ('settings_test', 'ok', ?)")
+ .bind(Date.now() - AI_EVENT_RETENTION_MS - 1)
+ .run();
+ await cleanupExpiredAuthRecords(env);
+ expect((await env.DB.prepare('SELECT count(*) AS count FROM ai_events').first<{ count: number }>())?.count).toBe(0);
+ });
+
+ it('redacts sensitive values from AI audit previews', async () => {
+ await env.DB.prepare('DELETE FROM ai_events').run();
+ await recordAiEvent(env, {
+ kind: 'settings_test',
+ outcome: 'rejected',
+ reason: 'Failed at https://private.example.com with sk-secret-4f2a',
+ contextPreview: 'Probe https://api.example.com from 10.0.0.1 using Bearer private-token',
+ outputPreview: 'Visit status.example.com with sk-output-secret',
+ });
+ const text = JSON.stringify(await env.DB.prepare('SELECT reason, context_preview, output_preview FROM ai_events').first());
+ expect(text).not.toContain('private.example.com');
+ expect(text).not.toContain('api.example.com');
+ expect(text).not.toContain('10.0.0.1');
+ expect(text).not.toContain('private-token');
+ expect(text).not.toContain('sk-secret-4f2a');
+ });
+});
diff --git a/test/channels.spec.ts b/test/channels.spec.ts
index e0b37d2..301ed0b 100644
--- a/test/channels.spec.ts
+++ b/test/channels.spec.ts
@@ -34,6 +34,7 @@ describe('notification channels', () => {
});
beforeEach(async () => {
await env.DB.batch([
+ env.DB.prepare('DELETE FROM ai_events'),
env.DB.prepare('DELETE FROM notification_deliveries'),
env.DB.prepare('DELETE FROM notification_channel_monitors'),
env.DB.prepare('DELETE FROM notification_channels'),
diff --git a/test/checks.spec.ts b/test/checks.spec.ts
index 4b67d6c..566906e 100644
--- a/test/checks.spec.ts
+++ b/test/checks.spec.ts
@@ -1,10 +1,12 @@
import { applyD1Migrations, type D1Migration } from 'cloudflare:test';
import { env } from 'cloudflare:workers';
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import { runAutopilot } from '../src/worker/autopilot';
import { MAX_RETRY_ATTEMPTS_PER_RUN, runDueChecks } from '../src/worker/checks/run-due-checks';
async function clearMonitoringTables() {
await env.DB.batch([
+ env.DB.prepare('DELETE FROM ai_events'),
env.DB.prepare('DELETE FROM maintenance_window_monitors'),
env.DB.prepare('DELETE FROM maintenance_windows'),
env.DB.prepare('DELETE FROM checks'),
@@ -100,7 +102,7 @@ describe('scheduled monitor checks', () => {
);
const summary = await runDueChecks(env);
- expect(summary).toEqual({ checked: 1, up: 1, down: 0, pending: 0, opened: 0, retries: 0 });
+ expect(summary).toEqual({ checked: 1, up: 1, down: 0, pending: 0, opened: 0, retries: 0, events: [] });
const monitor = await env.DB.prepare('SELECT last_ok, last_status_code, last_latency_ms, last_checked_at FROM monitors WHERE id = ?')
.bind(id)
@@ -213,7 +215,7 @@ describe('scheduled monitor checks', () => {
vi.stubGlobal('fetch', fetchMock);
const summary = await runDueChecks(env);
- expect(summary).toEqual({ checked: 1, up: 0, down: 1, pending: 1, opened: 0, retries: 0 });
+ expect(summary).toEqual({ checked: 1, up: 0, down: 1, pending: 1, opened: 0, retries: 0, events: [] });
const monitor = await env.DB.prepare('SELECT last_ok, consecutive_failures, last_status_code, last_error FROM monitors WHERE id = ?')
.bind(id)
@@ -338,7 +340,7 @@ describe('scheduled monitor checks', () => {
VALUES ('Legacy webhook', 'webhook', '{"url":"https://hooks.example.test/events"}', 1, 1, ?, ?)`,
).bind(now, now),
env.DB.prepare(
- "INSERT INTO ai_settings (id, enabled, base_url, api_key, model, created_at, updated_at) VALUES (1, 1, 'https://ai.example.test/v1', 'secret', 'test-model', ?, ?)",
+ "INSERT INTO ai_settings (id, enabled, base_url, api_key, model, autopilot_enabled, created_at, updated_at) VALUES (1, 1, 'https://ai.example.test/v1', 'secret', 'test-model', 1, ?, ?)",
).bind(now, now),
]);
const webhookBodies: Array> = [];
@@ -357,7 +359,8 @@ describe('scheduled monitor checks', () => {
choices: [
{
message: {
- content: 'Some visitors may be unable to use the service. The team has been alerted and restoration work is underway.',
+ content:
+ 'TITLE: Service disruption\nBODY: Some visitors may be unable to use the service. Automated monitoring continues while service is restored.',
},
},
],
@@ -373,7 +376,8 @@ describe('scheduled monitor checks', () => {
expect((await env.DB.prepare('SELECT COUNT(*) AS count FROM incident_updates').first<{ count: number }>())?.count).toBe(0);
await env.DB.prepare('UPDATE monitors SET last_checked_at = NULL WHERE id = ?').bind(id).run();
- await runDueChecks(env);
+ const confirmed = await runDueChecks(env);
+ await runAutopilot(env, { events: confirmed.events });
expect(webhookBodies).toHaveLength(1);
expect(webhookBodies[0]).toMatchObject({ event: 'down' });
expect(aiCalls).toBe(1);
@@ -453,7 +457,7 @@ describe('scheduled monitor checks', () => {
vi.stubGlobal('fetch', fetchMock);
const summary = await runDueChecks(env);
- expect(summary).toEqual({ checked: 0, up: 0, down: 0, pending: 0, opened: 0, retries: 0 });
+ expect(summary).toEqual({ checked: 0, up: 0, down: 0, pending: 0, opened: 0, retries: 0, events: [] });
expect(fetchMock).not.toHaveBeenCalled();
});
});
diff --git a/test/incident-draft.spec.ts b/test/incident-draft.spec.ts
index fa0f51b..03f1720 100644
--- a/test/incident-draft.spec.ts
+++ b/test/incident-draft.spec.ts
@@ -6,6 +6,7 @@ import { hashPassword } from '../src/worker/lib/password';
const PASSWORD = 'correct-horse-battery-staple';
async function reset() {
await env.DB.batch([
+ env.DB.prepare('DELETE FROM ai_events'),
env.DB.prepare('DELETE FROM incident_updates'),
env.DB.prepare('DELETE FROM incident_monitors'),
env.DB.prepare('DELETE FROM incidents'),
diff --git a/test/incidents.spec.ts b/test/incidents.spec.ts
index 41ea8d0..c5033e4 100644
--- a/test/incidents.spec.ts
+++ b/test/incidents.spec.ts
@@ -6,6 +6,7 @@ import { hashPassword } from '../src/worker/lib/password';
const PASSWORD = 'correct-horse-battery-staple';
async function reset() {
await env.DB.batch([
+ env.DB.prepare('DELETE FROM ai_events'),
env.DB.prepare('DELETE FROM notification_deliveries'),
env.DB.prepare('DELETE FROM notification_channel_monitors'),
env.DB.prepare('DELETE FROM notification_channels'),
@@ -123,4 +124,23 @@ describe('incident lifecycle API', () => {
expect(await env.DB.prepare('SELECT id FROM incidents WHERE id = ?').bind(incidentId).first()).toBeTruthy();
expect(await env.DB.prepare('SELECT * FROM incident_monitors WHERE incident_id = ?').bind(incidentId).first()).toBeNull();
});
+ it('bumps updated_at when only monitor assignments change', async () => {
+ const firstId = await monitor('API');
+ const secondId = await monitor('Web');
+ const auth = await cookie();
+ const created = await request('/api/incidents', 'POST', auth, {
+ title: 'API issue',
+ impact: 'major',
+ status: 'investigating',
+ body: 'Some requests are failing.',
+ monitorIds: [firstId],
+ });
+ const incidentId = (await created.json<{ incident: { id: number } }>()).incident.id;
+ await env.DB.prepare('UPDATE incidents SET updated_at = 1 WHERE id = ?').bind(incidentId).run();
+ expect((await request(`/api/incidents/${incidentId}`, 'PATCH', auth, { monitorIds: [secondId] })).status).toBe(200);
+ const row = await env.DB.prepare('SELECT updated_at AS updatedAt FROM incidents WHERE id = ?')
+ .bind(incidentId)
+ .first<{ updatedAt: number }>();
+ expect(row?.updatedAt).toBeGreaterThan(1);
+ });
});
diff --git a/test/maintenance.spec.ts b/test/maintenance.spec.ts
index f628689..56e719a 100644
--- a/test/maintenance.spec.ts
+++ b/test/maintenance.spec.ts
@@ -21,6 +21,7 @@ function windowRow(overrides: Partial = {}): MaintenanceWi
async function resetDatabase() {
await env.DB.batch([
+ env.DB.prepare('DELETE FROM ai_events'),
env.DB.prepare('DELETE FROM maintenance_window_monitors'),
env.DB.prepare('DELETE FROM maintenance_windows'),
env.DB.prepare('DELETE FROM checks'),
diff --git a/test/monitors.spec.ts b/test/monitors.spec.ts
index 6578047..d5d5bef 100644
--- a/test/monitors.spec.ts
+++ b/test/monitors.spec.ts
@@ -16,6 +16,7 @@ const VALID_MONITOR = {
async function seedAdmin() {
await env.DB.batch([
+ env.DB.prepare('DELETE FROM ai_events'),
env.DB.prepare('DELETE FROM checks'),
env.DB.prepare('DELETE FROM incident_updates'),
env.DB.prepare('DELETE FROM incident_monitors'),
diff --git a/test/rollup.spec.ts b/test/rollup.spec.ts
index c95dfc6..8abdd71 100644
--- a/test/rollup.spec.ts
+++ b/test/rollup.spec.ts
@@ -10,6 +10,7 @@ describe('daily monitor rollups', () => {
});
beforeEach(async () => {
await env.DB.batch([
+ env.DB.prepare('DELETE FROM ai_events'),
env.DB.prepare('DELETE FROM maintenance_window_monitors'),
env.DB.prepare('DELETE FROM maintenance_windows'),
env.DB.prepare('DELETE FROM monitor_daily_stats'),
diff --git a/test/status.spec.ts b/test/status.spec.ts
index 800ef9c..cfa3262 100644
--- a/test/status.spec.ts
+++ b/test/status.spec.ts
@@ -37,6 +37,7 @@ type IncidentHistoryResponse = {
async function resetDatabase() {
await env.DB.batch([
+ env.DB.prepare('DELETE FROM ai_events'),
env.DB.prepare('DELETE FROM maintenance_window_monitors'),
env.DB.prepare('DELETE FROM maintenance_windows'),
env.DB.prepare('DELETE FROM checks'),
@@ -165,6 +166,52 @@ describe('public status API', () => {
expect(body.services[0]).toMatchObject({ status: 'degraded', message: DEGRADED_MESSAGE });
});
+ it('uses an open degraded incident update as the public latency message', async () => {
+ const monitorId = await insertMonitor({ name: 'Slow API', lastOk: true, lastDegraded: true, lastLatencyMs: 2200 });
+ const now = Date.now();
+ const inserted = await env.DB.prepare(
+ "INSERT INTO incidents (status, impact, source, kind, started_at, created_at, updated_at) VALUES ('investigating', 'minor', 'auto', 'degraded', ?, ?, ?)",
+ )
+ .bind(now, now, now)
+ .run();
+ const incidentId = Number(inserted.meta.last_row_id);
+ await env.DB.batch([
+ env.DB.prepare('INSERT INTO incident_monitors (incident_id, monitor_id) VALUES (?, ?)').bind(incidentId, monitorId),
+ env.DB.prepare(
+ "INSERT INTO incident_updates (incident_id, status, body, source, created_at) VALUES (?, 'investigating', 'Performance remains slower than usual.', 'ai', ?)",
+ ).bind(incidentId, now),
+ ]);
+ const body = await (await statusFetch()).json();
+ expect(body.services[0].message).toBe('Performance remains slower than usual.');
+ });
+
+ it('never shadows a down incident message with a degraded incident', async () => {
+ const monitorId = await insertMonitor({ name: 'API', lastOk: false, lastDegraded: false, lastStatusCode: 503 });
+ const now = Date.now();
+ const degraded = await env.DB.prepare(
+ "INSERT INTO incidents (status, impact, source, kind, started_at, created_at, updated_at) VALUES ('investigating', 'minor', 'auto', 'degraded', ?, ?, ?)",
+ )
+ .bind(now + 1, now + 1, now + 1)
+ .run();
+ const down = await env.DB.prepare(
+ "INSERT INTO incidents (status, impact, source, kind, started_at, start_status_code, created_at, updated_at) VALUES ('investigating', 'major', 'auto', 'down', ?, 503, ?, ?)",
+ )
+ .bind(now, now, now)
+ .run();
+ await env.DB.batch([
+ env.DB.prepare('INSERT INTO incident_monitors (incident_id, monitor_id) VALUES (?, ?)').bind(degraded.meta.last_row_id, monitorId),
+ env.DB.prepare('INSERT INTO incident_monitors (incident_id, monitor_id) VALUES (?, ?)').bind(down.meta.last_row_id, monitorId),
+ env.DB.prepare(
+ "INSERT INTO incident_updates (incident_id, status, body, source, created_at) VALUES (?, 'investigating', 'Degraded copy.', 'ai', ?)",
+ ).bind(degraded.meta.last_row_id, now),
+ env.DB.prepare(
+ "INSERT INTO incident_updates (incident_id, status, body, source, created_at) VALUES (?, 'investigating', 'Down copy.', 'ai', ?)",
+ ).bind(down.meta.last_row_id, now),
+ ]);
+ const body = await (await statusFetch()).json();
+ expect(body.services[0]).toMatchObject({ status: 'down', message: 'Down copy.' });
+ });
+
it('excludes disabled monitors and reports degraded health for a partial outage', async () => {
await insertMonitor({ name: 'Healthy', lastOk: true });
await insertMonitor({ name: 'Unavailable', lastOk: false });