fix: improve detect error case

migrations/0009_shocking_butterfly.sql

@@ -0,0 +1,8 @@
+ALTER TABLE `checks` ADD `degraded` integer DEFAULT false NOT NULL;--> statement-breakpoint
+ALTER TABLE `monitors` ADD `expect_keyword` text;--> statement-breakpoint
+ALTER TABLE `monitors` ADD `keyword_inverted` integer DEFAULT false NOT NULL;--> statement-breakpoint
+ALTER TABLE `monitors` ADD `request_headers` text;--> statement-breakpoint
+ALTER TABLE `monitors` ADD `request_body` text;--> statement-breakpoint
+ALTER TABLE `monitors` ADD `degraded_latency_ms` integer;--> statement-breakpoint
+ALTER TABLE `monitors` ADD `consecutive_slow` integer DEFAULT 0 NOT NULL;--> statement-breakpoint
+ALTER TABLE `monitors` ADD `last_degraded` integer DEFAULT false NOT NULL;

migrations/meta/_journal.json

@@ -64,6 +64,13 @@
       "when": 1788059207578,
       "tag": "0008_aromatic_chat",
       "breakpoints": true
+    },
+    {
+      "idx": 9,
+      "version": "6",
+      "when": 1788062139636,
+      "tag": "0009_shocking_butterfly",
+      "breakpoints": true
     }
   ]
 }

src/client/api/monitors.ts

@@ -8,6 +8,11 @@ export type Monitor = {
 	url: string;
 	method: MonitorMethod;
 	expectedStatus: number;
+	expectKeyword: string | null;
+	keywordInverted: boolean;
+	requestHeaders: string | null;
+	requestBody: string | null;
+	degradedLatencyMs: number | null;
 	intervalSeconds: number;
 	timeoutMs: number;
 	enabled: boolean;
@@ -15,7 +20,9 @@ export type Monitor = {
 	retryCount: number;
 	failureThreshold: number;
 	consecutiveFailures: number;
+	consecutiveSlow: number;
 	lastOk: boolean | null;
+	lastDegraded: boolean;
 	lastStatusCode: number | null;
 	lastLatencyMs: number | null;
 	lastError: string | null;
@@ -29,6 +36,11 @@ export type MonitorInput = {
 	url: string;
 	method: MonitorMethod;
 	expectedStatus: number;
+	expectKeyword?: string | null;
+	keywordInverted?: boolean;
+	requestHeaders?: Record<string, string> | null;
+	requestBody?: string | null;
+	degradedLatencyMs?: number | null;
 	intervalSeconds: number;
 	timeoutMs: number;
 	retryCount?: number;
@@ -39,6 +51,7 @@ export type MonitorInput = {
 
 export type CheckResult = {
 	ok: boolean;
+	degraded: boolean;
 	statusCode: number | null;
 	latencyMs: number;
 	error: string | null;
@@ -49,6 +62,7 @@ export type Check = {
 	id: number;
 	monitorId: number;
 	ok: boolean;
+	degraded: boolean;
 	statusCode: number | null;
 	latencyMs: number;
 	error: string | null;
@@ -57,6 +71,7 @@ export type Check = {
 };
 
 export type CheckTransition = 'opened' | 'pending' | 'cleared' | 'resolved' | null;
+export type LatencyTransition = 'degraded' | 'recovered' | null;
 
 export type Incident = {
 	id: number;
@@ -122,5 +137,7 @@ export function deleteMonitor(id: number) {
 }
 
 export function runMonitorCheck(id: number) {
-	return postJson<{ result: CheckResult; transition: CheckTransition; monitor: Monitor }>(`/api/monitors/${id}/check`);
+	return postJson<{ result: CheckResult; transition: CheckTransition; latencyTransition: LatencyTransition; monitor: Monitor }>(
+		`/api/monitors/${id}/check`,
+	);
 }

src/client/api/status.ts

@@ -1,6 +1,6 @@
 import { getJson } from './http';
 
-export type PublicServiceStatus = 'up' | 'down' | 'unknown' | 'maintenance';
+export type PublicServiceStatus = 'up' | 'degraded' | 'down' | 'unknown' | 'maintenance';
 export type PublicOverallStatus = 'operational' | 'degraded' | 'down';
 
 export type PublicIncidentUpdate = { status: string; body: string; createdAt: string };

src/client/components/charts/UptimeBar.tsx

@@ -12,7 +12,7 @@ type UptimeDatum = {
 	id: string;
 	successfulChecks: number;
 	totalChecks: number;
-	status: 'up' | 'down';
+	status: 'up' | 'degraded' | 'down';
 	fill: string;
 };
 
@@ -20,6 +20,7 @@ const MAX_VISIBLE_SEGMENTS = 32;
 
 const uptimeConfig = {
 	up: { label: 'Up', color: 'var(--primary)' },
+	degraded: { label: 'Degraded', color: 'var(--chart-degraded)' },
 	down: { label: 'Down', color: 'var(--chart-down)' },
 } satisfies ChartConfig;
 
@@ -58,6 +59,8 @@ function groupChecks(checks: Check[]): UptimeDatum[] {
 		const last = bucket[bucket.length - 1];
 		const successfulChecks = bucket.filter((check) => check.ok).length;
 		const ok = successfulChecks === bucket.length;
+		const degraded = ok && bucket.some((check) => check.degraded);
+		const status = !ok ? 'down' : degraded ? 'degraded' : 'up';
 
 		return {
 			startTime: first.checkedAt,
@@ -67,8 +70,8 @@ function groupChecks(checks: Check[]): UptimeDatum[] {
 			id: `${first.id}-${last.id}`,
 			successfulChecks,
 			totalChecks: bucket.length,
-			status: ok ? 'up' : 'down',
+			status,
-			fill: ok ? 'var(--color-up)' : 'var(--color-down)',
+			fill: `var(--color-${status})`,
 		};
 	});
 }
@@ -114,7 +117,7 @@ export function UptimeBar({ checks }: { checks: Check[] }) {
 							onAnimationEnd={() => setHasAnimated(true)}
 						>
 							{data.map((point) => (
-								<Cell key={point.id} fill={point.ok ? 'var(--color-up)' : 'var(--color-down)'} />
+								<Cell key={point.id} fill={point.fill} />
 							))}
 						</Bar>
 					</BarChart>
@@ -123,7 +126,7 @@ export function UptimeBar({ checks }: { checks: Check[] }) {
 			<div className="uptime-legend">
 				<span>Oldest</span>
 				<span>
-					<i className="legend-up" /> Up <i className="legend-down" /> Down
+					<i className="legend-up" /> Up <i className="legend-degraded" /> Degraded <i className="legend-down" /> Down
 				</span>
 				<span>Latest</span>
 			</div>

src/client/components/dashboard/MonitorFormDialog.tsx

@@ -1,4 +1,5 @@
 import { type FormEvent, useState } from 'react';
+import { ChevronDown, Plus, Trash2 } from 'lucide-react';
 import { Button } from '@/components/ui/button';
 import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from '@/components/ui/dialog';
 import { Input } from '@/components/ui/input';
@@ -16,7 +17,13 @@ export const DEFAULT_MONITOR_INPUT: MonitorInput = {
 	timeoutMs: 10_000,
 	retryCount: 1,
 	failureThreshold: 2,
+	expectKeyword: null,
+	keywordInverted: false,
+	requestHeaders: null,
+	requestBody: null,
+	degradedLatencyMs: null,
 	enabled: true,
+	alertsEnabled: true,
 };
 
 export const INTERVAL_OPTIONS = [
@@ -32,6 +39,17 @@ type MonitorFormDialogProps = {
 	onClose: () => void;
 };
 
+type HeaderRow = { id: number; name: string; value: string };
+
+function headerRows(requestHeaders: string | null): HeaderRow[] {
+	if (!requestHeaders) return [];
+	try {
+		return Object.entries(JSON.parse(requestHeaders) as Record<string, string>).map(([name, value], index) => ({ id: index, name, value }));
+	} catch {
+		return [];
+	}
+}
+
 function errorMessage(error: unknown, fallback: string) {
 	return error instanceof Error ? error.message : fallback;
 }
@@ -47,6 +65,11 @@ function monitorInput(monitor: Monitor | null): MonitorInput {
 		timeoutMs: monitor.timeoutMs,
 		retryCount: monitor.retryCount,
 		failureThreshold: monitor.failureThreshold,
+		expectKeyword: monitor.expectKeyword,
+		keywordInverted: monitor.keywordInverted,
+		requestHeaders: monitor.requestHeaders ? (JSON.parse(monitor.requestHeaders) as Record<string, string>) : null,
+		requestBody: monitor.requestBody,
+		degradedLatencyMs: monitor.degradedLatencyMs,
 		enabled: monitor.enabled,
 		alertsEnabled: monitor.alertsEnabled,
 	};
@@ -56,6 +79,8 @@ export function MonitorFormDialog({ editing, onClose }: MonitorFormDialogProps)
 	const createMutation = useCreateMonitorMutation();
 	const updateMutation = useUpdateMonitorMutation();
 	const [form, setForm] = useState<MonitorInput>(() => monitorInput(editing));
+	const [headers, setHeaders] = useState<HeaderRow[]>(() => headerRows(editing?.requestHeaders ?? null));
+	const [nextHeaderId, setNextHeaderId] = useState(() => headers.length);
 	const formMutation = editing ? updateMutation : createMutation;
 
 	function closeForm() {
@@ -65,13 +90,31 @@ export function MonitorFormDialog({ editing, onClose }: MonitorFormDialogProps)
 
 	function handleSubmit(event: FormEvent<HTMLFormElement>) {
 		event.preventDefault();
+		const requestHeaders = Object.fromEntries(
+			headers.filter((header) => header.name.trim()).map((header) => [header.name.trim(), header.value]),
+		);
+		const input: MonitorInput = {
+			...form,
+			expectKeyword: form.expectKeyword?.trim() || null,
+			requestHeaders: Object.keys(requestHeaders).length > 0 ? requestHeaders : null,
+			requestBody: form.method === 'POST' ? form.requestBody || null : null,
+		};
 		if (editing) {
-			updateMutation.mutate({ id: editing.id, input: form }, { onSuccess: onClose });
+			updateMutation.mutate({ id: editing.id, input }, { onSuccess: onClose });
 		} else {
-			createMutation.mutate(form, { onSuccess: onClose });
+			createMutation.mutate(input, { onSuccess: onClose });
 		}
 	}
 
+	function addHeader() {
+		setHeaders((current) => [...current, { id: nextHeaderId, name: '', value: '' }]);
+		setNextHeaderId((current) => current + 1);
+	}
+
+	function updateHeader(id: number, field: 'name' | 'value', value: string) {
+		setHeaders((current) => current.map((header) => (header.id === id ? { ...header, [field]: value } : header)));
+	}
+
 	return (
 		<Dialog
 			open
@@ -217,6 +260,109 @@ export function MonitorFormDialog({ editing, onClose }: MonitorFormDialogProps)
 						/>
 						<label htmlFor="monitor-alerts-enabled">Enable incident alerts</label>
 					</div>
+					<details className="monitor-advanced">
+						<summary>
+							<span>Advanced request and response checks</span>
+							<ChevronDown aria-hidden="true" />
+						</summary>
+						<div className="monitor-advanced-grid">
+							<label className="field advanced-keyword" htmlFor="monitor-keyword">
+								<span>Expected response keyword</span>
+								<Input
+									id="monitor-keyword"
+									value={form.expectKeyword ?? ''}
+									onChange={(event) => setForm({ ...form, expectKeyword: event.target.value || null })}
+									maxLength={200}
+									placeholder="healthy"
+								/>
+								<small>Case-insensitive match within the first 256 KB of the response.</small>
+							</label>
+							<div className="toggle-field advanced-inverted">
+								<Switch
+									id="monitor-keyword-inverted"
+									checked={form.keywordInverted ?? false}
+									onCheckedChange={(keywordInverted) => setForm({ ...form, keywordInverted })}
+									disabled={!form.expectKeyword}
+								/>
+								<label htmlFor="monitor-keyword-inverted">Fail when present</label>
+							</div>
+							<label className="field advanced-latency" htmlFor="monitor-degraded-latency">
+								<span>Degraded above (ms)</span>
+								<Input
+									id="monitor-degraded-latency"
+									type="number"
+									min="1"
+									max="30000"
+									value={form.degradedLatencyMs ?? ''}
+									onChange={(event) => setForm({ ...form, degradedLatencyMs: event.target.value ? event.target.valueAsNumber : null })}
+									placeholder="1500"
+								/>
+								<small>Publishes degraded performance after the configured confirmation count.</small>
+							</label>
+							<div className="advanced-headers">
+								<div className="advanced-section-heading">
+									<div>
+										<span>Request headers</span>
+										<small>Up to 10 headers. Restricted transport headers are blocked.</small>
+									</div>
+									<Button
+										variant="unstyled"
+										className="secondary-button header-add-button"
+										type="button"
+										onClick={addHeader}
+										disabled={headers.length >= 10}
+									>
+										<Plus aria-hidden="true" /> Add header
+									</Button>
+								</div>
+								{headers.length > 0 && (
+									<div className="header-rows">
+										{headers.map((header) => (
+											<div className="header-row" key={header.id}>
+												<Input
+													aria-label="Header name"
+													value={header.name}
+													onChange={(event) => updateHeader(header.id, 'name', event.target.value)}
+													maxLength={64}
+													placeholder="Authorization"
+												/>
+												<Input
+													aria-label="Header value"
+													value={header.value}
+													onChange={(event) => updateHeader(header.id, 'value', event.target.value)}
+													maxLength={512}
+													placeholder="Bearer …"
+												/>
+												<Button
+													variant="unstyled"
+													className="icon-button header-remove-button"
+													type="button"
+													aria-label={`Remove ${header.name || 'header'}`}
+													onClick={() => setHeaders((current) => current.filter((item) => item.id !== header.id))}
+												>
+													<Trash2 aria-hidden="true" />
+												</Button>
+											</div>
+										))}
+									</div>
+								)}
+							</div>
+							{form.method === 'POST' && (
+								<label className="field advanced-body" htmlFor="monitor-request-body">
+									<span>Request body</span>
+									<textarea
+										id="monitor-request-body"
+										value={form.requestBody ?? ''}
+										onChange={(event) => setForm({ ...form, requestBody: event.target.value || null })}
+										maxLength={8192}
+										rows={6}
+										placeholder={'{"query":"health"}'}
+									/>
+									<small>Sent with POST requests. JSON content type is added unless overridden above.</small>
+								</label>
+							)}
+						</div>
+					</details>
 					<div className="form-actions compact-actions">
 						<Button variant="unstyled" className="secondary-button" type="button" onClick={closeForm}>
 							Cancel

src/client/lib/monitor-status.ts

@@ -1,14 +1,26 @@
 import type { Monitor } from '../api/monitors';
 
-export function monitorState(monitor: Pick<Monitor, 'lastOk' | 'consecutiveFailures' | 'failureThreshold'>) {
+export function monitorState(
+	monitor: Pick<Monitor, 'lastOk' | 'lastDegraded' | 'degradedLatencyMs' | 'consecutiveFailures' | 'failureThreshold'>,
+) {
 	if (monitor.lastOk === false) return { label: 'Down', variant: 'offline' as const, detail: null };
 	if (monitor.consecutiveFailures > 0) {
 		return {
-			label: 'Degrading',
+			label: 'Failing',
 			variant: 'pending' as const,
 			detail: `${monitor.consecutiveFailures} of ${monitor.failureThreshold} failed checks — an incident opens if the next check fails`,
 		};
 	}
+	if (monitor.lastDegraded) {
+		return {
+			label: 'Degraded',
+			variant: 'pending' as const,
+			detail:
+				monitor.degradedLatencyMs === null
+					? 'Response time is above the configured threshold'
+					: `Response time exceeds ${monitor.degradedLatencyMs} ms`,
+		};
+	}
 	if (monitor.lastOk === true) return { label: 'Up', variant: 'online' as const, detail: null };
 	return { label: 'Not checked', variant: 'checking' as const, detail: null };
 }

src/client/pages/StatusPage.tsx

@@ -27,6 +27,7 @@ const OVERALL_COPY: Record<PublicOverallStatus, { title: string; detail: string
 
 const SERVICE_STATUS: Record<PublicServiceStatus, { label: string; className: BadgeVariant }> = {
 	up: { label: 'Operational', className: 'online' },
+	degraded: { label: 'Degraded performance', className: 'pending' },
 	down: { label: 'Down', className: 'offline' },
 	unknown: { label: 'Awaiting data', className: 'checking' },
 	maintenance: { label: 'Under maintenance', className: 'maintenance' },

src/client/styles.css

@@ -15,6 +15,7 @@
 	--shadcn-primary: oklch(0.205 0 0);
 	--primary-deep: #24b47e;
 	--chart-down: #d95c5c;
+	--chart-degraded: #d97706;
 	--ink: #171717;
 	--muted: #707070;
 	--shadcn-muted: oklch(0.97 0 0);
@@ -424,6 +425,119 @@ button {
 	color: #9f2f2f;
 	background: #fff6f6;
 }
+.monitor-advanced {
+	grid-column: 1 / -1;
+	border: 1px solid var(--hairline);
+	border-radius: 7px;
+	background: #fcfcfc;
+}
+.monitor-advanced summary {
+	display: flex;
+	align-items: center;
+	justify-content: space-between;
+	padding: 14px 16px;
+	font-size: 13px;
+	font-weight: 500;
+	cursor: pointer;
+	list-style: none;
+}
+.monitor-advanced summary::-webkit-details-marker {
+	display: none;
+}
+.monitor-advanced summary svg {
+	width: 16px;
+	height: 16px;
+	transition: transform 160ms ease;
+}
+.monitor-advanced[open] summary svg {
+	transform: rotate(180deg);
+}
+.monitor-advanced-grid {
+	display: grid;
+	grid-template-columns: repeat(4, 1fr);
+	gap: 20px 16px;
+	padding: 2px 16px 18px;
+	border-top: 1px solid #ededed;
+}
+.monitor-advanced-grid > * {
+	margin-top: 16px;
+}
+.monitor-advanced-grid small,
+.advanced-section-heading small {
+	font-size: 11px;
+	line-height: 1.45;
+	color: var(--muted);
+}
+.advanced-keyword {
+	grid-column: span 2;
+}
+.advanced-inverted {
+	grid-column: span 1;
+}
+.advanced-latency {
+	grid-column: span 1;
+}
+.advanced-headers,
+.advanced-body {
+	grid-column: 1 / -1;
+}
+.advanced-section-heading {
+	display: flex;
+	align-items: center;
+	justify-content: space-between;
+	gap: 16px;
+}
+.advanced-section-heading > div {
+	display: grid;
+	gap: 3px;
+}
+.advanced-section-heading span {
+	font-size: 13px;
+	font-weight: 500;
+}
+.header-add-button {
+	display: inline-flex;
+	align-items: center;
+	gap: 6px;
+	min-height: 32px;
+	padding: 6px 10px;
+}
+.header-add-button svg,
+.header-remove-button svg {
+	width: 14px;
+	height: 14px;
+}
+.header-rows {
+	display: grid;
+	gap: 8px;
+	margin-top: 10px;
+}
+.header-row {
+	display: grid;
+	grid-template-columns: minmax(130px, 1fr) minmax(180px, 2fr) 36px;
+	gap: 8px;
+}
+.header-remove-button {
+	width: 36px;
+	height: 36px;
+	color: #9f2f2f;
+}
+.advanced-body textarea {
+	width: 100%;
+	resize: vertical;
+	border: 1px solid var(--input);
+	border-radius: 6px;
+	padding: 10px 12px;
+	font:
+		12px/1.5 'IBM Plex Mono',
+		monospace;
+	color: var(--ink);
+	background: #fff;
+}
+.advanced-body textarea:focus-visible {
+	outline: 2px solid rgb(36 180 126 / 0.3);
+	outline-offset: 1px;
+}
 
 .services-panel {
 	margin-top: 24px;
@@ -1191,6 +1305,10 @@ button {
 	margin-left: 5px;
 	background: var(--chart-down);
 }
+.uptime-legend i.legend-degraded {
+	margin-left: 5px;
+	background: var(--chart-degraded);
+}
 .data-table-wrap {
 	overflow-x: auto;
 }
@@ -3380,6 +3498,7 @@ button {
 	--color-chart-2: var(--chart-2);
 	--color-chart-1: var(--chart-1);
 	--color-chart-down: var(--chart-down);
+	--color-chart-degraded: var(--chart-degraded);
 	--color-ring: var(--ring);
 	--color-input: var(--input);
 	--color-border: var(--border);

src/worker/ai/fallback-message.ts

@@ -3,10 +3,12 @@ export const INCIDENT_REASSURANCE =
 	'The problem was detected automatically, our team has been alerted, and we are working to restore normal service as soon as possible.';
 
 export const RECOVERY_UPDATE_BODY = 'The service has recovered and is responding normally again.';
+export const DEGRADED_MESSAGE = 'This service is responding more slowly than usual and some pages may take longer to load.';
 
 /** Plain-language, non-technical description of the impact, used when no AI message is available. */
 export function describeFailure(statusCode: number | null): string {
 	if (statusCode === null) return 'This service is currently unreachable and may not load for visitors.';
+	if (statusCode >= 200 && statusCode <= 299) return 'This service is responding but is not returning the expected content.';
 	if (statusCode >= 500 && statusCode <= 599) return 'This service is having problems and some requests may fail or load incorrectly.';
 	if (statusCode === 429) return 'This service is under heavy load and is temporarily turning away some requests.';
 	if (statusCode >= 400 && statusCode <= 499) return 'This service is not responding correctly and some features may not work right now.';

src/worker/checks/persist-result.ts

@@ -7,6 +7,7 @@ import type { CheckResult, Monitor } from './run-check';
 /** Only opened and resolved transitions are allowed to send alerts. */
 export type CheckTransition = 'opened' | 'pending' | 'cleared' | 'resolved' | null;
 export type AlertTransition = Extract<CheckTransition, 'opened' | 'resolved'>;
+export type LatencyTransition = 'degraded' | 'recovered' | null;
 
 type BatchStatement = Parameters<Database['batch']>[0][number];
 
@@ -15,6 +16,7 @@ export function buildResultStatements(db: Database, monitor: Monitor, result: Ch
 		db.insert(checks).values({
 			monitorId: monitor.id,
 			ok: result.ok,
+			degraded: result.degraded,
 			statusCode: result.statusCode,
 			latencyMs: result.latencyMs,
 			error: result.error,
@@ -36,7 +38,12 @@ export function buildResultStatements(db: Database, monitor: Monitor, result: Ch
 				})
 				.where(eq(monitors.id, monitor.id)),
 		);
-		return { statements, transition: null as CheckTransition, consecutiveFailures: monitor.consecutiveFailures };
+		return {
+			statements,
+			transition: null as CheckTransition,
+			latencyTransition: null as LatencyTransition,
+			consecutiveFailures: monitor.consecutiveFailures,
+		};
 	}
 
 	const threshold = Math.max(1, monitor.failureThreshold);
@@ -44,9 +51,13 @@ export function buildResultStatements(db: Database, monitor: Monitor, result: Ch
 	// This deliberately uses the monitor snapshot. Concurrent manual and scheduled checks may lose one increment,
 	// which delays confirmation by one check but cannot publish a false incident.
 	const nextFailures = result.ok ? 0 : previousFailures + 1;
+	const nextSlow = result.degraded ? monitor.consecutiveSlow + 1 : 0;
 	const wasDown = monitor.lastOk === false;
 	const isDown = !result.ok && nextFailures >= threshold;
 	const confirmed = result.ok ? true : isDown ? false : undefined;
+	const confirmedDegraded = nextSlow >= threshold;
+	const latencyTransition: LatencyTransition =
+		!monitor.lastDegraded && confirmedDegraded ? 'degraded' : monitor.lastDegraded && !confirmedDegraded ? 'recovered' : null;
 
 	statements.push(
 		db
@@ -54,6 +65,8 @@ export function buildResultStatements(db: Database, monitor: Monitor, result: Ch
 			.set({
 				...(confirmed === undefined ? {} : { lastOk: confirmed }),
 				consecutiveFailures: nextFailures,
+				consecutiveSlow: nextSlow,
+				lastDegraded: confirmedDegraded,
 				lastStatusCode: result.statusCode,
 				lastLatencyMs: result.latencyMs,
 				lastError: result.error,
@@ -114,5 +127,5 @@ export function buildResultStatements(db: Database, monitor: Monitor, result: Ch
 	} else if (!wasDown && !result.ok) transition = 'pending';
 	else if (!wasDown && result.ok && previousFailures > 0) transition = 'cleared';
 
-	return { statements, transition, consecutiveFailures: nextFailures };
+	return { statements, transition, latencyTransition, consecutiveFailures: nextFailures };
 }

src/worker/checks/run-check.ts

@@ -1,14 +1,35 @@
 import type { monitors } from '../db/schema';
+import { readBodyLimited } from '../lib/read-body';
 
 export type Monitor = typeof monitors.$inferSelect;
 
 export type CheckResult = {
 	ok: boolean;
+	degraded: boolean;
 	statusCode: number | null;
 	latencyMs: number;
 	error: string | null;
 };
 
+export const MAX_BODY_MATCH_BYTES = 256 * 1024;
+
+function requestHeaders(monitor: Monitor) {
+	let configured: Record<string, string> = {};
+	if (monitor.requestHeaders) {
+		try {
+			configured = JSON.parse(monitor.requestHeaders) as Record<string, string>;
+		} catch {
+			// Stored values are API-validated. Ignore malformed legacy values instead of failing the check.
+		}
+	}
+	const headers = new Headers(configured);
+	if (!headers.has('User-Agent')) headers.set('User-Agent', 'Upwatch/1.0 (+uptime monitor)');
+	if (monitor.method === 'POST' && monitor.requestBody !== null && !headers.has('Content-Type')) {
+		headers.set('Content-Type', 'application/json');
+	}
+	return headers;
+}
+
 export async function runCheck(monitor: Monitor): Promise<CheckResult> {
 	const startedAt = Date.now();
 	try {
@@ -16,19 +37,37 @@ export async function runCheck(monitor: Monitor): Promise<CheckResult> {
 			method: monitor.method,
 			redirect: 'follow',
 			signal: AbortSignal.timeout(monitor.timeoutMs),
-			headers: { 'User-Agent': 'Upwatch/1.0 (+uptime monitor)' },
+			headers: requestHeaders(monitor),
+			body: monitor.method === 'POST' ? monitor.requestBody : undefined,
 		});
-		await response.body?.cancel();
+		const latencyMs = Date.now() - startedAt;
-		const ok = response.status === monitor.expectedStatus;
+		const statusOk = response.status === monitor.expectedStatus;
+		let keywordOk = true;
+		if (monitor.expectKeyword !== null && monitor.method !== 'HEAD') {
+			const body = await readBodyLimited(response, MAX_BODY_MATCH_BYTES, true);
+			const contains =
+				body !== null && new TextDecoder().decode(body).toLocaleLowerCase().includes(monitor.expectKeyword.toLocaleLowerCase());
+			keywordOk = monitor.keywordInverted ? !contains : contains;
+		} else {
+			await response.body?.cancel();
+		}
+		const ok = statusOk && keywordOk;
+		let error: string | null = null;
+		if (!statusOk) error = `Expected HTTP ${monitor.expectedStatus}, received ${response.status}`;
+		else if (!keywordOk) {
+			error = `${monitor.keywordInverted ? 'Response contained' : 'Response did not contain'} "${monitor.expectKeyword}"`.slice(0, 200);
+		}
 		return {
 			ok,
+			degraded: ok && monitor.degradedLatencyMs !== null && latencyMs > monitor.degradedLatencyMs,
 			statusCode: response.status,
-			latencyMs: Date.now() - startedAt,
+			latencyMs,
-			error: ok ? null : `Expected HTTP ${monitor.expectedStatus}, received ${response.status}`,
+			error,
 		};
 	} catch (error) {
 		return {
 			ok: false,
+			degraded: false,
 			statusCode: null,
 			latencyMs: Date.now() - startedAt,
 			error: error instanceof Error ? error.message.slice(0, 200) : 'Request failed',

src/worker/checks/run-due-checks.ts

@@ -74,24 +74,45 @@ export async function runDueChecks(env: Env, ctx?: Pick<ExecutionContext, 'waitU
 	let aiMessagesQueued = 0;
 	const notificationBudget: NotificationBudget = { remaining: MAX_NOTIFICATIONS_PER_RUN };
 	const notifications = persisted.flatMap((item) => {
-		if (item.transition !== 'opened' && item.transition !== 'resolved') return [];
+		const work: Promise<unknown>[] = [];
-		const kind: AlertTransition = item.transition;
+		if (item.transition === 'opened' || item.transition === 'resolved') {
-		const work: Promise<unknown>[] = [
+			const kind: AlertTransition = item.transition;
-			dispatchNotification(
+			work.push(
-				env,
+				dispatchNotification(
-				{
+					env,
-					monitor: { id: item.monitor.id, name: item.monitor.name, url: item.monitor.url },
+					{
-					kind: kind === 'opened' ? 'down' : 'recovered',
+						monitor: { id: item.monitor.id, name: item.monitor.name, url: item.monitor.url },
-					incidentId: null,
+						kind: kind === 'opened' ? 'down' : 'recovered',
-					title: kind === 'opened' ? `${item.monitor.name} is down` : `${item.monitor.name} recovered`,
+						incidentId: null,
-					body: item.result.error,
+						title: kind === 'opened' ? `${item.monitor.name} is down` : `${item.monitor.name} recovered`,
-					statusCode: item.result.statusCode,
+						body: item.result.error,
-					error: item.result.error,
+						statusCode: item.result.statusCode,
-					at: item.checkedAt,
+						error: item.result.error,
-				},
+						at: item.checkedAt,
-				notificationBudget,
+					},
-			),
+					notificationBudget,
-		];
+				),
+			);
+		}
+		if (item.latencyTransition) {
+			const degraded = item.latencyTransition === 'degraded';
+			work.push(
+				dispatchNotification(
+					env,
+					{
+						monitor: { id: item.monitor.id, name: item.monitor.name, url: item.monitor.url },
+						kind: degraded ? 'degraded' : 'recovered_degraded',
+						incidentId: null,
+						title: degraded ? `${item.monitor.name} performance degraded` : `${item.monitor.name} performance recovered`,
+						body: degraded ? `Response time was ${item.result.latencyMs} ms.` : 'Response time returned to normal.',
+						statusCode: item.result.statusCode,
+						error: item.result.error,
+						at: item.checkedAt,
+					},
+					notificationBudget,
+				),
+			);
+		}
 		if (item.transition === 'opened' && item.monitor.alertsEnabled && aiMessagesQueued < MAX_AI_MESSAGES_PER_RUN) {
 			aiMessagesQueued += 1;
 			work.push(generateIncidentMessage(env, { monitor: item.monitor, result: item.result }));

src/worker/db/schema.ts

@@ -46,6 +46,11 @@ export const monitors = sqliteTable(
 		url: text('url').notNull(),
 		method: text('method').notNull().default('GET'),
 		expectedStatus: integer('expected_status').notNull().default(200),
+		expectKeyword: text('expect_keyword'),
+		keywordInverted: integer('keyword_inverted', { mode: 'boolean' }).notNull().default(false),
+		requestHeaders: text('request_headers'),
+		requestBody: text('request_body'),
+		degradedLatencyMs: integer('degraded_latency_ms'),
 		intervalSeconds: integer('interval_seconds').notNull().default(300),
 		timeoutMs: integer('timeout_ms').notNull().default(10_000),
 		enabled: integer('enabled', { mode: 'boolean' }).notNull().default(true),
@@ -56,8 +61,12 @@ export const monitors = sqliteTable(
 		failureThreshold: integer('failure_threshold').notNull().default(2),
 		/** Failures since the last successful check. Maintenance checks do not change this value. */
 		consecutiveFailures: integer('consecutive_failures').notNull().default(0),
+		/** Slow successful checks since latency last recovered. Maintenance checks do not change this value. */
+		consecutiveSlow: integer('consecutive_slow').notNull().default(0),
 		/** Confirmed state, not the raw latest result. */
 		lastOk: integer('last_ok', { mode: 'boolean' }),
+		/** Confirmed degraded state. A down monitor is never degraded. */
+		lastDegraded: integer('last_degraded', { mode: 'boolean' }).notNull().default(false),
 		lastStatusCode: integer('last_status_code'),
 		lastLatencyMs: integer('last_latency_ms'),
 		lastError: text('last_error'),
@@ -112,6 +121,7 @@ export const checks = sqliteTable(
 		error: text('error'),
 		checkedAt: integer('checked_at', { mode: 'timestamp_ms' }).notNull(),
 		maintenance: integer('maintenance', { mode: 'boolean' }).notNull().default(false),
+		degraded: integer('degraded', { mode: 'boolean' }).notNull().default(false),
 	},
 	(table) => [index('checks_monitor_id_checked_at_idx').on(table.monitorId, table.checkedAt)],
 );

src/worker/lib/read-body.ts

@@ -0,0 +1,37 @@
+export async function readBodyLimited(response: Response, maximum: number, truncate: boolean) {
+	if (!response.body) return null;
+	const reader = response.body.getReader();
+	const chunks: Uint8Array[] = [];
+	let total = 0;
+	let exceeded = false;
+
+	try {
+		while (true) {
+			const { done, value } = await reader.read();
+			if (done) break;
+			const remaining = maximum - total;
+			if (value.byteLength > remaining) {
+				if (truncate && remaining > 0) {
+					chunks.push(value.subarray(0, remaining));
+					total += remaining;
+				}
+				exceeded = true;
+				await reader.cancel();
+				break;
+			}
+			chunks.push(value);
+			total += value.byteLength;
+		}
+	} catch {
+		return null;
+	}
+
+	if (exceeded && !truncate) return null;
+	const body = new Uint8Array(total);
+	let offset = 0;
+	for (const chunk of chunks) {
+		body.set(chunk, offset);
+		offset += chunk.byteLength;
+	}
+	return body.buffer;
+}

src/worker/notifications/providers/types.ts

@@ -8,7 +8,7 @@ export type TelegramConfig = { botToken: string; chatId: string };
 export type ChannelConfig = UrlConfig | TelegramConfig;
 
 export type NotificationEvent = {
-	kind: 'down' | 'recovered' | 'manual_opened' | 'manual_update' | 'test';
+	kind: 'down' | 'recovered' | 'degraded' | 'recovered_degraded' | 'manual_opened' | 'manual_update' | 'test';
 	monitor: { id: number; name: string; url: string } | null;
 	incidentId: number | null;
 	title: string;
@@ -49,6 +49,8 @@ export function eventLabel(kind: NotificationEvent['kind']) {
 	return {
 		down: 'Service down',
 		recovered: 'Service recovered',
+		degraded: 'Service degraded',
+		recovered_degraded: 'Performance recovered',
 		manual_opened: 'Incident opened',
 		manual_update: 'Incident update',
 		test: 'Test notification',
@@ -57,6 +59,7 @@ export function eventLabel(kind: NotificationEvent['kind']) {
 
 export function eventColor(kind: NotificationEvent['kind']) {
 	if (kind === 'down' || kind === 'manual_opened') return '#dc2626';
-	if (kind === 'recovered') return '#16a34a';
+	if (kind === 'recovered' || kind === 'recovered_degraded') return '#16a34a';
+	if (kind === 'degraded') return '#d97706';
 	return '#2563eb';
 }

src/worker/routes/monitors.ts

@@ -8,6 +8,7 @@ import { checks, incidentMonitors, incidents, maintenanceWindowMonitors, monitor
 import { requireAuth, type AuthVariables } from '../lib/require-auth';
 import { loadActiveMaintenance } from '../maintenance/windows';
 import { isSafeRemoteUrl } from '../lib/safe-url';
+import { readBodyLimited } from '../lib/read-body';
 import { dispatchNotification } from '../notifications/dispatch';
 
 type MonitorMethod = 'GET' | 'HEAD' | 'POST';
@@ -17,6 +18,11 @@ type ParsedMonitorInput = {
 	url?: string;
 	method?: MonitorMethod;
 	expectedStatus?: number;
+	expectKeyword?: string | null;
+	keywordInverted?: boolean;
+	requestHeaders?: string | null;
+	requestBody?: string | null;
+	degradedLatencyMs?: number | null;
 	intervalSeconds?: number;
 	timeoutMs?: number;
 	retryCount?: number;
@@ -33,6 +39,8 @@ const FAVICON_FETCH_TIMEOUT_MS = 5_000;
 const MAX_FAVICON_BYTES = 1024 * 1024;
 const MAX_HEAD_BYTES = 128 * 1024;
 const MAX_REDIRECTS = 3;
+const HEADER_NAME = /^[!#$%&'*+\-.^_`|~0-9A-Za-z]{1,64}$/;
+const FORBIDDEN_HEADERS = new Set(['host', 'content-length', 'transfer-encoding', 'connection']);
 
 type FaviconResult = {
 	body: ArrayBuffer;
@@ -44,44 +52,6 @@ type EdgeCache = {
 	put(request: RequestInfo | URL, response: Response): Promise<void>;
 };
 
-async function readBodyLimited(response: Response, maximum: number, truncate: boolean) {
-	if (!response.body) return null;
-	const reader = response.body.getReader();
-	const chunks: Uint8Array[] = [];
-	let total = 0;
-	let exceeded = false;
-
-	try {
-		while (true) {
-			const { done, value } = await reader.read();
-			if (done) break;
-			const remaining = maximum - total;
-			if (value.byteLength > remaining) {
-				if (truncate && remaining > 0) {
-					chunks.push(value.subarray(0, remaining));
-					total += remaining;
-				}
-				exceeded = true;
-				await reader.cancel();
-				break;
-			}
-			chunks.push(value);
-			total += value.byteLength;
-		}
-	} catch {
-		return null;
-	}
-
-	if (exceeded && !truncate) return null;
-	const body = new Uint8Array(total);
-	let offset = 0;
-	for (const chunk of chunks) {
-		body.set(chunk, offset);
-		offset += chunk.byteLength;
-	}
-	return body.buffer;
-}
-
 async function fetchRemote(url: URL, maximumBytes: number, truncate = false) {
 	const controller = new AbortController();
 	const timeout = setTimeout(() => controller.abort(), FAVICON_FETCH_TIMEOUT_MS);
@@ -232,7 +202,7 @@ export function parseInteger(
 	return { ok: true, value: value as number };
 }
 
-export function parseMonitorInput(body: unknown, partial = false): ParseResult {
+export function parseMonitorInput(body: unknown, partial = false, storedMethod?: MonitorMethod): ParseResult {
 	if (!isRecord(body)) return { ok: false, message: 'Invalid request body' };
 
 	const value: ParsedMonitorInput = {};
@@ -263,6 +233,55 @@ export function parseMonitorInput(body: unknown, partial = false): ParseResult {
 		value.method = body.method as MonitorMethod;
 	}
 
+	if ('expectKeyword' in body) {
+		if (body.expectKeyword === null) value.expectKeyword = null;
+		else if (typeof body.expectKeyword !== 'string' || body.expectKeyword.trim().length < 1 || body.expectKeyword.trim().length > 200) {
+			return { ok: false, message: 'expectKeyword must be between 1 and 200 characters or null' };
+		} else value.expectKeyword = body.expectKeyword.trim();
+	}
+	if ('keywordInverted' in body) {
+		if (typeof body.keywordInverted !== 'boolean') return { ok: false, message: 'keywordInverted must be a boolean' };
+		value.keywordInverted = body.keywordInverted;
+	}
+	if ('requestHeaders' in body) {
+		if (body.requestHeaders === null) value.requestHeaders = null;
+		else if (!isRecord(body.requestHeaders)) return { ok: false, message: 'requestHeaders must be an object or null' };
+		else {
+			const entries = Object.entries(body.requestHeaders);
+			if (entries.length > 10) return { ok: false, message: 'requestHeaders cannot contain more than 10 headers' };
+			const normalized: Record<string, string> = {};
+			for (const [name, headerValue] of entries) {
+				const lowerName = name.toLowerCase();
+				if (!HEADER_NAME.test(name)) return { ok: false, message: `Invalid header name: ${name}` };
+				if (FORBIDDEN_HEADERS.has(lowerName) || lowerName.startsWith('cf-')) {
+					return { ok: false, message: `Header is not allowed: ${name}` };
+				}
+				if (typeof headerValue !== 'string' || headerValue.length > 512 || /[\r\n]/.test(headerValue)) {
+					return { ok: false, message: `Invalid value for header: ${name}` };
+				}
+				normalized[name] = headerValue;
+			}
+			value.requestHeaders = JSON.stringify(normalized);
+		}
+	}
+	if ('requestBody' in body) {
+		if (body.requestBody === null) value.requestBody = null;
+		else if (typeof body.requestBody !== 'string' || body.requestBody.length > 8_192) {
+			return { ok: false, message: 'requestBody must be no more than 8192 characters or null' };
+		} else value.requestBody = body.requestBody;
+		if (value.requestBody !== null && (value.method ?? storedMethod) !== 'POST') {
+			return { ok: false, message: 'requestBody can only be used with POST monitors' };
+		}
+	}
+	if ('degradedLatencyMs' in body) {
+		if (body.degradedLatencyMs === null) value.degradedLatencyMs = null;
+		else {
+			const parsed = parseInteger(body.degradedLatencyMs, 'degradedLatencyMs', 1, 30_000);
+			if (!parsed.ok) return parsed;
+			value.degradedLatencyMs = parsed.value;
+		}
+	}
+
 	for (const [key, label, minimum, maximum] of [
 		['expectedStatus', 'expectedStatus', 100, 599],
 		['intervalSeconds', 'intervalSeconds', 300, 86_400],
@@ -504,6 +523,11 @@ monitorRoutes.post('/', async (context) => {
 			url: parsed.value.url!,
 			method: parsed.value.method!,
 			expectedStatus: parsed.value.expectedStatus!,
+			expectKeyword: parsed.value.expectKeyword ?? null,
+			keywordInverted: parsed.value.keywordInverted ?? false,
+			requestHeaders: parsed.value.requestHeaders ?? null,
+			requestBody: parsed.value.requestBody ?? null,
+			degradedLatencyMs: parsed.value.degradedLatencyMs ?? null,
 			intervalSeconds: parsed.value.intervalSeconds!,
 			timeoutMs: parsed.value.timeoutMs!,
 			retryCount: parsed.value.retryCount ?? 1,
@@ -521,6 +545,9 @@ monitorRoutes.post('/', async (context) => {
 monitorRoutes.patch('/:id', async (context) => {
 	const id = parseId(context.req.param('id'));
 	if (id === null) return context.json({ message: 'Monitor not found' }, 404);
+	const db = getDb(context.env);
+	const [existing] = await db.select({ method: monitors.method }).from(monitors).where(eq(monitors.id, id)).limit(1);
+	if (!existing) return context.json({ message: 'Monitor not found' }, 404);
 
 	let body: unknown;
 	try {
@@ -529,13 +556,13 @@ monitorRoutes.patch('/:id', async (context) => {
 		return context.json({ message: 'Invalid request body' }, 400);
 	}
 
-	const parsed = parseMonitorInput(body, true);
+	const parsed = parseMonitorInput(body, true, existing.method as MonitorMethod);
 	if (!parsed.ok) return context.json({ message: parsed.message }, 400);
 	if (Object.keys(parsed.value).length === 0) {
 		return context.json({ message: 'Provide at least one field to update' }, 400);
 	}
 
-	const [monitor] = await getDb(context.env)
+	const [monitor] = await db
 		.update(monitors)
 		.set({ ...parsed.value, updatedAt: new Date() })
 		.where(eq(monitors.id, id))
@@ -573,7 +600,13 @@ monitorRoutes.post('/:id/check', async (context) => {
 	const result = await runCheckWithRetries(monitor);
 	const checkedAt = new Date();
 	const activeMaintenance = await loadActiveMaintenance(db, checkedAt);
-	const { statements, transition } = buildResultStatements(db, monitor, result, checkedAt, activeMaintenance.has(monitor.id));
+	const { statements, transition, latencyTransition } = buildResultStatements(
+		db,
+		monitor,
+		result,
+		checkedAt,
+		activeMaintenance.has(monitor.id),
+	);
 	await db.batch(statements as [(typeof statements)[number], ...typeof statements]);
 	if (transition === 'opened' || transition === 'resolved') {
 		await dispatchNotification(context.env, {
@@ -590,9 +623,22 @@ monitorRoutes.post('/:id/check', async (context) => {
 			await generateIncidentMessage(context.env, { monitor, result });
 		}
 	}
+	if (latencyTransition) {
+		const degraded = latencyTransition === 'degraded';
+		await dispatchNotification(context.env, {
+			monitor: { id: monitor.id, name: monitor.name, url: monitor.url },
+			kind: degraded ? 'degraded' : 'recovered_degraded',
+			incidentId: null,
+			title: degraded ? `${monitor.name} performance degraded` : `${monitor.name} performance recovered`,
+			body: degraded ? `Response time was ${result.latencyMs} ms.` : 'Response time returned to normal.',
+			statusCode: result.statusCode,
+			error: result.error,
+			at: checkedAt,
+		});
+	}
 	const [updated] = await db.select().from(monitors).where(eq(monitors.id, monitor.id)).limit(1);
 
-	return context.json({ result, transition, monitor: updated });
+	return context.json({ result, transition, latencyTransition, monitor: updated });
 });
 
 export default monitorRoutes;

src/worker/routes/status.ts

@@ -1,329 +1,333 @@
-import { and, desc, eq, gte, inArray, isNull, lt, sql } from 'drizzle-orm';
+import { and, desc, eq, gte, inArray, isNull, lt, sql } from 'drizzle-orm';
-import { Hono } from 'hono';
+import { Hono } from 'hono';
-import { deterministicIncidentMessage } from '../ai/fallback-message';
+import { DEGRADED_MESSAGE, deterministicIncidentMessage } from '../ai/fallback-message';
-import { getDb } from '../db/client';
+import { getDb } from '../db/client';
-import { checks, incidentMonitors, incidents, incidentUpdates, monitorDailyStats, monitors } from '../db/schema';
+import { checks, incidentMonitors, incidents, incidentUpdates, monitorDailyStats, monitors } from '../db/schema';
-import { loadActiveMaintenance, type ActiveMaintenance } from '../maintenance/windows';
+import { loadActiveMaintenance, type ActiveMaintenance } from '../maintenance/windows';
-import { resolveFavicon } from './monitors';
+import { resolveFavicon } from './monitors';
-
+
-const DAY_MS = 24 * 60 * 60 * 1000;
+const DAY_MS = 24 * 60 * 60 * 1000;
-const FAVICON_CACHE_SECONDS = 86_400;
+const FAVICON_CACHE_SECONDS = 86_400;
-type ServiceStatus = 'up' | 'down' | 'unknown' | 'maintenance';
+type ServiceStatus = 'up' | 'degraded' | 'down' | 'unknown' | 'maintenance';
-type OverallStatus = 'operational' | 'degraded' | 'down';
+type OverallStatus = 'operational' | 'degraded' | 'down';
-type DailyAggregate = { monitorId: number; day: Date; totalChecks: number; upChecks: number };
+type DailyAggregate = { monitorId: number; day: Date; totalChecks: number; upChecks: number };
-type EdgeCache = {
+type EdgeCache = {
-	match(request: RequestInfo | URL): Promise<Response | undefined>;
+	match(request: RequestInfo | URL): Promise<Response | undefined>;
-	put(request: RequestInfo | URL, response: Response): Promise<void>;
+	put(request: RequestInfo | URL, response: Response): Promise<void>;
-};
+};
-
+
-type PublicUpdate = { body: string; status: string; createdAt: Date };
+type PublicUpdate = { body: string; status: string; createdAt: Date };
-type PublicIncident = {
+type PublicIncident = {
-	id: number;
+	id: number;
-	title: string | null;
+	title: string | null;
-	status: string;
+	status: string;
-	impact: string;
+	impact: string;
-	source: string;
+	source: string;
-	startedAt: Date;
+	startedAt: Date;
-	resolvedAt: Date | null;
+	resolvedAt: Date | null;
-	durationMs: number | null;
+	durationMs: number | null;
-	startStatusCode: number | null;
+	startStatusCode: number | null;
-};
+};
-
+
-function parseId(rawId: string) {
+function parseId(rawId: string) {
-	const id = Number(rawId);
+	const id = Number(rawId);
-	return Number.isSafeInteger(id) && id > 0 ? id : null;
+	return Number.isSafeInteger(id) && id > 0 ? id : null;
-}
+}
-function parseLimit(raw: string | undefined, fallback: number, maximum: number) {
+function parseLimit(raw: string | undefined, fallback: number, maximum: number) {
-	if (!raw) return fallback;
+	if (!raw) return fallback;
-	const value = Number(raw);
+	const value = Number(raw);
-	return Number.isSafeInteger(value) && value > 0 ? Math.min(value, maximum) : fallback;
+	return Number.isSafeInteger(value) && value > 0 ? Math.min(value, maximum) : fallback;
-}
+}
-function roundUptime(upChecks: number, totalChecks: number) {
+function roundUptime(upChecks: number, totalChecks: number) {
-	return totalChecks > 0 ? Math.round((upChecks / totalChecks) * 1_000) / 10 : null;
+	return totalChecks > 0 ? Math.round((upChecks / totalChecks) * 1_000) / 10 : null;
-}
+}
-function serviceStatus(lastOk: boolean | null): ServiceStatus {
+function serviceStatus(lastOk: boolean | null, lastDegraded: boolean): ServiceStatus {
-	if (lastOk === true) return 'up';
+	if (lastOk === true) return lastDegraded ? 'degraded' : 'up';
-	if (lastOk === false) return 'down';
+	if (lastOk === false) return 'down';
-	return 'unknown';
+	return 'unknown';
-}
+}
-function overallStatus(statuses: ServiceStatus[], manualImpacts: string[]): OverallStatus {
+function overallStatus(statuses: ServiceStatus[], manualImpacts: string[]): OverallStatus {
-	let checked = 0;
+	let checked = 0;
-	let down = 0;
+	let down = 0;
-	for (const status of statuses) {
+	for (const status of statuses) {
-		if (status === 'unknown' || status === 'maintenance') continue;
+		if (status === 'unknown' || status === 'maintenance') continue;
-		checked += 1;
+		checked += 1;
-		if (status === 'down') down += 1;
+		if (status === 'down') down += 1;
-	}
+	}
-	let severity = down === 0 ? 0 : down === checked ? 2 : 1;
+	let severity = down === 0 ? 0 : down === checked ? 2 : 1;
-	for (const impact of manualImpacts)
+	if (statuses.includes('degraded')) severity = Math.max(severity, 1);
-		severity = Math.max(severity, impact === 'critical' ? 2 : impact === 'minor' || impact === 'major' ? 1 : 0);
+	for (const impact of manualImpacts)
-	return severity === 2 ? 'down' : severity === 1 ? 'degraded' : 'operational';
+		severity = Math.max(severity, impact === 'critical' ? 2 : impact === 'minor' || impact === 'major' ? 1 : 0);
-}
+	return severity === 2 ? 'down' : severity === 1 ? 'degraded' : 'operational';
-function publicIncidentTitle(incident: Pick<PublicIncident, 'title' | 'source'>) {
+}
-	return incident.title ?? (incident.source === 'auto' ? 'Service disruption' : 'Incident update');
+function publicIncidentTitle(incident: Pick<PublicIncident, 'title' | 'source'>) {
-}
+	return incident.title ?? (incident.source === 'auto' ? 'Service disruption' : 'Incident update');
-
+}
-async function loadServices(db: ReturnType<typeof getDb>, incidentIds: number[]) {
+
-	if (incidentIds.length === 0) return new Map<number, Array<{ id: number; name: string }>>();
+async function loadServices(db: ReturnType<typeof getDb>, incidentIds: number[]) {
-	const rows = await db
+	if (incidentIds.length === 0) return new Map<number, Array<{ id: number; name: string }>>();
-		.select({ incidentId: incidentMonitors.incidentId, id: monitors.id, name: monitors.name })
+	const rows = await db
-		.from(incidentMonitors)
+		.select({ incidentId: incidentMonitors.incidentId, id: monitors.id, name: monitors.name })
-		.innerJoin(monitors, eq(monitors.id, incidentMonitors.monitorId))
+		.from(incidentMonitors)
-		.where(inArray(incidentMonitors.incidentId, incidentIds));
+		.innerJoin(monitors, eq(monitors.id, incidentMonitors.monitorId))
-	const grouped = new Map<number, Array<{ id: number; name: string }>>();
+		.where(inArray(incidentMonitors.incidentId, incidentIds));
-	for (const row of rows) {
+	const grouped = new Map<number, Array<{ id: number; name: string }>>();
-		const services = grouped.get(row.incidentId);
+	for (const row of rows) {
-		if (services) services.push({ id: row.id, name: row.name });
+		const services = grouped.get(row.incidentId);
-		else grouped.set(row.incidentId, [{ id: row.id, name: row.name }]);
+		if (services) services.push({ id: row.id, name: row.name });
-	}
+		else grouped.set(row.incidentId, [{ id: row.id, name: row.name }]);
-	return grouped;
+	}
-}
+	return grouped;
-
+}
-async function loadLatestUpdates(db: ReturnType<typeof getDb>, incidentIds: number[]) {
+
-	if (incidentIds.length === 0) return new Map<number, PublicUpdate>();
+async function loadLatestUpdates(db: ReturnType<typeof getDb>, incidentIds: number[]) {
-	const rows = await db
+	if (incidentIds.length === 0) return new Map<number, PublicUpdate>();
-		.select({
+	const rows = await db
-			incidentId: incidentUpdates.incidentId,
+		.select({
-			body: incidentUpdates.body,
+			incidentId: incidentUpdates.incidentId,
-			status: incidentUpdates.status,
+			body: incidentUpdates.body,
-			createdAt: incidentUpdates.createdAt,
+			status: incidentUpdates.status,
-		})
+			createdAt: incidentUpdates.createdAt,
-		.from(incidentUpdates)
+		})
-		.where(inArray(incidentUpdates.incidentId, incidentIds))
+		.from(incidentUpdates)
-		.orderBy(desc(incidentUpdates.createdAt), desc(incidentUpdates.id));
+		.where(inArray(incidentUpdates.incidentId, incidentIds))
-	const latest = new Map<number, PublicUpdate>();
+		.orderBy(desc(incidentUpdates.createdAt), desc(incidentUpdates.id));
-	for (const row of rows) if (!latest.has(row.incidentId)) latest.set(row.incidentId, row);
+	const latest = new Map<number, PublicUpdate>();
-	return latest;
+	for (const row of rows) if (!latest.has(row.incidentId)) latest.set(row.incidentId, row);
-}
+	return latest;
-
+}
-const incidentSelection = {
+
-	id: incidents.id,
+const incidentSelection = {
-	title: incidents.title,
+	id: incidents.id,
-	status: incidents.status,
+	title: incidents.title,
-	impact: incidents.impact,
+	status: incidents.status,
-	source: incidents.source,
+	impact: incidents.impact,
-	startedAt: incidents.startedAt,
+	source: incidents.source,
-	resolvedAt: incidents.resolvedAt,
+	startedAt: incidents.startedAt,
-	durationMs: incidents.durationMs,
+	resolvedAt: incidents.resolvedAt,
-	startStatusCode: incidents.startStatusCode,
+	durationMs: incidents.durationMs,
-};
+	startStatusCode: incidents.startStatusCode,
-
+};
-const statusRoutes = new Hono<{ Bindings: Env }>();
+
-
+const statusRoutes = new Hono<{ Bindings: Env }>();
-statusRoutes.get('/', async (context) => {
+
-	const db = getDb(context.env);
+statusRoutes.get('/', async (context) => {
-	const monitorRows = await db
+	const db = getDb(context.env);
-		.select({
+	const monitorRows = await db
-			id: monitors.id,
+		.select({
-			name: monitors.name,
+			id: monitors.id,
-			lastOk: monitors.lastOk,
+			name: monitors.name,
-			lastStatusCode: monitors.lastStatusCode,
+			lastOk: monitors.lastOk,
-			lastCheckedAt: monitors.lastCheckedAt,
+			lastDegraded: monitors.lastDegraded,
-		})
+			lastStatusCode: monitors.lastStatusCode,
-		.from(monitors)
+			lastCheckedAt: monitors.lastCheckedAt,
-		.where(eq(monitors.enabled, true))
+		})
-		.orderBy(monitors.createdAt);
+		.from(monitors)
-	const activeIncidentRows = await db
+		.where(eq(monitors.enabled, true))
-		.select(incidentSelection)
+		.orderBy(monitors.createdAt);
-		.from(incidents)
+	const activeIncidentRows = await db
-		.where(isNull(incidents.resolvedAt))
+		.select(incidentSelection)
-		.orderBy(desc(incidents.startedAt));
+		.from(incidents)
-	const incidentIds = activeIncidentRows.map((incident) => incident.id);
+		.where(isNull(incidents.resolvedAt))
-	const [incidentServices, latestUpdates] = await Promise.all([loadServices(db, incidentIds), loadLatestUpdates(db, incidentIds)]);
+		.orderBy(desc(incidents.startedAt));
-
+	const incidentIds = activeIncidentRows.map((incident) => incident.id);
-	const now = new Date();
+	const [incidentServices, latestUpdates] = await Promise.all([loadServices(db, incidentIds), loadLatestUpdates(db, incidentIds)]);
-	const today = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate());
+
-	const cutoff = today - 89 * DAY_MS;
+	const now = new Date();
-	const monitorIds = monitorRows.map((monitor) => monitor.id);
+	const today = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate());
-	let historicalRows: DailyAggregate[] = [];
+	const cutoff = today - 89 * DAY_MS;
-	let todayRows: DailyAggregate[] = [];
+	const monitorIds = monitorRows.map((monitor) => monitor.id);
-	let activeMaintenance = new Map<number, ActiveMaintenance>();
+	let historicalRows: DailyAggregate[] = [];
-	if (monitorIds.length > 0) {
+	let todayRows: DailyAggregate[] = [];
-		[historicalRows, todayRows, activeMaintenance] = await Promise.all([
+	let activeMaintenance = new Map<number, ActiveMaintenance>();
-			db
+	if (monitorIds.length > 0) {
-				.select({
+		[historicalRows, todayRows, activeMaintenance] = await Promise.all([
-					monitorId: monitorDailyStats.monitorId,
+			db
-					day: monitorDailyStats.day,
+				.select({
-					totalChecks: monitorDailyStats.totalChecks,
+					monitorId: monitorDailyStats.monitorId,
-					upChecks: monitorDailyStats.upChecks,
+					day: monitorDailyStats.day,
-				})
+					totalChecks: monitorDailyStats.totalChecks,
-				.from(monitorDailyStats)
+					upChecks: monitorDailyStats.upChecks,
-				.where(
+				})
-					and(
+				.from(monitorDailyStats)
-						inArray(monitorDailyStats.monitorId, monitorIds),
+				.where(
-						gte(monitorDailyStats.day, new Date(cutoff)),
+					and(
-						lt(monitorDailyStats.day, new Date(today)),
+						inArray(monitorDailyStats.monitorId, monitorIds),
-					),
+						gte(monitorDailyStats.day, new Date(cutoff)),
-				)
+						lt(monitorDailyStats.day, new Date(today)),
-				.orderBy(monitorDailyStats.day),
+					),
-			db
+				)
-				.select({
+				.orderBy(monitorDailyStats.day),
-					monitorId: checks.monitorId,
+			db
-					day: sql<Date>`cast(${today} as integer)`,
+				.select({
-					totalChecks: sql<number>`count(*)`,
+					monitorId: checks.monitorId,
-					upChecks: sql<number>`coalesce(sum(case when ${checks.ok} = 1 then 1 else 0 end), 0)`,
+					day: sql<Date>`cast(${today} as integer)`,
-				})
+					totalChecks: sql<number>`count(*)`,
-				.from(checks)
+					upChecks: sql<number>`coalesce(sum(case when ${checks.ok} = 1 then 1 else 0 end), 0)`,
-				.where(and(inArray(checks.monitorId, monitorIds), eq(checks.maintenance, false), gte(checks.checkedAt, new Date(today))))
+				})
-				.groupBy(checks.monitorId),
+				.from(checks)
-			loadActiveMaintenance(db, now),
+				.where(and(inArray(checks.monitorId, monitorIds), eq(checks.maintenance, false), gte(checks.checkedAt, new Date(today))))
-		]);
+				.groupBy(checks.monitorId),
-	}
+			loadActiveMaintenance(db, now),
-	const bucketsByMonitor = new Map<number, DailyAggregate[]>();
+		]);
-	for (const row of [...historicalRows, ...todayRows]) {
+	}
-		const buckets = bucketsByMonitor.get(row.monitorId);
+	const bucketsByMonitor = new Map<number, DailyAggregate[]>();
-		if (buckets) buckets.push(row);
+	for (const row of [...historicalRows, ...todayRows]) {
-		else bucketsByMonitor.set(row.monitorId, [row]);
+		const buckets = bucketsByMonitor.get(row.monitorId);
-	}
+		if (buckets) buckets.push(row);
-	const activeIncidentByMonitor = new Map<number, PublicIncident>();
+		else bucketsByMonitor.set(row.monitorId, [row]);
-	for (const incident of activeIncidentRows) {
+	}
-		for (const service of incidentServices.get(incident.id) ?? [])
+	const activeIncidentByMonitor = new Map<number, PublicIncident>();
-			if (!activeIncidentByMonitor.has(service.id)) activeIncidentByMonitor.set(service.id, incident);
+	for (const incident of activeIncidentRows) {
-	}
+		for (const service of incidentServices.get(incident.id) ?? [])
-	const services = monitorRows.map((monitor) => {
+			if (!activeIncidentByMonitor.has(service.id)) activeIncidentByMonitor.set(service.id, incident);
-		const buckets = bucketsByMonitor.get(monitor.id) ?? [];
+	}
-		let totalChecks = 0;
+	const services = monitorRows.map((monitor) => {
-		let upChecks = 0;
+		const buckets = bucketsByMonitor.get(monitor.id) ?? [];
-		const history = buckets.map((bucket) => {
+		let totalChecks = 0;
-			totalChecks += bucket.totalChecks;
+		let upChecks = 0;
-			upChecks += bucket.upChecks;
+		const history = buckets.map((bucket) => {
-			return {
+			totalChecks += bucket.totalChecks;
-				day: bucket.day instanceof Date ? bucket.day.getTime() : Number(bucket.day),
+			upChecks += bucket.upChecks;
-				uptimePct: roundUptime(bucket.upChecks, bucket.totalChecks),
+			return {
-			};
+				day: bucket.day instanceof Date ? bucket.day.getTime() : Number(bucket.day),
-		});
+				uptimePct: roundUptime(bucket.upChecks, bucket.totalChecks),
-		const incident = activeIncidentByMonitor.get(monitor.id);
+			};
-		const maintenance = activeMaintenance.get(monitor.id);
+		});
-		return {
+		const incident = activeIncidentByMonitor.get(monitor.id);
-			id: monitor.id,
+		const maintenance = activeMaintenance.get(monitor.id);
-			name: monitor.name,
+		return {
-			status: maintenance ? ('maintenance' as const) : serviceStatus(monitor.lastOk),
+			id: monitor.id,
-			message:
+			name: monitor.name,
-				!maintenance && monitor.lastOk === false
+			status: maintenance ? ('maintenance' as const) : serviceStatus(monitor.lastOk, monitor.lastDegraded),
-					? incident
+			message:
-						? (latestUpdates.get(incident.id)?.body ?? deterministicIncidentMessage(incident.startStatusCode))
+				!maintenance && monitor.lastOk === false
-						: deterministicIncidentMessage(monitor.lastStatusCode)
+					? incident
-					: null,
+						? (latestUpdates.get(incident.id)?.body ?? deterministicIncidentMessage(incident.startStatusCode))
-			maintenance: maintenance ? { name: maintenance.name, endsAt: maintenance.endsAt.toISOString() } : null,
+						: deterministicIncidentMessage(monitor.lastStatusCode)
-			lastCheckedAt: monitor.lastCheckedAt?.toISOString() ?? null,
+					: !maintenance && monitor.lastOk === true && monitor.lastDegraded
-			uptime90d: roundUptime(upChecks, totalChecks),
+						? DEGRADED_MESSAGE
-			history,
+						: null,
-		};
+			maintenance: maintenance ? { name: maintenance.name, endsAt: maintenance.endsAt.toISOString() } : null,
-	});
+			lastCheckedAt: monitor.lastCheckedAt?.toISOString() ?? null,
-	const activeIncidents = activeIncidentRows.map((incident) => ({
+			uptime90d: roundUptime(upChecks, totalChecks),
-		id: incident.id,
+			history,
-		title: publicIncidentTitle(incident),
+		};
-		status: incident.status,
+	});
-		impact: incident.impact,
+	const activeIncidents = activeIncidentRows.map((incident) => ({
-		source: incident.source,
+		id: incident.id,
-		startedAt: incident.startedAt.toISOString(),
+		title: publicIncidentTitle(incident),
-		latestUpdate: latestUpdates.get(incident.id) ?? null,
+		status: incident.status,
-		services: incidentServices.get(incident.id) ?? [],
+		impact: incident.impact,
-	}));
+		source: incident.source,
-	return context.json({
+		startedAt: incident.startedAt.toISOString(),
-		overall: overallStatus(
+		latestUpdate: latestUpdates.get(incident.id) ?? null,
-			services.map((service) => service.status),
+		services: incidentServices.get(incident.id) ?? [],
-			activeIncidentRows.filter((incident) => incident.source === 'manual').map((incident) => incident.impact),
+	}));
-		),
+	return context.json({
-		updatedAt: Date.now(),
+		overall: overallStatus(
-		services,
+			services.map((service) => service.status),
-		activeIncidents,
+			activeIncidentRows.filter((incident) => incident.source === 'manual').map((incident) => incident.impact),
-	});
+		),
-});
+		updatedAt: Date.now(),
-
+		services,
-statusRoutes.get('/incidents', async (context) => {
+		activeIncidents,
-	const db = getDb(context.env);
+	});
-	const limit = parseLimit(context.req.query('limit'), 20, 20);
+});
-	const rows = await db
+
-		.select(incidentSelection)
+statusRoutes.get('/incidents', async (context) => {
-		.from(incidents)
+	const db = getDb(context.env);
-		.where(and(eq(incidents.status, 'resolved'), gte(incidents.resolvedAt, new Date(Date.now() - 30 * DAY_MS))))
+	const limit = parseLimit(context.req.query('limit'), 20, 20);
-		.orderBy(desc(incidents.resolvedAt))
+	const rows = await db
-		.limit(limit);
+		.select(incidentSelection)
-	const services = await loadServices(
+		.from(incidents)
-		db,
+		.where(and(eq(incidents.status, 'resolved'), gte(incidents.resolvedAt, new Date(Date.now() - 30 * DAY_MS))))
-		rows.map((row) => row.id),
+		.orderBy(desc(incidents.resolvedAt))
-	);
+		.limit(limit);
-	return context.json({
+	const services = await loadServices(
-		incidents: rows.map((incident) => ({
+		db,
-			id: incident.id,
+		rows.map((row) => row.id),
-			title: publicIncidentTitle(incident),
+	);
-			status: incident.status,
+	return context.json({
-			impact: incident.impact,
+		incidents: rows.map((incident) => ({
-			source: incident.source,
+			id: incident.id,
-			startedAt: incident.startedAt.toISOString(),
+			title: publicIncidentTitle(incident),
-			resolvedAt: incident.resolvedAt?.toISOString() ?? null,
+			status: incident.status,
-			durationMs: incident.durationMs,
+			impact: incident.impact,
-			services: services.get(incident.id) ?? [],
+			source: incident.source,
-		})),
+			startedAt: incident.startedAt.toISOString(),
-	});
+			resolvedAt: incident.resolvedAt?.toISOString() ?? null,
-});
+			durationMs: incident.durationMs,
-
+			services: services.get(incident.id) ?? [],
-statusRoutes.get('/incidents/:id', async (context) => {
+		})),
-	const id = parseId(context.req.param('id'));
+	});
-	if (id === null) return context.json({ message: 'Incident not found' }, 404);
+});
-	const db = getDb(context.env);
+
-	const [incident] = await db.select(incidentSelection).from(incidents).where(eq(incidents.id, id)).limit(1);
+statusRoutes.get('/incidents/:id', async (context) => {
-	if (!incident) return context.json({ message: 'Incident not found' }, 404);
+	const id = parseId(context.req.param('id'));
-	const [services, updates] = await Promise.all([
+	if (id === null) return context.json({ message: 'Incident not found' }, 404);
-		loadServices(db, [id]),
+	const db = getDb(context.env);
-		db
+	const [incident] = await db.select(incidentSelection).from(incidents).where(eq(incidents.id, id)).limit(1);
-			.select({ status: incidentUpdates.status, body: incidentUpdates.body, createdAt: incidentUpdates.createdAt })
+	if (!incident) return context.json({ message: 'Incident not found' }, 404);
-			.from(incidentUpdates)
+	const [services, updates] = await Promise.all([
-			.where(eq(incidentUpdates.incidentId, id))
+		loadServices(db, [id]),
-			.orderBy(incidentUpdates.createdAt, incidentUpdates.id),
+		db
-	]);
+			.select({ status: incidentUpdates.status, body: incidentUpdates.body, createdAt: incidentUpdates.createdAt })
-	const timeline =
+			.from(incidentUpdates)
-		updates.length > 0
+			.where(eq(incidentUpdates.incidentId, id))
-			? updates
+			.orderBy(incidentUpdates.createdAt, incidentUpdates.id),
-			: [{ status: incident.status, body: deterministicIncidentMessage(incident.startStatusCode), createdAt: incident.startedAt }];
+	]);
-	return context.json({
+	const timeline =
-		incident: {
+		updates.length > 0
-			id: incident.id,
+			? updates
-			title: publicIncidentTitle(incident),
+			: [{ status: incident.status, body: deterministicIncidentMessage(incident.startStatusCode), createdAt: incident.startedAt }];
-			status: incident.status,
+	return context.json({
-			impact: incident.impact,
+		incident: {
-			source: incident.source,
+			id: incident.id,
-			startedAt: incident.startedAt.toISOString(),
+			title: publicIncidentTitle(incident),
-			resolvedAt: incident.resolvedAt?.toISOString() ?? null,
+			status: incident.status,
-			durationMs: incident.durationMs,
+			impact: incident.impact,
-			services: services.get(id) ?? [],
+			source: incident.source,
-			updates: timeline,
+			startedAt: incident.startedAt.toISOString(),
-		},
+			resolvedAt: incident.resolvedAt?.toISOString() ?? null,
-	});
+			durationMs: incident.durationMs,
-});
+			services: services.get(id) ?? [],
-
+			updates: timeline,
-statusRoutes.get('/:id/favicon', async (context) => {
+		},
-	const id = parseId(context.req.param('id'));
+	});
-	if (id === null) return context.json({ message: 'Service not found' }, 404);
+});
-	const [monitor] = await getDb(context.env)
+
-		.select({ url: monitors.url })
+statusRoutes.get('/:id/favicon', async (context) => {
-		.from(monitors)
+	const id = parseId(context.req.param('id'));
-		.where(and(eq(monitors.id, id), eq(monitors.enabled, true)))
+	if (id === null) return context.json({ message: 'Service not found' }, 404);
-		.limit(1);
+	const [monitor] = await getDb(context.env)
-	if (!monitor) return context.json({ message: 'Service not found' }, 404);
+		.select({ url: monitors.url })
-	const cacheKey = new Request(`${new URL(context.req.url).origin}/api/status/${id}/favicon`);
+		.from(monitors)
-	let cache: EdgeCache | null = null;
+		.where(and(eq(monitors.id, id), eq(monitors.enabled, true)))
-	try {
+		.limit(1);
-		const defaultCache = (caches as CacheStorage & { readonly default: EdgeCache }).default;
+	if (!monitor) return context.json({ message: 'Service not found' }, 404);
-		const cached = await defaultCache.match(cacheKey);
+	const cacheKey = new Request(`${new URL(context.req.url).origin}/api/status/${id}/favicon`);
-		if (cached) return cached;
+	let cache: EdgeCache | null = null;
-		cache = defaultCache;
+	try {
-	} catch {
+		const defaultCache = (caches as CacheStorage & { readonly default: EdgeCache }).default;
-		// Cache API availability is best-effort.
+		const cached = await defaultCache.match(cacheKey);
-	}
+		if (cached) return cached;
-	const favicon = await resolveFavicon(monitor.url);
+		cache = defaultCache;
-	if (!favicon) return context.json({ message: 'No favicon' }, 404);
+	} catch {
-	const response = new Response(favicon.body, {
+		// Cache API availability is best-effort.
-		headers: {
+	}
-			'Cache-Control': `public, max-age=${FAVICON_CACHE_SECONDS}`,
+	const favicon = await resolveFavicon(monitor.url);
-			'Content-Length': String(favicon.body.byteLength),
+	if (!favicon) return context.json({ message: 'No favicon' }, 404);
-			'Content-Type': favicon.contentType,
+	const response = new Response(favicon.body, {
-			'X-Content-Type-Options': 'nosniff',
+		headers: {
-		},
+			'Cache-Control': `public, max-age=${FAVICON_CACHE_SECONDS}`,
-	});
+			'Content-Length': String(favicon.body.byteLength),
-	if (cache) context.executionCtx.waitUntil(cache.put(cacheKey, response.clone()).catch(() => undefined));
+			'Content-Type': favicon.contentType,
-	return response;
+			'X-Content-Type-Options': 'nosniff',
-});
+		},
-
+	});
-export default statusRoutes;
+	if (cache) context.executionCtx.waitUntil(cache.put(cacheKey, response.clone()).catch(() => undefined));
+	return response;
+});
+
+export default statusRoutes;

test/checks.spec.ts

@@ -27,12 +27,19 @@ async function insertMonitor(overrides: Record<string, unknown> = {}) {
 		url: 'https://example.com/health',
 		method: 'GET',
 		expected_status: 200,
+		expect_keyword: null,
+		keyword_inverted: 0,
+		request_headers: null,
+		request_body: null,
+		degraded_latency_ms: null,
 		interval_seconds: 300,
 		timeout_ms: 10_000,
 		retry_count: 0,
 		failure_threshold: 2,
 		consecutive_failures: 0,
+		consecutive_slow: 0,
 		enabled: 1,
+		last_degraded: 0,
 		last_checked_at: null,
 		created_at: now,
 		updated_at: now,
@@ -41,9 +48,10 @@ async function insertMonitor(overrides: Record<string, unknown> = {}) {
 	const result = await env.DB.prepare(
 		`
 		INSERT INTO monitors
-		(name, url, method, expected_status, interval_seconds, timeout_ms, retry_count, failure_threshold,
+		(name, url, method, expected_status, expect_keyword, keyword_inverted, request_headers, request_body,
-		 consecutive_failures, enabled, alerts_enabled, last_ok, last_checked_at, created_at, updated_at)
+		 degraded_latency_ms, interval_seconds, timeout_ms, retry_count, failure_threshold, consecutive_failures,
-		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?)
+		 consecutive_slow, enabled, alerts_enabled, last_ok, last_degraded, last_checked_at, created_at, updated_at)
+		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?)
 	`,
 	)
 		.bind(
@@ -51,13 +59,20 @@ async function insertMonitor(overrides: Record<string, unknown> = {}) {
 			values.url,
 			values.method,
 			values.expected_status,
+			values.expect_keyword,
+			values.keyword_inverted,
+			values.request_headers,
+			values.request_body,
+			values.degraded_latency_ms,
 			values.interval_seconds,
 			values.timeout_ms,
 			values.retry_count,
 			values.failure_threshold,
 			values.consecutive_failures,
+			values.consecutive_slow,
 			values.enabled,
 			overrides.last_ok ?? null,
+			values.last_degraded,
 			values.last_checked_at,
 			values.created_at,
 			values.updated_at,
@@ -100,6 +115,98 @@ describe('scheduled monitor checks', () => {
 		expect(checkCount?.count).toBe(1);
 	});
 
+	it('passes a case-insensitive keyword assertion and sends configured request data', async () => {
+		await insertMonitor({
+			method: 'POST',
+			expect_keyword: 'healthy',
+			request_headers: JSON.stringify({ Authorization: 'Bearer test' }),
+			request_body: '{"probe":true}',
+		});
+		const fetchMock = vi.fn(async () => new Response('{"status":"HEALTHY"}', { status: 200 }));
+		vi.stubGlobal('fetch', fetchMock);
+
+		const summary = await runDueChecks(env);
+		expect(summary.up).toBe(1);
+		const init = fetchMock.mock.calls[0][1] as RequestInit;
+		expect(init.method).toBe('POST');
+		expect(init.body).toBe('{"probe":true}');
+		expect(new Headers(init.headers).get('Authorization')).toBe('Bearer test');
+		expect(new Headers(init.headers).get('Content-Type')).toBe('application/json');
+	});
+
+	it('fails and opens an incident when the expected keyword is absent', async () => {
+		const id = await insertMonitor({ last_ok: 1, expect_keyword: 'healthy' });
+		vi.stubGlobal(
+			'fetch',
+			vi.fn(async () => new Response('{"status":"error"}', { status: 200 })),
+		);
+
+		await runDueChecks(env);
+		await env.DB.prepare('UPDATE monitors SET last_checked_at = NULL WHERE id = ?').bind(id).run();
+		const summary = await runDueChecks(env);
+
+		expect(summary.opened).toBe(1);
+		const check = await env.DB.prepare('SELECT ok, error FROM checks WHERE monitor_id = ? ORDER BY id DESC LIMIT 1')
+			.bind(id)
+			.first<{ ok: number; error: string }>();
+		expect(check).toEqual({ ok: 0, error: 'Response did not contain "healthy"' });
+	});
+
+	it('supports inverted keyword assertions and safely truncates large bodies', async () => {
+		await insertMonitor({ expect_keyword: 'maintenance', keyword_inverted: 1 });
+		vi.stubGlobal(
+			'fetch',
+			vi.fn(async () => new Response(`${'a'.repeat(256 * 1024)}maintenance`, { status: 200 })),
+		);
+
+		const summary = await runDueChecks(env);
+		expect(summary.up).toBe(1);
+		expect((await env.DB.prepare('SELECT ok FROM checks').first<{ ok: number }>())?.ok).toBe(1);
+	});
+
+	it('confirms degraded latency, notifies, and recovers on a fast check', async () => {
+		const id = await insertMonitor({ last_ok: 1, degraded_latency_ms: 1, failure_threshold: 2 });
+		const now = Date.now();
+		await env.DB.prepare(
+			`INSERT INTO notification_channels (name, type, config, enabled, notify_manual, created_at, updated_at)
+			 VALUES ('Webhook', 'webhook', '{"url":"https://hooks.example.test/events"}', 1, 1, ?, ?)`,
+		)
+			.bind(now, now)
+			.run();
+		const events: string[] = [];
+		let slow = true;
+		vi.stubGlobal(
+			'fetch',
+			vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
+				if (new URL(input.toString()).hostname === 'hooks.example.test') {
+					events.push((JSON.parse(String(init?.body)) as { event: string }).event);
+					return new Response(null, { status: 204 });
+				}
+				if (slow) await new Promise((resolve) => setTimeout(resolve, 5));
+				return new Response(null, { status: 200 });
+			}),
+		);
+
+		await runDueChecks(env);
+		await env.DB.prepare('UPDATE monitors SET last_checked_at = NULL WHERE id = ?').bind(id).run();
+		await runDueChecks(env);
+		expect(events).toEqual(['degraded']);
+		expect(
+			await env.DB.prepare('SELECT last_degraded, consecutive_slow FROM monitors WHERE id = ?')
+				.bind(id)
+				.first<{ last_degraded: number; consecutive_slow: number }>(),
+		).toEqual({ last_degraded: 1, consecutive_slow: 2 });
+		expect((await env.DB.prepare('SELECT degraded FROM checks ORDER BY id DESC LIMIT 1').first<{ degraded: number }>())?.degraded).toBe(1);
+
+		slow = false;
+		await env.DB.prepare('UPDATE monitors SET last_checked_at = NULL, degraded_latency_ms = 30000 WHERE id = ?').bind(id).run();
+		await runDueChecks(env);
+		expect(events).toEqual(['degraded', 'recovered_degraded']);
+		expect(
+			(await env.DB.prepare('SELECT last_degraded FROM monitors WHERE id = ?').bind(id).first<{ last_degraded: number }>())?.last_degraded,
+		).toBe(0);
+	});
+
 	it('records a first mismatched status as pending without opening an incident', async () => {
 		const id = await insertMonitor({ last_ok: 1 });
 		const fetchMock = vi.fn(async () => new Response(null, { status: 500 }));

test/maintenance.spec.ts

@@ -83,21 +83,33 @@ describe('maintenance windows', () => {
 
 	it('records failed probes as maintenance without incidents or monitor state changes', async () => {
 		const id = await insertMonitor(1);
-		await env.DB.prepare('UPDATE monitors SET retry_count = 3, consecutive_failures = 1 WHERE id = ?').bind(id).run();
+		await env.DB.prepare(
+			'UPDATE monitors SET retry_count = 3, consecutive_failures = 1, consecutive_slow = 2, last_degraded = 1 WHERE id = ?',
+		)
+			.bind(id)
+			.run();
 		await insertActiveWindow(id);
 		const fetchMock = vi.fn(async () => new Response(null, { status: 503 }));
 		vi.stubGlobal('fetch', fetchMock);
 
 		await runDueChecks(env);
 		const check = await env.DB.prepare('SELECT maintenance, ok FROM checks WHERE monitor_id = ?').bind(id).first();
-		const monitor = await env.DB.prepare('SELECT last_ok, consecutive_failures, last_status_code FROM monitors WHERE id = ?')
+		const monitor = await env.DB.prepare(
+			'SELECT last_ok, consecutive_failures, consecutive_slow, last_degraded, last_status_code FROM monitors WHERE id = ?',
+		)
 			.bind(id)
 			.first();
 		const incident = await env.DB.prepare('SELECT COUNT(*) AS count FROM incident_monitors WHERE monitor_id = ?')
 			.bind(id)
 			.first<{ count: number }>();
 		expect(check).toMatchObject({ maintenance: 1, ok: 0 });
-		expect(monitor).toMatchObject({ last_ok: 1, consecutive_failures: 1, last_status_code: 503 });
+		expect(monitor).toMatchObject({
+			last_ok: 1,
+			consecutive_failures: 1,
+			consecutive_slow: 2,
+			last_degraded: 1,
+			last_status_code: 503,
+		});
 		expect(incident?.count).toBe(0);
 		expect(fetchMock).toHaveBeenCalledTimes(1);
 	});

test/monitors.spec.ts

@@ -180,6 +180,15 @@ describe('monitor API', () => {
 		[{ retryCount: 4 }, 'retryCount must be an integer between 0 and 3'],
 		[{ failureThreshold: 0 }, 'failureThreshold must be an integer between 1 and 10'],
 		[{ failureThreshold: 11 }, 'failureThreshold must be an integer between 1 and 10'],
+		[{ requestHeaders: { Authorization: 'Bearer ok\r\nX-Evil: true' } }, 'Invalid value for header: Authorization'],
+		[{ requestHeaders: { Host: 'example.test' } }, 'Header is not allowed: Host'],
+		[
+			{ requestHeaders: Object.fromEntries(Array.from({ length: 11 }, (_, index) => [`X-${index}`, 'value'])) },
+			'requestHeaders cannot contain more than 10 headers',
+		],
+		[{ requestBody: '{}' }, 'requestBody can only be used with POST monitors'],
+		[{ degradedLatencyMs: 0 }, 'degradedLatencyMs must be an integer between 1 and 30000'],
+		[{ degradedLatencyMs: 30_001 }, 'degradedLatencyMs must be an integer between 1 and 30000'],
 	] as const)('rejects invalid monitor input %o', async (overrides, message) => {
 		const response = await createMonitor(await authenticatedCookie(), overrides);
 		expect(response.status).toBe(400);
@@ -209,6 +218,32 @@ describe('monitor API', () => {
 		});
 	});
 
+	it('stores advanced settings and allows nullable fields to be cleared', async () => {
+		const cookie = await authenticatedCookie();
+		const created = await (
+			await createMonitor(cookie, {
+				method: 'POST',
+				expectKeyword: 'healthy',
+				keywordInverted: true,
+				requestHeaders: { Authorization: 'Bearer test' },
+				requestBody: '{"probe":true}',
+				degradedLatencyMs: 1500,
+			})
+		).json<{ monitor: { id: number; requestHeaders: string } }>();
+		expect(JSON.parse(created.monitor.requestHeaders)).toEqual({ Authorization: 'Bearer test' });
+
+		const response = await apiFetch(`/api/monitors/${created.monitor.id}`, 'PATCH', cookie, {
+			expectKeyword: null,
+			requestHeaders: null,
+			requestBody: null,
+			degradedLatencyMs: null,
+		});
+		const body = await response.json<{
+			monitor: { expectKeyword: null; requestHeaders: null; requestBody: null; degradedLatencyMs: null };
+		}>();
+		expect(body.monitor).toMatchObject({ expectKeyword: null, requestHeaders: null, requestBody: null, degradedLatencyMs: null });
+	});
+
 	it('does not allow clients to set the internal failure counter', async () => {
 		const cookie = await authenticatedCookie();
 		const created = await (await createMonitor(cookie)).json<{ monitor: { id: number } }>();

test/status.spec.ts

@@ -1,6 +1,7 @@
 import { applyD1Migrations, type D1Migration } from 'cloudflare:test';
 import { env, exports as worker } from 'cloudflare:workers';
 import { beforeAll, beforeEach, describe, expect, it } from 'vitest';
+import { DEGRADED_MESSAGE } from '../src/worker/ai/fallback-message';
 
 const DAY_MS = 24 * 60 * 60 * 1000;
 
@@ -10,7 +11,7 @@ type PublicStatusResponse = {
 	services: Array<{
 		id: number;
 		name: string;
-		status: 'up' | 'down' | 'unknown' | 'maintenance';
+		status: 'up' | 'degraded' | 'down' | 'unknown' | 'maintenance';
 		message: string | null;
 		maintenance: { name: string; endsAt: string } | null;
 		lastCheckedAt: string | null;
@@ -46,15 +47,16 @@ async function insertMonitor(input: {
 	lastError?: string | null;
 	consecutiveFailures?: number;
 	failureThreshold?: number;
+	lastDegraded?: boolean;
 }) {
 	const now = Date.now();
 	const result = await env.DB.prepare(
 		`
 		INSERT INTO monitors (
 			name, url, method, expected_status, interval_seconds, timeout_ms,
-			enabled, alerts_enabled, failure_threshold, consecutive_failures, last_ok, last_status_code, last_latency_ms,
+			enabled, alerts_enabled, failure_threshold, consecutive_failures, last_ok, last_degraded, last_status_code, last_latency_ms,
 			last_error, last_checked_at, created_at, updated_at
-		) VALUES (?, ?, 'GET', 200, 300, 10000, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+		) VALUES (?, ?, 'GET', 200, 300, 10000, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
 	`,
 	)
 		.bind(
@@ -64,6 +66,7 @@ async function insertMonitor(input: {
 			input.failureThreshold ?? 2,
 			input.consecutiveFailures ?? 0,
 			input.lastOk === null || input.lastOk === undefined ? null : input.lastOk ? 1 : 0,
+			input.lastDegraded ? 1 : 0,
 			input.lastStatusCode ?? null,
 			input.lastLatencyMs ?? null,
 			input.lastError ?? null,
@@ -139,6 +142,14 @@ describe('public status API', () => {
 		expect(body.services[0]).toMatchObject({ status: 'up', message: null });
 	});
 
+	it('reports a confirmed degraded monitor and public latency message', async () => {
+		await insertMonitor({ name: 'Slow API', lastOk: true, lastDegraded: true, lastLatencyMs: 2200 });
+
+		const body = await (await statusFetch()).json<PublicStatusResponse>();
+		expect(body.overall).toBe('degraded');
+		expect(body.services[0]).toMatchObject({ status: 'degraded', message: DEGRADED_MESSAGE });
+	});
+
 	it('excludes disabled monitors and reports degraded health for a partial outage', async () => {
 		await insertMonitor({ name: 'Healthy', lastOk: true });
 		await insertMonitor({ name: 'Unavailable', lastOk: false });