fix: improve detect error case

migrations/0009_shocking_butterfly.sql

@@ -0,0 +1,8 @@
+ALTER TABLE `checks` ADD `degraded` integer DEFAULT false NOT NULL;--> statement-breakpoint
+ALTER TABLE `monitors` ADD `expect_keyword` text;--> statement-breakpoint
+ALTER TABLE `monitors` ADD `keyword_inverted` integer DEFAULT false NOT NULL;--> statement-breakpoint
+ALTER TABLE `monitors` ADD `request_headers` text;--> statement-breakpoint
+ALTER TABLE `monitors` ADD `request_body` text;--> statement-breakpoint
+ALTER TABLE `monitors` ADD `degraded_latency_ms` integer;--> statement-breakpoint
+ALTER TABLE `monitors` ADD `consecutive_slow` integer DEFAULT 0 NOT NULL;--> statement-breakpoint
+ALTER TABLE `monitors` ADD `last_degraded` integer DEFAULT false NOT NULL;

migrations/meta/_journal.json

@@ -64,6 +64,13 @@
       "when": 1788059207578,
       "tag": "0008_aromatic_chat",
       "breakpoints": true
+    },
+    {
+      "idx": 9,
+      "version": "6",
+      "when": 1788062139636,
+      "tag": "0009_shocking_butterfly",
+      "breakpoints": true
     }
   ]
 }

src/client/api/monitors.ts

@@ -8,6 +8,11 @@ export type Monitor = {
 	url: string;
 	method: MonitorMethod;
 	expectedStatus: number;
+	expectKeyword: string | null;
+	keywordInverted: boolean;
+	requestHeaders: string | null;
+	requestBody: string | null;
+	degradedLatencyMs: number | null;
 	intervalSeconds: number;
 	timeoutMs: number;
 	enabled: boolean;
@@ -15,7 +20,9 @@ export type Monitor = {
 	retryCount: number;
 	failureThreshold: number;
 	consecutiveFailures: number;
+	consecutiveSlow: number;
 	lastOk: boolean | null;
+	lastDegraded: boolean;
 	lastStatusCode: number | null;
 	lastLatencyMs: number | null;
 	lastError: string | null;
@@ -29,6 +36,11 @@ export type MonitorInput = {
 	url: string;
 	method: MonitorMethod;
 	expectedStatus: number;
+	expectKeyword?: string | null;
+	keywordInverted?: boolean;
+	requestHeaders?: Record<string, string> | null;
+	requestBody?: string | null;
+	degradedLatencyMs?: number | null;
 	intervalSeconds: number;
 	timeoutMs: number;
 	retryCount?: number;
@@ -39,6 +51,7 @@ export type MonitorInput = {
 
 export type CheckResult = {
 	ok: boolean;
+	degraded: boolean;
 	statusCode: number | null;
 	latencyMs: number;
 	error: string | null;
@@ -49,6 +62,7 @@ export type Check = {
 	id: number;
 	monitorId: number;
 	ok: boolean;
+	degraded: boolean;
 	statusCode: number | null;
 	latencyMs: number;
 	error: string | null;
@@ -57,6 +71,7 @@ export type Check = {
 };
 
 export type CheckTransition = 'opened' | 'pending' | 'cleared' | 'resolved' | null;
+export type LatencyTransition = 'degraded' | 'recovered' | null;
 
 export type Incident = {
 	id: number;
@@ -122,5 +137,7 @@ export function deleteMonitor(id: number) {
 }
 
 export function runMonitorCheck(id: number) {
-	return postJson<{ result: CheckResult; transition: CheckTransition; monitor: Monitor }>(`/api/monitors/${id}/check`);
+	return postJson<{ result: CheckResult; transition: CheckTransition; latencyTransition: LatencyTransition; monitor: Monitor }>(
+		`/api/monitors/${id}/check`,
+	);
 }

src/client/api/status.ts

@@ -1,6 +1,6 @@
 import { getJson } from './http';
 
-export type PublicServiceStatus = 'up' | 'down' | 'unknown' | 'maintenance';
+export type PublicServiceStatus = 'up' | 'degraded' | 'down' | 'unknown' | 'maintenance';
 export type PublicOverallStatus = 'operational' | 'degraded' | 'down';
 
 export type PublicIncidentUpdate = { status: string; body: string; createdAt: string };

src/client/components/charts/UptimeBar.tsx

@@ -12,7 +12,7 @@ type UptimeDatum = {
 	id: string;
 	successfulChecks: number;
 	totalChecks: number;
-	status: 'up' | 'down';
+	status: 'up' | 'degraded' | 'down';
 	fill: string;
 };
 
@@ -20,6 +20,7 @@ const MAX_VISIBLE_SEGMENTS = 32;
 
 const uptimeConfig = {
 	up: { label: 'Up', color: 'var(--primary)' },
+	degraded: { label: 'Degraded', color: 'var(--chart-degraded)' },
 	down: { label: 'Down', color: 'var(--chart-down)' },
 } satisfies ChartConfig;
 
@@ -58,6 +59,8 @@ function groupChecks(checks: Check[]): UptimeDatum[] {
 		const last = bucket[bucket.length - 1];
 		const successfulChecks = bucket.filter((check) => check.ok).length;
 		const ok = successfulChecks === bucket.length;
+		const degraded = ok && bucket.some((check) => check.degraded);
+		const status = !ok ? 'down' : degraded ? 'degraded' : 'up';
 
 		return {
 			startTime: first.checkedAt,
@@ -67,8 +70,8 @@ function groupChecks(checks: Check[]): UptimeDatum[] {
 			id: `${first.id}-${last.id}`,
 			successfulChecks,
 			totalChecks: bucket.length,
-			status: ok ? 'up' : 'down',
+			status,
-			fill: ok ? 'var(--color-up)' : 'var(--color-down)',
+			fill: `var(--color-${status})`,
 		};
 	});
 }
@@ -114,7 +117,7 @@ export function UptimeBar({ checks }: { checks: Check[] }) {
 							onAnimationEnd={() => setHasAnimated(true)}
 						>
 							{data.map((point) => (
-								<Cell key={point.id} fill={point.ok ? 'var(--color-up)' : 'var(--color-down)'} />
+								<Cell key={point.id} fill={point.fill} />
 							))}
 						</Bar>
 					</BarChart>
@@ -123,7 +126,7 @@ export function UptimeBar({ checks }: { checks: Check[] }) {
 			<div className="uptime-legend">
 				<span>Oldest</span>
 				<span>
-					<i className="legend-up" /> Up <i className="legend-down" /> Down
+					<i className="legend-up" /> Up <i className="legend-degraded" /> Degraded <i className="legend-down" /> Down
 				</span>
 				<span>Latest</span>
 			</div>

src/client/components/dashboard/MonitorFormDialog.tsx

@@ -1,4 +1,5 @@
 import { type FormEvent, useState } from 'react';
+import { ChevronDown, Plus, Trash2 } from 'lucide-react';
 import { Button } from '@/components/ui/button';
 import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from '@/components/ui/dialog';
 import { Input } from '@/components/ui/input';
@@ -16,7 +17,13 @@ export const DEFAULT_MONITOR_INPUT: MonitorInput = {
 	timeoutMs: 10_000,
 	retryCount: 1,
 	failureThreshold: 2,
+	expectKeyword: null,
+	keywordInverted: false,
+	requestHeaders: null,
+	requestBody: null,
+	degradedLatencyMs: null,
 	enabled: true,
+	alertsEnabled: true,
 };
 
 export const INTERVAL_OPTIONS = [
@@ -32,6 +39,17 @@ type MonitorFormDialogProps = {
 	onClose: () => void;
 };
 
+type HeaderRow = { id: number; name: string; value: string };
+
+function headerRows(requestHeaders: string | null): HeaderRow[] {
+	if (!requestHeaders) return [];
+	try {
+		return Object.entries(JSON.parse(requestHeaders) as Record<string, string>).map(([name, value], index) => ({ id: index, name, value }));
+	} catch {
+		return [];
+	}
+}
+
 function errorMessage(error: unknown, fallback: string) {
 	return error instanceof Error ? error.message : fallback;
 }
@@ -47,6 +65,11 @@ function monitorInput(monitor: Monitor | null): MonitorInput {
 		timeoutMs: monitor.timeoutMs,
 		retryCount: monitor.retryCount,
 		failureThreshold: monitor.failureThreshold,
+		expectKeyword: monitor.expectKeyword,
+		keywordInverted: monitor.keywordInverted,
+		requestHeaders: monitor.requestHeaders ? (JSON.parse(monitor.requestHeaders) as Record<string, string>) : null,
+		requestBody: monitor.requestBody,
+		degradedLatencyMs: monitor.degradedLatencyMs,
 		enabled: monitor.enabled,
 		alertsEnabled: monitor.alertsEnabled,
 	};
@@ -56,6 +79,8 @@ export function MonitorFormDialog({ editing, onClose }: MonitorFormDialogProps)
 	const createMutation = useCreateMonitorMutation();
 	const updateMutation = useUpdateMonitorMutation();
 	const [form, setForm] = useState<MonitorInput>(() => monitorInput(editing));
+	const [headers, setHeaders] = useState<HeaderRow[]>(() => headerRows(editing?.requestHeaders ?? null));
+	const [nextHeaderId, setNextHeaderId] = useState(() => headers.length);
 	const formMutation = editing ? updateMutation : createMutation;
 
 	function closeForm() {
@@ -65,13 +90,31 @@ export function MonitorFormDialog({ editing, onClose }: MonitorFormDialogProps)
 
 	function handleSubmit(event: FormEvent<HTMLFormElement>) {
 		event.preventDefault();
+		const requestHeaders = Object.fromEntries(
+			headers.filter((header) => header.name.trim()).map((header) => [header.name.trim(), header.value]),
+		);
+		const input: MonitorInput = {
+			...form,
+			expectKeyword: form.expectKeyword?.trim() || null,
+			requestHeaders: Object.keys(requestHeaders).length > 0 ? requestHeaders : null,
+			requestBody: form.method === 'POST' ? form.requestBody || null : null,
+		};
 		if (editing) {
-			updateMutation.mutate({ id: editing.id, input: form }, { onSuccess: onClose });
+			updateMutation.mutate({ id: editing.id, input }, { onSuccess: onClose });
 		} else {
-			createMutation.mutate(form, { onSuccess: onClose });
+			createMutation.mutate(input, { onSuccess: onClose });
 		}
 	}
 
+	function addHeader() {
+		setHeaders((current) => [...current, { id: nextHeaderId, name: '', value: '' }]);
+		setNextHeaderId((current) => current + 1);
+	}
+
+	function updateHeader(id: number, field: 'name' | 'value', value: string) {
+		setHeaders((current) => current.map((header) => (header.id === id ? { ...header, [field]: value } : header)));
+	}
+
 	return (
 		<Dialog
 			open
@@ -217,6 +260,109 @@ export function MonitorFormDialog({ editing, onClose }: MonitorFormDialogProps)
 						/>
 						<label htmlFor="monitor-alerts-enabled">Enable incident alerts</label>
 					</div>
+					<details className="monitor-advanced">
+						<summary>
+							<span>Advanced request and response checks</span>
+							<ChevronDown aria-hidden="true" />
+						</summary>
+						<div className="monitor-advanced-grid">
+							<label className="field advanced-keyword" htmlFor="monitor-keyword">
+								<span>Expected response keyword</span>
+								<Input
+									id="monitor-keyword"
+									value={form.expectKeyword ?? ''}
+									onChange={(event) => setForm({ ...form, expectKeyword: event.target.value || null })}
+									maxLength={200}
+									placeholder="healthy"
+								/>
+								<small>Case-insensitive match within the first 256 KB of the response.</small>
+							</label>
+							<div className="toggle-field advanced-inverted">
+								<Switch
+									id="monitor-keyword-inverted"
+									checked={form.keywordInverted ?? false}
+									onCheckedChange={(keywordInverted) => setForm({ ...form, keywordInverted })}
+									disabled={!form.expectKeyword}
+								/>
+								<label htmlFor="monitor-keyword-inverted">Fail when present</label>
+							</div>
+							<label className="field advanced-latency" htmlFor="monitor-degraded-latency">
+								<span>Degraded above (ms)</span>
+								<Input
+									id="monitor-degraded-latency"
+									type="number"
+									min="1"
+									max="30000"
+									value={form.degradedLatencyMs ?? ''}
+									onChange={(event) => setForm({ ...form, degradedLatencyMs: event.target.value ? event.target.valueAsNumber : null })}
+									placeholder="1500"
+								/>
+								<small>Publishes degraded performance after the configured confirmation count.</small>
+							</label>
+							<div className="advanced-headers">
+								<div className="advanced-section-heading">
+									<div>
+										<span>Request headers</span>
+										<small>Up to 10 headers. Restricted transport headers are blocked.</small>
+									</div>
+									<Button
+										variant="unstyled"
+										className="secondary-button header-add-button"
+										type="button"
+										onClick={addHeader}
+										disabled={headers.length >= 10}
+									>
+										<Plus aria-hidden="true" /> Add header
+									</Button>
+								</div>
+								{headers.length > 0 && (
+									<div className="header-rows">
+										{headers.map((header) => (
+											<div className="header-row" key={header.id}>
+												<Input
+													aria-label="Header name"
+													value={header.name}
+													onChange={(event) => updateHeader(header.id, 'name', event.target.value)}
+													maxLength={64}
+													placeholder="Authorization"
+												/>
+												<Input
+													aria-label="Header value"
+													value={header.value}
+													onChange={(event) => updateHeader(header.id, 'value', event.target.value)}
+													maxLength={512}
+													placeholder="Bearer …"
+												/>
+												<Button
+													variant="unstyled"
+													className="icon-button header-remove-button"
+													type="button"
+													aria-label={`Remove ${header.name || 'header'}`}
+													onClick={() => setHeaders((current) => current.filter((item) => item.id !== header.id))}
+												>
+													<Trash2 aria-hidden="true" />
+												</Button>
+											</div>
+										))}
+									</div>
+								)}
+							</div>
+							{form.method === 'POST' && (
+								<label className="field advanced-body" htmlFor="monitor-request-body">
+									<span>Request body</span>
+									<textarea
+										id="monitor-request-body"
+										value={form.requestBody ?? ''}
+										onChange={(event) => setForm({ ...form, requestBody: event.target.value || null })}
+										maxLength={8192}
+										rows={6}
+										placeholder={'{"query":"health"}'}
+									/>
+									<small>Sent with POST requests. JSON content type is added unless overridden above.</small>
+								</label>
+							)}
+						</div>
+					</details>
 					<div className="form-actions compact-actions">
 						<Button variant="unstyled" className="secondary-button" type="button" onClick={closeForm}>
 							Cancel

src/client/lib/monitor-status.ts

@@ -1,14 +1,26 @@
 import type { Monitor } from '../api/monitors';
 
-export function monitorState(monitor: Pick<Monitor, 'lastOk' | 'consecutiveFailures' | 'failureThreshold'>) {
+export function monitorState(
+	monitor: Pick<Monitor, 'lastOk' | 'lastDegraded' | 'degradedLatencyMs' | 'consecutiveFailures' | 'failureThreshold'>,
+) {
 	if (monitor.lastOk === false) return { label: 'Down', variant: 'offline' as const, detail: null };
 	if (monitor.consecutiveFailures > 0) {
 		return {
-			label: 'Degrading',
+			label: 'Failing',
 			variant: 'pending' as const,
 			detail: `${monitor.consecutiveFailures} of ${monitor.failureThreshold} failed checks — an incident opens if the next check fails`,
 		};
 	}
+	if (monitor.lastDegraded) {
+		return {
+			label: 'Degraded',
+			variant: 'pending' as const,
+			detail:
+				monitor.degradedLatencyMs === null
+					? 'Response time is above the configured threshold'
+					: `Response time exceeds ${monitor.degradedLatencyMs} ms`,
+		};
+	}
 	if (monitor.lastOk === true) return { label: 'Up', variant: 'online' as const, detail: null };
 	return { label: 'Not checked', variant: 'checking' as const, detail: null };
 }

src/client/pages/StatusPage.tsx

@@ -27,6 +27,7 @@ const OVERALL_COPY: Record<PublicOverallStatus, { title: string; detail: string
 
 const SERVICE_STATUS: Record<PublicServiceStatus, { label: string; className: BadgeVariant }> = {
 	up: { label: 'Operational', className: 'online' },
+	degraded: { label: 'Degraded performance', className: 'pending' },
 	down: { label: 'Down', className: 'offline' },
 	unknown: { label: 'Awaiting data', className: 'checking' },
 	maintenance: { label: 'Under maintenance', className: 'maintenance' },

src/client/styles.css

@@ -15,6 +15,7 @@
 	--shadcn-primary: oklch(0.205 0 0);
 	--primary-deep: #24b47e;
 	--chart-down: #d95c5c;
+	--chart-degraded: #d97706;
 	--ink: #171717;
 	--muted: #707070;
 	--shadcn-muted: oklch(0.97 0 0);
@@ -424,6 +425,119 @@ button {
 	color: #9f2f2f;
 	background: #fff6f6;
 }
+.monitor-advanced {
+	grid-column: 1 / -1;
+	border: 1px solid var(--hairline);
+	border-radius: 7px;
+	background: #fcfcfc;
+}
+.monitor-advanced summary {
+	display: flex;
+	align-items: center;
+	justify-content: space-between;
+	padding: 14px 16px;
+	font-size: 13px;
+	font-weight: 500;
+	cursor: pointer;
+	list-style: none;
+}
+.monitor-advanced summary::-webkit-details-marker {
+	display: none;
+}
+.monitor-advanced summary svg {
+	width: 16px;
+	height: 16px;
+	transition: transform 160ms ease;
+}
+.monitor-advanced[open] summary svg {
+	transform: rotate(180deg);
+}
+.monitor-advanced-grid {
+	display: grid;
+	grid-template-columns: repeat(4, 1fr);
+	gap: 20px 16px;
+	padding: 2px 16px 18px;
+	border-top: 1px solid #ededed;
+}
+.monitor-advanced-grid > * {
+	margin-top: 16px;
+}
+.monitor-advanced-grid small,
+.advanced-section-heading small {
+	font-size: 11px;
+	line-height: 1.45;
+	color: var(--muted);
+}
+.advanced-keyword {
+	grid-column: span 2;
+}
+.advanced-inverted {
+	grid-column: span 1;
+}
+.advanced-latency {
+	grid-column: span 1;
+}
+.advanced-headers,
+.advanced-body {
+	grid-column: 1 / -1;
+}
+.advanced-section-heading {
+	display: flex;
+	align-items: center;
+	justify-content: space-between;
+	gap: 16px;
+}
+.advanced-section-heading > div {
+	display: grid;
+	gap: 3px;
+}
+.advanced-section-heading span {
+	font-size: 13px;
+	font-weight: 500;
+}
+.header-add-button {
+	display: inline-flex;
+	align-items: center;
+	gap: 6px;
+	min-height: 32px;
+	padding: 6px 10px;
+}
+.header-add-button svg,
+.header-remove-button svg {
+	width: 14px;
+	height: 14px;
+}
+.header-rows {
+	display: grid;
+	gap: 8px;
+	margin-top: 10px;
+}
+.header-row {
+	display: grid;
+	grid-template-columns: minmax(130px, 1fr) minmax(180px, 2fr) 36px;
+	gap: 8px;
+}
+.header-remove-button {
+	width: 36px;
+	height: 36px;
+	color: #9f2f2f;
+}
+.advanced-body textarea {
+	width: 100%;
+	resize: vertical;
+	border: 1px solid var(--input);
+	border-radius: 6px;
+	padding: 10px 12px;
+	font:
+		12px/1.5 'IBM Plex Mono',
+		monospace;
+	color: var(--ink);
+	background: #fff;
+}
+.advanced-body textarea:focus-visible {
+	outline: 2px solid rgb(36 180 126 / 0.3);
+	outline-offset: 1px;
+}
 
 .services-panel {
 	margin-top: 24px;
@@ -1191,6 +1305,10 @@ button {
 	margin-left: 5px;
 	background: var(--chart-down);
 }
+.uptime-legend i.legend-degraded {
+	margin-left: 5px;
+	background: var(--chart-degraded);
+}
 .data-table-wrap {
 	overflow-x: auto;
 }
@@ -3380,6 +3498,7 @@ button {
 	--color-chart-2: var(--chart-2);
 	--color-chart-1: var(--chart-1);
 	--color-chart-down: var(--chart-down);
+	--color-chart-degraded: var(--chart-degraded);
 	--color-ring: var(--ring);
 	--color-input: var(--input);
 	--color-border: var(--border);

src/worker/ai/fallback-message.ts

@@ -3,10 +3,12 @@ export const INCIDENT_REASSURANCE =
 	'The problem was detected automatically, our team has been alerted, and we are working to restore normal service as soon as possible.';
 
 export const RECOVERY_UPDATE_BODY = 'The service has recovered and is responding normally again.';
+export const DEGRADED_MESSAGE = 'This service is responding more slowly than usual and some pages may take longer to load.';
 
 /** Plain-language, non-technical description of the impact, used when no AI message is available. */
 export function describeFailure(statusCode: number | null): string {
 	if (statusCode === null) return 'This service is currently unreachable and may not load for visitors.';
+	if (statusCode >= 200 && statusCode <= 299) return 'This service is responding but is not returning the expected content.';
 	if (statusCode >= 500 && statusCode <= 599) return 'This service is having problems and some requests may fail or load incorrectly.';
 	if (statusCode === 429) return 'This service is under heavy load and is temporarily turning away some requests.';
 	if (statusCode >= 400 && statusCode <= 499) return 'This service is not responding correctly and some features may not work right now.';

src/worker/checks/persist-result.ts

@@ -7,6 +7,7 @@ import type { CheckResult, Monitor } from './run-check';
 /** Only opened and resolved transitions are allowed to send alerts. */
 export type CheckTransition = 'opened' | 'pending' | 'cleared' | 'resolved' | null;
 export type AlertTransition = Extract<CheckTransition, 'opened' | 'resolved'>;
+export type LatencyTransition = 'degraded' | 'recovered' | null;
 
 type BatchStatement = Parameters<Database['batch']>[0][number];
 
@@ -15,6 +16,7 @@ export function buildResultStatements(db: Database, monitor: Monitor, result: Ch
 		db.insert(checks).values({
 			monitorId: monitor.id,
 			ok: result.ok,
+			degraded: result.degraded,
 			statusCode: result.statusCode,
 			latencyMs: result.latencyMs,
 			error: result.error,
@@ -36,7 +38,12 @@ export function buildResultStatements(db: Database, monitor: Monitor, result: Ch
 				})
 				.where(eq(monitors.id, monitor.id)),
 		);
-		return { statements, transition: null as CheckTransition, consecutiveFailures: monitor.consecutiveFailures };
+		return {
+			statements,
+			transition: null as CheckTransition,
+			latencyTransition: null as LatencyTransition,
+			consecutiveFailures: monitor.consecutiveFailures,
+		};
 	}
 
 	const threshold = Math.max(1, monitor.failureThreshold);
@@ -44,9 +51,13 @@ export function buildResultStatements(db: Database, monitor: Monitor, result: Ch
 	// This deliberately uses the monitor snapshot. Concurrent manual and scheduled checks may lose one increment,
 	// which delays confirmation by one check but cannot publish a false incident.
 	const nextFailures = result.ok ? 0 : previousFailures + 1;
+	const nextSlow = result.degraded ? monitor.consecutiveSlow + 1 : 0;
 	const wasDown = monitor.lastOk === false;
 	const isDown = !result.ok && nextFailures >= threshold;
 	const confirmed = result.ok ? true : isDown ? false : undefined;
+	const confirmedDegraded = nextSlow >= threshold;
+	const latencyTransition: LatencyTransition =
+		!monitor.lastDegraded && confirmedDegraded ? 'degraded' : monitor.lastDegraded && !confirmedDegraded ? 'recovered' : null;
 
 	statements.push(
 		db
@@ -54,6 +65,8 @@ export function buildResultStatements(db: Database, monitor: Monitor, result: Ch
 			.set({
 				...(confirmed === undefined ? {} : { lastOk: confirmed }),
 				consecutiveFailures: nextFailures,
+				consecutiveSlow: nextSlow,
+				lastDegraded: confirmedDegraded,
 				lastStatusCode: result.statusCode,
 				lastLatencyMs: result.latencyMs,
 				lastError: result.error,
@@ -114,5 +127,5 @@ export function buildResultStatements(db: Database, monitor: Monitor, result: Ch
 	} else if (!wasDown && !result.ok) transition = 'pending';
 	else if (!wasDown && result.ok && previousFailures > 0) transition = 'cleared';
 
-	return { statements, transition, consecutiveFailures: nextFailures };
+	return { statements, transition, latencyTransition, consecutiveFailures: nextFailures };
 }

src/worker/checks/run-check.ts

@@ -1,14 +1,35 @@
 import type { monitors } from '../db/schema';
+import { readBodyLimited } from '../lib/read-body';
 
 export type Monitor = typeof monitors.$inferSelect;
 
 export type CheckResult = {
 	ok: boolean;
+	degraded: boolean;
 	statusCode: number | null;
 	latencyMs: number;
 	error: string | null;
 };
 
+export const MAX_BODY_MATCH_BYTES = 256 * 1024;
+
+function requestHeaders(monitor: Monitor) {
+	let configured: Record<string, string> = {};
+	if (monitor.requestHeaders) {
+		try {
+			configured = JSON.parse(monitor.requestHeaders) as Record<string, string>;
+		} catch {
+			// Stored values are API-validated. Ignore malformed legacy values instead of failing the check.
+		}
+	}
+	const headers = new Headers(configured);
+	if (!headers.has('User-Agent')) headers.set('User-Agent', 'Upwatch/1.0 (+uptime monitor)');
+	if (monitor.method === 'POST' && monitor.requestBody !== null && !headers.has('Content-Type')) {
+		headers.set('Content-Type', 'application/json');
+	}
+	return headers;
+}
+
 export async function runCheck(monitor: Monitor): Promise<CheckResult> {
 	const startedAt = Date.now();
 	try {
@@ -16,19 +37,37 @@ export async function runCheck(monitor: Monitor): Promise<CheckResult> {
 			method: monitor.method,
 			redirect: 'follow',
 			signal: AbortSignal.timeout(monitor.timeoutMs),
-			headers: { 'User-Agent': 'Upwatch/1.0 (+uptime monitor)' },
+			headers: requestHeaders(monitor),
+			body: monitor.method === 'POST' ? monitor.requestBody : undefined,
 		});
+		const latencyMs = Date.now() - startedAt;
+		const statusOk = response.status === monitor.expectedStatus;
+		let keywordOk = true;
+		if (monitor.expectKeyword !== null && monitor.method !== 'HEAD') {
+			const body = await readBodyLimited(response, MAX_BODY_MATCH_BYTES, true);
+			const contains =
+				body !== null && new TextDecoder().decode(body).toLocaleLowerCase().includes(monitor.expectKeyword.toLocaleLowerCase());
+			keywordOk = monitor.keywordInverted ? !contains : contains;
+		} else {
 			await response.body?.cancel();
-		const ok = response.status === monitor.expectedStatus;
+		}
+		const ok = statusOk && keywordOk;
+		let error: string | null = null;
+		if (!statusOk) error = `Expected HTTP ${monitor.expectedStatus}, received ${response.status}`;
+		else if (!keywordOk) {
+			error = `${monitor.keywordInverted ? 'Response contained' : 'Response did not contain'} "${monitor.expectKeyword}"`.slice(0, 200);
+		}
 		return {
 			ok,
+			degraded: ok && monitor.degradedLatencyMs !== null && latencyMs > monitor.degradedLatencyMs,
 			statusCode: response.status,
-			latencyMs: Date.now() - startedAt,
+			latencyMs,
-			error: ok ? null : `Expected HTTP ${monitor.expectedStatus}, received ${response.status}`,
+			error,
 		};
 	} catch (error) {
 		return {
 			ok: false,
+			degraded: false,
 			statusCode: null,
 			latencyMs: Date.now() - startedAt,
 			error: error instanceof Error ? error.message.slice(0, 200) : 'Request failed',

src/worker/checks/run-due-checks.ts

@@ -74,9 +74,10 @@ export async function runDueChecks(env: Env, ctx?: Pick<ExecutionContext, 'waitU
 	let aiMessagesQueued = 0;
 	const notificationBudget: NotificationBudget = { remaining: MAX_NOTIFICATIONS_PER_RUN };
 	const notifications = persisted.flatMap((item) => {
-		if (item.transition !== 'opened' && item.transition !== 'resolved') return [];
+		const work: Promise<unknown>[] = [];
+		if (item.transition === 'opened' || item.transition === 'resolved') {
 			const kind: AlertTransition = item.transition;
-		const work: Promise<unknown>[] = [
+			work.push(
 				dispatchNotification(
 					env,
 					{
@@ -91,7 +92,27 @@ export async function runDueChecks(env: Env, ctx?: Pick<ExecutionContext, 'waitU
 					},
 					notificationBudget,
 				),
-		];
+			);
+		}
+		if (item.latencyTransition) {
+			const degraded = item.latencyTransition === 'degraded';
+			work.push(
+				dispatchNotification(
+					env,
+					{
+						monitor: { id: item.monitor.id, name: item.monitor.name, url: item.monitor.url },
+						kind: degraded ? 'degraded' : 'recovered_degraded',
+						incidentId: null,
+						title: degraded ? `${item.monitor.name} performance degraded` : `${item.monitor.name} performance recovered`,
+						body: degraded ? `Response time was ${item.result.latencyMs} ms.` : 'Response time returned to normal.',
+						statusCode: item.result.statusCode,
+						error: item.result.error,
+						at: item.checkedAt,
+					},
+					notificationBudget,
+				),
+			);
+		}
 		if (item.transition === 'opened' && item.monitor.alertsEnabled && aiMessagesQueued < MAX_AI_MESSAGES_PER_RUN) {
 			aiMessagesQueued += 1;
 			work.push(generateIncidentMessage(env, { monitor: item.monitor, result: item.result }));

src/worker/db/schema.ts

@@ -46,6 +46,11 @@ export const monitors = sqliteTable(
 		url: text('url').notNull(),
 		method: text('method').notNull().default('GET'),
 		expectedStatus: integer('expected_status').notNull().default(200),
+		expectKeyword: text('expect_keyword'),
+		keywordInverted: integer('keyword_inverted', { mode: 'boolean' }).notNull().default(false),
+		requestHeaders: text('request_headers'),
+		requestBody: text('request_body'),
+		degradedLatencyMs: integer('degraded_latency_ms'),
 		intervalSeconds: integer('interval_seconds').notNull().default(300),
 		timeoutMs: integer('timeout_ms').notNull().default(10_000),
 		enabled: integer('enabled', { mode: 'boolean' }).notNull().default(true),
@@ -56,8 +61,12 @@ export const monitors = sqliteTable(
 		failureThreshold: integer('failure_threshold').notNull().default(2),
 		/** Failures since the last successful check. Maintenance checks do not change this value. */
 		consecutiveFailures: integer('consecutive_failures').notNull().default(0),
+		/** Slow successful checks since latency last recovered. Maintenance checks do not change this value. */
+		consecutiveSlow: integer('consecutive_slow').notNull().default(0),
 		/** Confirmed state, not the raw latest result. */
 		lastOk: integer('last_ok', { mode: 'boolean' }),
+		/** Confirmed degraded state. A down monitor is never degraded. */
+		lastDegraded: integer('last_degraded', { mode: 'boolean' }).notNull().default(false),
 		lastStatusCode: integer('last_status_code'),
 		lastLatencyMs: integer('last_latency_ms'),
 		lastError: text('last_error'),
@@ -112,6 +121,7 @@ export const checks = sqliteTable(
 		error: text('error'),
 		checkedAt: integer('checked_at', { mode: 'timestamp_ms' }).notNull(),
 		maintenance: integer('maintenance', { mode: 'boolean' }).notNull().default(false),
+		degraded: integer('degraded', { mode: 'boolean' }).notNull().default(false),
 	},
 	(table) => [index('checks_monitor_id_checked_at_idx').on(table.monitorId, table.checkedAt)],
 );

src/worker/lib/read-body.ts

@@ -0,0 +1,37 @@
+export async function readBodyLimited(response: Response, maximum: number, truncate: boolean) {
+	if (!response.body) return null;
+	const reader = response.body.getReader();
+	const chunks: Uint8Array[] = [];
+	let total = 0;
+	let exceeded = false;
+
+	try {
+		while (true) {
+			const { done, value } = await reader.read();
+			if (done) break;
+			const remaining = maximum - total;
+			if (value.byteLength > remaining) {
+				if (truncate && remaining > 0) {
+					chunks.push(value.subarray(0, remaining));
+					total += remaining;
+				}
+				exceeded = true;
+				await reader.cancel();
+				break;
+			}
+			chunks.push(value);
+			total += value.byteLength;
+		}
+	} catch {
+		return null;
+	}
+
+	if (exceeded && !truncate) return null;
+	const body = new Uint8Array(total);
+	let offset = 0;
+	for (const chunk of chunks) {
+		body.set(chunk, offset);
+		offset += chunk.byteLength;
+	}
+	return body.buffer;
+}

src/worker/notifications/providers/types.ts

@@ -8,7 +8,7 @@ export type TelegramConfig = { botToken: string; chatId: string };
 export type ChannelConfig = UrlConfig | TelegramConfig;
 
 export type NotificationEvent = {
-	kind: 'down' | 'recovered' | 'manual_opened' | 'manual_update' | 'test';
+	kind: 'down' | 'recovered' | 'degraded' | 'recovered_degraded' | 'manual_opened' | 'manual_update' | 'test';
 	monitor: { id: number; name: string; url: string } | null;
 	incidentId: number | null;
 	title: string;
@@ -49,6 +49,8 @@ export function eventLabel(kind: NotificationEvent['kind']) {
 	return {
 		down: 'Service down',
 		recovered: 'Service recovered',
+		degraded: 'Service degraded',
+		recovered_degraded: 'Performance recovered',
 		manual_opened: 'Incident opened',
 		manual_update: 'Incident update',
 		test: 'Test notification',
@@ -57,6 +59,7 @@ export function eventLabel(kind: NotificationEvent['kind']) {
 
 export function eventColor(kind: NotificationEvent['kind']) {
 	if (kind === 'down' || kind === 'manual_opened') return '#dc2626';
-	if (kind === 'recovered') return '#16a34a';
+	if (kind === 'recovered' || kind === 'recovered_degraded') return '#16a34a';
+	if (kind === 'degraded') return '#d97706';
 	return '#2563eb';
 }

src/worker/routes/monitors.ts

@@ -8,6 +8,7 @@ import { checks, incidentMonitors, incidents, maintenanceWindowMonitors, monitor
 import { requireAuth, type AuthVariables } from '../lib/require-auth';
 import { loadActiveMaintenance } from '../maintenance/windows';
 import { isSafeRemoteUrl } from '../lib/safe-url';
+import { readBodyLimited } from '../lib/read-body';
 import { dispatchNotification } from '../notifications/dispatch';
 
 type MonitorMethod = 'GET' | 'HEAD' | 'POST';
@@ -17,6 +18,11 @@ type ParsedMonitorInput = {
 	url?: string;
 	method?: MonitorMethod;
 	expectedStatus?: number;
+	expectKeyword?: string | null;
+	keywordInverted?: boolean;
+	requestHeaders?: string | null;
+	requestBody?: string | null;
+	degradedLatencyMs?: number | null;
 	intervalSeconds?: number;
 	timeoutMs?: number;
 	retryCount?: number;
@@ -33,6 +39,8 @@ const FAVICON_FETCH_TIMEOUT_MS = 5_000;
 const MAX_FAVICON_BYTES = 1024 * 1024;
 const MAX_HEAD_BYTES = 128 * 1024;
 const MAX_REDIRECTS = 3;
+const HEADER_NAME = /^[!#$%&'*+\-.^_`|~0-9A-Za-z]{1,64}$/;
+const FORBIDDEN_HEADERS = new Set(['host', 'content-length', 'transfer-encoding', 'connection']);
 
 type FaviconResult = {
 	body: ArrayBuffer;
@@ -44,44 +52,6 @@ type EdgeCache = {
 	put(request: RequestInfo | URL, response: Response): Promise<void>;
 };
 
-async function readBodyLimited(response: Response, maximum: number, truncate: boolean) {
-	if (!response.body) return null;
-	const reader = response.body.getReader();
-	const chunks: Uint8Array[] = [];
-	let total = 0;
-	let exceeded = false;
-
-	try {
-		while (true) {
-			const { done, value } = await reader.read();
-			if (done) break;
-			const remaining = maximum - total;
-			if (value.byteLength > remaining) {
-				if (truncate && remaining > 0) {
-					chunks.push(value.subarray(0, remaining));
-					total += remaining;
-				}
-				exceeded = true;
-				await reader.cancel();
-				break;
-			}
-			chunks.push(value);
-			total += value.byteLength;
-		}
-	} catch {
-		return null;
-	}
-
-	if (exceeded && !truncate) return null;
-	const body = new Uint8Array(total);
-	let offset = 0;
-	for (const chunk of chunks) {
-		body.set(chunk, offset);
-		offset += chunk.byteLength;
-	}
-	return body.buffer;
-}
-
 async function fetchRemote(url: URL, maximumBytes: number, truncate = false) {
 	const controller = new AbortController();
 	const timeout = setTimeout(() => controller.abort(), FAVICON_FETCH_TIMEOUT_MS);
@@ -232,7 +202,7 @@ export function parseInteger(
 	return { ok: true, value: value as number };
 }
 
-export function parseMonitorInput(body: unknown, partial = false): ParseResult {
+export function parseMonitorInput(body: unknown, partial = false, storedMethod?: MonitorMethod): ParseResult {
 	if (!isRecord(body)) return { ok: false, message: 'Invalid request body' };
 
 	const value: ParsedMonitorInput = {};
@@ -263,6 +233,55 @@ export function parseMonitorInput(body: unknown, partial = false): ParseResult {
 		value.method = body.method as MonitorMethod;
 	}
 
+	if ('expectKeyword' in body) {
+		if (body.expectKeyword === null) value.expectKeyword = null;
+		else if (typeof body.expectKeyword !== 'string' || body.expectKeyword.trim().length < 1 || body.expectKeyword.trim().length > 200) {
+			return { ok: false, message: 'expectKeyword must be between 1 and 200 characters or null' };
+		} else value.expectKeyword = body.expectKeyword.trim();
+	}
+	if ('keywordInverted' in body) {
+		if (typeof body.keywordInverted !== 'boolean') return { ok: false, message: 'keywordInverted must be a boolean' };
+		value.keywordInverted = body.keywordInverted;
+	}
+	if ('requestHeaders' in body) {
+		if (body.requestHeaders === null) value.requestHeaders = null;
+		else if (!isRecord(body.requestHeaders)) return { ok: false, message: 'requestHeaders must be an object or null' };
+		else {
+			const entries = Object.entries(body.requestHeaders);
+			if (entries.length > 10) return { ok: false, message: 'requestHeaders cannot contain more than 10 headers' };
+			const normalized: Record<string, string> = {};
+			for (const [name, headerValue] of entries) {
+				const lowerName = name.toLowerCase();
+				if (!HEADER_NAME.test(name)) return { ok: false, message: `Invalid header name: ${name}` };
+				if (FORBIDDEN_HEADERS.has(lowerName) || lowerName.startsWith('cf-')) {
+					return { ok: false, message: `Header is not allowed: ${name}` };
+				}
+				if (typeof headerValue !== 'string' || headerValue.length > 512 || /[\r\n]/.test(headerValue)) {
+					return { ok: false, message: `Invalid value for header: ${name}` };
+				}
+				normalized[name] = headerValue;
+			}
+			value.requestHeaders = JSON.stringify(normalized);
+		}
+	}
+	if ('requestBody' in body) {
+		if (body.requestBody === null) value.requestBody = null;
+		else if (typeof body.requestBody !== 'string' || body.requestBody.length > 8_192) {
+			return { ok: false, message: 'requestBody must be no more than 8192 characters or null' };
+		} else value.requestBody = body.requestBody;
+		if (value.requestBody !== null && (value.method ?? storedMethod) !== 'POST') {
+			return { ok: false, message: 'requestBody can only be used with POST monitors' };
+		}
+	}
+	if ('degradedLatencyMs' in body) {
+		if (body.degradedLatencyMs === null) value.degradedLatencyMs = null;
+		else {
+			const parsed = parseInteger(body.degradedLatencyMs, 'degradedLatencyMs', 1, 30_000);
+			if (!parsed.ok) return parsed;
+			value.degradedLatencyMs = parsed.value;
+		}
+	}
+
 	for (const [key, label, minimum, maximum] of [
 		['expectedStatus', 'expectedStatus', 100, 599],
 		['intervalSeconds', 'intervalSeconds', 300, 86_400],
@@ -504,6 +523,11 @@ monitorRoutes.post('/', async (context) => {
 			url: parsed.value.url!,
 			method: parsed.value.method!,
 			expectedStatus: parsed.value.expectedStatus!,
+			expectKeyword: parsed.value.expectKeyword ?? null,
+			keywordInverted: parsed.value.keywordInverted ?? false,
+			requestHeaders: parsed.value.requestHeaders ?? null,
+			requestBody: parsed.value.requestBody ?? null,
+			degradedLatencyMs: parsed.value.degradedLatencyMs ?? null,
 			intervalSeconds: parsed.value.intervalSeconds!,
 			timeoutMs: parsed.value.timeoutMs!,
 			retryCount: parsed.value.retryCount ?? 1,
@@ -521,6 +545,9 @@ monitorRoutes.post('/', async (context) => {
 monitorRoutes.patch('/:id', async (context) => {
 	const id = parseId(context.req.param('id'));
 	if (id === null) return context.json({ message: 'Monitor not found' }, 404);
+	const db = getDb(context.env);
+	const [existing] = await db.select({ method: monitors.method }).from(monitors).where(eq(monitors.id, id)).limit(1);
+	if (!existing) return context.json({ message: 'Monitor not found' }, 404);
 
 	let body: unknown;
 	try {
@@ -529,13 +556,13 @@ monitorRoutes.patch('/:id', async (context) => {
 		return context.json({ message: 'Invalid request body' }, 400);
 	}
 
-	const parsed = parseMonitorInput(body, true);
+	const parsed = parseMonitorInput(body, true, existing.method as MonitorMethod);
 	if (!parsed.ok) return context.json({ message: parsed.message }, 400);
 	if (Object.keys(parsed.value).length === 0) {
 		return context.json({ message: 'Provide at least one field to update' }, 400);
 	}
 
-	const [monitor] = await getDb(context.env)
+	const [monitor] = await db
 		.update(monitors)
 		.set({ ...parsed.value, updatedAt: new Date() })
 		.where(eq(monitors.id, id))
@@ -573,7 +600,13 @@ monitorRoutes.post('/:id/check', async (context) => {
 	const result = await runCheckWithRetries(monitor);
 	const checkedAt = new Date();
 	const activeMaintenance = await loadActiveMaintenance(db, checkedAt);
-	const { statements, transition } = buildResultStatements(db, monitor, result, checkedAt, activeMaintenance.has(monitor.id));
+	const { statements, transition, latencyTransition } = buildResultStatements(
+		db,
+		monitor,
+		result,
+		checkedAt,
+		activeMaintenance.has(monitor.id),
+	);
 	await db.batch(statements as [(typeof statements)[number], ...typeof statements]);
 	if (transition === 'opened' || transition === 'resolved') {
 		await dispatchNotification(context.env, {
@@ -590,9 +623,22 @@ monitorRoutes.post('/:id/check', async (context) => {
 			await generateIncidentMessage(context.env, { monitor, result });
 		}
 	}
+	if (latencyTransition) {
+		const degraded = latencyTransition === 'degraded';
+		await dispatchNotification(context.env, {
+			monitor: { id: monitor.id, name: monitor.name, url: monitor.url },
+			kind: degraded ? 'degraded' : 'recovered_degraded',
+			incidentId: null,
+			title: degraded ? `${monitor.name} performance degraded` : `${monitor.name} performance recovered`,
+			body: degraded ? `Response time was ${result.latencyMs} ms.` : 'Response time returned to normal.',
+			statusCode: result.statusCode,
+			error: result.error,
+			at: checkedAt,
+		});
+	}
 	const [updated] = await db.select().from(monitors).where(eq(monitors.id, monitor.id)).limit(1);
 
-	return context.json({ result, transition, monitor: updated });
+	return context.json({ result, transition, latencyTransition, monitor: updated });
 });
 
 export default monitorRoutes;

src/worker/routes/status.ts

@@ -1,6 +1,6 @@
 import { and, desc, eq, gte, inArray, isNull, lt, sql } from 'drizzle-orm';
 import { Hono } from 'hono';
-import { deterministicIncidentMessage } from '../ai/fallback-message';
+import { DEGRADED_MESSAGE, deterministicIncidentMessage } from '../ai/fallback-message';
 import { getDb } from '../db/client';
 import { checks, incidentMonitors, incidents, incidentUpdates, monitorDailyStats, monitors } from '../db/schema';
 import { loadActiveMaintenance, type ActiveMaintenance } from '../maintenance/windows';
@@ -8,7 +8,7 @@ import { resolveFavicon } from './monitors';
 
 const DAY_MS = 24 * 60 * 60 * 1000;
 const FAVICON_CACHE_SECONDS = 86_400;
-type ServiceStatus = 'up' | 'down' | 'unknown' | 'maintenance';
+type ServiceStatus = 'up' | 'degraded' | 'down' | 'unknown' | 'maintenance';
 type OverallStatus = 'operational' | 'degraded' | 'down';
 type DailyAggregate = { monitorId: number; day: Date; totalChecks: number; upChecks: number };
 type EdgeCache = {
@@ -41,8 +41,8 @@ function parseLimit(raw: string | undefined, fallback: number, maximum: number)
 function roundUptime(upChecks: number, totalChecks: number) {
 	return totalChecks > 0 ? Math.round((upChecks / totalChecks) * 1_000) / 10 : null;
 }
-function serviceStatus(lastOk: boolean | null): ServiceStatus {
+function serviceStatus(lastOk: boolean | null, lastDegraded: boolean): ServiceStatus {
-	if (lastOk === true) return 'up';
+	if (lastOk === true) return lastDegraded ? 'degraded' : 'up';
 	if (lastOk === false) return 'down';
 	return 'unknown';
 }
@@ -55,6 +55,7 @@ function overallStatus(statuses: ServiceStatus[], manualImpacts: string[]): Over
 		if (status === 'down') down += 1;
 	}
 	let severity = down === 0 ? 0 : down === checked ? 2 : 1;
+	if (statuses.includes('degraded')) severity = Math.max(severity, 1);
 	for (const impact of manualImpacts)
 		severity = Math.max(severity, impact === 'critical' ? 2 : impact === 'minor' || impact === 'major' ? 1 : 0);
 	return severity === 2 ? 'down' : severity === 1 ? 'degraded' : 'operational';
@@ -117,6 +118,7 @@ statusRoutes.get('/', async (context) => {
 			id: monitors.id,
 			name: monitors.name,
 			lastOk: monitors.lastOk,
+			lastDegraded: monitors.lastDegraded,
 			lastStatusCode: monitors.lastStatusCode,
 			lastCheckedAt: monitors.lastCheckedAt,
 		})
@@ -197,12 +199,14 @@ statusRoutes.get('/', async (context) => {
 		return {
 			id: monitor.id,
 			name: monitor.name,
-			status: maintenance ? ('maintenance' as const) : serviceStatus(monitor.lastOk),
+			status: maintenance ? ('maintenance' as const) : serviceStatus(monitor.lastOk, monitor.lastDegraded),
 			message:
 				!maintenance && monitor.lastOk === false
 					? incident
 						? (latestUpdates.get(incident.id)?.body ?? deterministicIncidentMessage(incident.startStatusCode))
 						: deterministicIncidentMessage(monitor.lastStatusCode)
+					: !maintenance && monitor.lastOk === true && monitor.lastDegraded
+						? DEGRADED_MESSAGE
 						: null,
 			maintenance: maintenance ? { name: maintenance.name, endsAt: maintenance.endsAt.toISOString() } : null,
 			lastCheckedAt: monitor.lastCheckedAt?.toISOString() ?? null,

test/checks.spec.ts

@@ -27,12 +27,19 @@ async function insertMonitor(overrides: Record<string, unknown> = {}) {
 		url: 'https://example.com/health',
 		method: 'GET',
 		expected_status: 200,
+		expect_keyword: null,
+		keyword_inverted: 0,
+		request_headers: null,
+		request_body: null,
+		degraded_latency_ms: null,
 		interval_seconds: 300,
 		timeout_ms: 10_000,
 		retry_count: 0,
 		failure_threshold: 2,
 		consecutive_failures: 0,
+		consecutive_slow: 0,
 		enabled: 1,
+		last_degraded: 0,
 		last_checked_at: null,
 		created_at: now,
 		updated_at: now,
@@ -41,9 +48,10 @@ async function insertMonitor(overrides: Record<string, unknown> = {}) {
 	const result = await env.DB.prepare(
 		`
 		INSERT INTO monitors
-		(name, url, method, expected_status, interval_seconds, timeout_ms, retry_count, failure_threshold,
+		(name, url, method, expected_status, expect_keyword, keyword_inverted, request_headers, request_body,
-		 consecutive_failures, enabled, alerts_enabled, last_ok, last_checked_at, created_at, updated_at)
+		 degraded_latency_ms, interval_seconds, timeout_ms, retry_count, failure_threshold, consecutive_failures,
-		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?)
+		 consecutive_slow, enabled, alerts_enabled, last_ok, last_degraded, last_checked_at, created_at, updated_at)
+		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?)
 	`,
 	)
 		.bind(
@@ -51,13 +59,20 @@ async function insertMonitor(overrides: Record<string, unknown> = {}) {
 			values.url,
 			values.method,
 			values.expected_status,
+			values.expect_keyword,
+			values.keyword_inverted,
+			values.request_headers,
+			values.request_body,
+			values.degraded_latency_ms,
 			values.interval_seconds,
 			values.timeout_ms,
 			values.retry_count,
 			values.failure_threshold,
 			values.consecutive_failures,
+			values.consecutive_slow,
 			values.enabled,
 			overrides.last_ok ?? null,
+			values.last_degraded,
 			values.last_checked_at,
 			values.created_at,
 			values.updated_at,
@@ -100,6 +115,98 @@ describe('scheduled monitor checks', () => {
 		expect(checkCount?.count).toBe(1);
 	});
 
+	it('passes a case-insensitive keyword assertion and sends configured request data', async () => {
+		await insertMonitor({
+			method: 'POST',
+			expect_keyword: 'healthy',
+			request_headers: JSON.stringify({ Authorization: 'Bearer test' }),
+			request_body: '{"probe":true}',
+		});
+		const fetchMock = vi.fn(async () => new Response('{"status":"HEALTHY"}', { status: 200 }));
+		vi.stubGlobal('fetch', fetchMock);
+
+		const summary = await runDueChecks(env);
+		expect(summary.up).toBe(1);
+		const init = fetchMock.mock.calls[0][1] as RequestInit;
+		expect(init.method).toBe('POST');
+		expect(init.body).toBe('{"probe":true}');
+		expect(new Headers(init.headers).get('Authorization')).toBe('Bearer test');
+		expect(new Headers(init.headers).get('Content-Type')).toBe('application/json');
+	});
+
+	it('fails and opens an incident when the expected keyword is absent', async () => {
+		const id = await insertMonitor({ last_ok: 1, expect_keyword: 'healthy' });
+		vi.stubGlobal(
+			'fetch',
+			vi.fn(async () => new Response('{"status":"error"}', { status: 200 })),
+		);
+
+		await runDueChecks(env);
+		await env.DB.prepare('UPDATE monitors SET last_checked_at = NULL WHERE id = ?').bind(id).run();
+		const summary = await runDueChecks(env);
+
+		expect(summary.opened).toBe(1);
+		const check = await env.DB.prepare('SELECT ok, error FROM checks WHERE monitor_id = ? ORDER BY id DESC LIMIT 1')
+			.bind(id)
+			.first<{ ok: number; error: string }>();
+		expect(check).toEqual({ ok: 0, error: 'Response did not contain "healthy"' });
+	});
+
+	it('supports inverted keyword assertions and safely truncates large bodies', async () => {
+		await insertMonitor({ expect_keyword: 'maintenance', keyword_inverted: 1 });
+		vi.stubGlobal(
+			'fetch',
+			vi.fn(async () => new Response(`${'a'.repeat(256 * 1024)}maintenance`, { status: 200 })),
+		);
+
+		const summary = await runDueChecks(env);
+		expect(summary.up).toBe(1);
+		expect((await env.DB.prepare('SELECT ok FROM checks').first<{ ok: number }>())?.ok).toBe(1);
+	});
+
+	it('confirms degraded latency, notifies, and recovers on a fast check', async () => {
+		const id = await insertMonitor({ last_ok: 1, degraded_latency_ms: 1, failure_threshold: 2 });
+		const now = Date.now();
+		await env.DB.prepare(
+			`INSERT INTO notification_channels (name, type, config, enabled, notify_manual, created_at, updated_at)
+			 VALUES ('Webhook', 'webhook', '{"url":"https://hooks.example.test/events"}', 1, 1, ?, ?)`,
+		)
+			.bind(now, now)
+			.run();
+		const events: string[] = [];
+		let slow = true;
+		vi.stubGlobal(
+			'fetch',
+			vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
+				if (new URL(input.toString()).hostname === 'hooks.example.test') {
+					events.push((JSON.parse(String(init?.body)) as { event: string }).event);
+					return new Response(null, { status: 204 });
+				}
+				if (slow) await new Promise((resolve) => setTimeout(resolve, 5));
+				return new Response(null, { status: 200 });
+			}),
+		);
+
+		await runDueChecks(env);
+		await env.DB.prepare('UPDATE monitors SET last_checked_at = NULL WHERE id = ?').bind(id).run();
+		await runDueChecks(env);
+		expect(events).toEqual(['degraded']);
+		expect(
+			await env.DB.prepare('SELECT last_degraded, consecutive_slow FROM monitors WHERE id = ?')
+				.bind(id)
+				.first<{ last_degraded: number; consecutive_slow: number }>(),
+		).toEqual({ last_degraded: 1, consecutive_slow: 2 });
+		expect((await env.DB.prepare('SELECT degraded FROM checks ORDER BY id DESC LIMIT 1').first<{ degraded: number }>())?.degraded).toBe(1);
+
+		slow = false;
+		await env.DB.prepare('UPDATE monitors SET last_checked_at = NULL, degraded_latency_ms = 30000 WHERE id = ?').bind(id).run();
+		await runDueChecks(env);
+		expect(events).toEqual(['degraded', 'recovered_degraded']);
+		expect(
+			(await env.DB.prepare('SELECT last_degraded FROM monitors WHERE id = ?').bind(id).first<{ last_degraded: number }>())?.last_degraded,
+		).toBe(0);
+	});
+
 	it('records a first mismatched status as pending without opening an incident', async () => {
 		const id = await insertMonitor({ last_ok: 1 });
 		const fetchMock = vi.fn(async () => new Response(null, { status: 500 }));

test/maintenance.spec.ts

@@ -83,21 +83,33 @@ describe('maintenance windows', () => {
 
 	it('records failed probes as maintenance without incidents or monitor state changes', async () => {
 		const id = await insertMonitor(1);
-		await env.DB.prepare('UPDATE monitors SET retry_count = 3, consecutive_failures = 1 WHERE id = ?').bind(id).run();
+		await env.DB.prepare(
+			'UPDATE monitors SET retry_count = 3, consecutive_failures = 1, consecutive_slow = 2, last_degraded = 1 WHERE id = ?',
+		)
+			.bind(id)
+			.run();
 		await insertActiveWindow(id);
 		const fetchMock = vi.fn(async () => new Response(null, { status: 503 }));
 		vi.stubGlobal('fetch', fetchMock);
 
 		await runDueChecks(env);
 		const check = await env.DB.prepare('SELECT maintenance, ok FROM checks WHERE monitor_id = ?').bind(id).first();
-		const monitor = await env.DB.prepare('SELECT last_ok, consecutive_failures, last_status_code FROM monitors WHERE id = ?')
+		const monitor = await env.DB.prepare(
+			'SELECT last_ok, consecutive_failures, consecutive_slow, last_degraded, last_status_code FROM monitors WHERE id = ?',
+		)
 			.bind(id)
 			.first();
 		const incident = await env.DB.prepare('SELECT COUNT(*) AS count FROM incident_monitors WHERE monitor_id = ?')
 			.bind(id)
 			.first<{ count: number }>();
 		expect(check).toMatchObject({ maintenance: 1, ok: 0 });
-		expect(monitor).toMatchObject({ last_ok: 1, consecutive_failures: 1, last_status_code: 503 });
+		expect(monitor).toMatchObject({
+			last_ok: 1,
+			consecutive_failures: 1,
+			consecutive_slow: 2,
+			last_degraded: 1,
+			last_status_code: 503,
+		});
 		expect(incident?.count).toBe(0);
 		expect(fetchMock).toHaveBeenCalledTimes(1);
 	});

test/monitors.spec.ts

@@ -180,6 +180,15 @@ describe('monitor API', () => {
 		[{ retryCount: 4 }, 'retryCount must be an integer between 0 and 3'],
 		[{ failureThreshold: 0 }, 'failureThreshold must be an integer between 1 and 10'],
 		[{ failureThreshold: 11 }, 'failureThreshold must be an integer between 1 and 10'],
+		[{ requestHeaders: { Authorization: 'Bearer ok\r\nX-Evil: true' } }, 'Invalid value for header: Authorization'],
+		[{ requestHeaders: { Host: 'example.test' } }, 'Header is not allowed: Host'],
+		[
+			{ requestHeaders: Object.fromEntries(Array.from({ length: 11 }, (_, index) => [`X-${index}`, 'value'])) },
+			'requestHeaders cannot contain more than 10 headers',
+		],
+		[{ requestBody: '{}' }, 'requestBody can only be used with POST monitors'],
+		[{ degradedLatencyMs: 0 }, 'degradedLatencyMs must be an integer between 1 and 30000'],
+		[{ degradedLatencyMs: 30_001 }, 'degradedLatencyMs must be an integer between 1 and 30000'],
 	] as const)('rejects invalid monitor input %o', async (overrides, message) => {
 		const response = await createMonitor(await authenticatedCookie(), overrides);
 		expect(response.status).toBe(400);
@@ -209,6 +218,32 @@ describe('monitor API', () => {
 		});
 	});
 
+	it('stores advanced settings and allows nullable fields to be cleared', async () => {
+		const cookie = await authenticatedCookie();
+		const created = await (
+			await createMonitor(cookie, {
+				method: 'POST',
+				expectKeyword: 'healthy',
+				keywordInverted: true,
+				requestHeaders: { Authorization: 'Bearer test' },
+				requestBody: '{"probe":true}',
+				degradedLatencyMs: 1500,
+			})
+		).json<{ monitor: { id: number; requestHeaders: string } }>();
+		expect(JSON.parse(created.monitor.requestHeaders)).toEqual({ Authorization: 'Bearer test' });
+
+		const response = await apiFetch(`/api/monitors/${created.monitor.id}`, 'PATCH', cookie, {
+			expectKeyword: null,
+			requestHeaders: null,
+			requestBody: null,
+			degradedLatencyMs: null,
+		});
+		const body = await response.json<{
+			monitor: { expectKeyword: null; requestHeaders: null; requestBody: null; degradedLatencyMs: null };
+		}>();
+		expect(body.monitor).toMatchObject({ expectKeyword: null, requestHeaders: null, requestBody: null, degradedLatencyMs: null });
+	});
+
 	it('does not allow clients to set the internal failure counter', async () => {
 		const cookie = await authenticatedCookie();
 		const created = await (await createMonitor(cookie)).json<{ monitor: { id: number } }>();

test/status.spec.ts

@@ -1,6 +1,7 @@
 import { applyD1Migrations, type D1Migration } from 'cloudflare:test';
 import { env, exports as worker } from 'cloudflare:workers';
 import { beforeAll, beforeEach, describe, expect, it } from 'vitest';
+import { DEGRADED_MESSAGE } from '../src/worker/ai/fallback-message';
 
 const DAY_MS = 24 * 60 * 60 * 1000;
 
@@ -10,7 +11,7 @@ type PublicStatusResponse = {
 	services: Array<{
 		id: number;
 		name: string;
-		status: 'up' | 'down' | 'unknown' | 'maintenance';
+		status: 'up' | 'degraded' | 'down' | 'unknown' | 'maintenance';
 		message: string | null;
 		maintenance: { name: string; endsAt: string } | null;
 		lastCheckedAt: string | null;
@@ -46,15 +47,16 @@ async function insertMonitor(input: {
 	lastError?: string | null;
 	consecutiveFailures?: number;
 	failureThreshold?: number;
+	lastDegraded?: boolean;
 }) {
 	const now = Date.now();
 	const result = await env.DB.prepare(
 		`
 		INSERT INTO monitors (
 			name, url, method, expected_status, interval_seconds, timeout_ms,
-			enabled, alerts_enabled, failure_threshold, consecutive_failures, last_ok, last_status_code, last_latency_ms,
+			enabled, alerts_enabled, failure_threshold, consecutive_failures, last_ok, last_degraded, last_status_code, last_latency_ms,
 			last_error, last_checked_at, created_at, updated_at
-		) VALUES (?, ?, 'GET', 200, 300, 10000, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+		) VALUES (?, ?, 'GET', 200, 300, 10000, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
 	`,
 	)
 		.bind(
@@ -64,6 +66,7 @@ async function insertMonitor(input: {
 			input.failureThreshold ?? 2,
 			input.consecutiveFailures ?? 0,
 			input.lastOk === null || input.lastOk === undefined ? null : input.lastOk ? 1 : 0,
+			input.lastDegraded ? 1 : 0,
 			input.lastStatusCode ?? null,
 			input.lastLatencyMs ?? null,
 			input.lastError ?? null,
@@ -139,6 +142,14 @@ describe('public status API', () => {
 		expect(body.services[0]).toMatchObject({ status: 'up', message: null });
 	});
 
+	it('reports a confirmed degraded monitor and public latency message', async () => {
+		await insertMonitor({ name: 'Slow API', lastOk: true, lastDegraded: true, lastLatencyMs: 2200 });
+
+		const body = await (await statusFetch()).json<PublicStatusResponse>();
+		expect(body.overall).toBe('degraded');
+		expect(body.services[0]).toMatchObject({ status: 'degraded', message: DEGRADED_MESSAGE });
+	});
+
 	it('excludes disabled monitors and reports degraded health for a partial outage', async () => {
 		await insertMonitor({ name: 'Healthy', lastOk: true });
 		await insertMonitor({ name: 'Unavailable', lastOk: false });