diff --git a/migrations/0007_true_mauler.sql b/migrations/0007_true_mauler.sql new file mode 100644 index 0000000..64da692 --- /dev/null +++ b/migrations/0007_true_mauler.sql @@ -0,0 +1,4 @@ +ALTER TABLE `monitors` ADD `retry_count` integer DEFAULT 1 NOT NULL;--> statement-breakpoint +ALTER TABLE `monitors` ADD `failure_threshold` integer DEFAULT 2 NOT NULL;--> statement-breakpoint +ALTER TABLE `monitors` ADD `consecutive_failures` integer DEFAULT 0 NOT NULL;--> statement-breakpoint +UPDATE `monitors` SET `consecutive_failures` = `failure_threshold` WHERE `last_ok` = 0; diff --git a/migrations/meta/0007_snapshot.json b/migrations/meta/0007_snapshot.json new file mode 100644 index 0000000..d185312 --- /dev/null +++ b/migrations/meta/0007_snapshot.json @@ -0,0 +1,993 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "3e14e2ff-5a06-45c1-83c8-91d471c8cdf7", + "prevId": "6e3277e0-8fe4-4619-8cb4-52c7c5185416", + "tables": { + "admin_credentials": { + "name": "admin_credentials", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "password_hash": { + "name": "password_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ai_settings": { + "name": "ai_settings", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "base_url": { + "name": "base_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "api_key": { + "name": "api_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "checks": { + "name": "checks", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "monitor_id": { + "name": "monitor_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ok": { + "name": "ok", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status_code": { + "name": "status_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "latency_ms": { + "name": "latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "checked_at": { + "name": "checked_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "maintenance": { + "name": "maintenance", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "checks_monitor_id_checked_at_idx": { + "name": "checks_monitor_id_checked_at_idx", + "columns": [ + "monitor_id", + "checked_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "checks_monitor_id_monitors_id_fk": { + "name": "checks_monitor_id_monitors_id_fk", + "tableFrom": "checks", + "tableTo": "monitors", + "columnsFrom": [ + "monitor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "incident_monitors": { + "name": "incident_monitors", + "columns": { + "incident_id": { + "name": "incident_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "monitor_id": { + "name": "monitor_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "incident_monitors_monitor_id_idx": { + "name": "incident_monitors_monitor_id_idx", + "columns": [ + "monitor_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "incident_monitors_incident_id_incidents_id_fk": { + "name": "incident_monitors_incident_id_incidents_id_fk", + "tableFrom": "incident_monitors", + "tableTo": "incidents", + "columnsFrom": [ + "incident_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "incident_monitors_monitor_id_monitors_id_fk": { + "name": "incident_monitors_monitor_id_monitors_id_fk", + "tableFrom": "incident_monitors", + "tableTo": "monitors", + "columnsFrom": [ + "monitor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "incident_monitors_incident_id_monitor_id_pk": { + "columns": [ + "incident_id", + "monitor_id" + ], + "name": "incident_monitors_incident_id_monitor_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "incident_updates": { + "name": "incident_updates", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "incident_id": { + "name": "incident_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'manual'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "incident_updates_incident_id_created_at_idx": { + "name": "incident_updates_incident_id_created_at_idx", + "columns": [ + "incident_id", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "incident_updates_incident_id_incidents_id_fk": { + "name": "incident_updates_incident_id_incidents_id_fk", + "tableFrom": "incident_updates", + "tableTo": "incidents", + "columnsFrom": [ + "incident_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "incidents": { + "name": "incidents", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'investigating'" + }, + "impact": { + "name": "impact", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'major'" + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'auto'" + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resolved_at": { + "name": "resolved_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "start_status_code": { + "name": "start_status_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "start_error": { + "name": "start_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "incidents_started_at_idx": { + "name": "incidents_started_at_idx", + "columns": [ + "started_at" + ], + "isUnique": false + }, + "incidents_resolved_at_idx": { + "name": "incidents_resolved_at_idx", + "columns": [ + "resolved_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "login_attempts": { + "name": "login_attempts", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "attempted_at": { + "name": "attempted_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "login_attempts_ip_attempted_at_idx": { + "name": "login_attempts_ip_attempted_at_idx", + "columns": [ + "ip_address", + "attempted_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "maintenance_window_monitors": { + "name": "maintenance_window_monitors", + "columns": { + "window_id": { + "name": "window_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "monitor_id": { + "name": "monitor_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "maintenance_window_monitors_monitor_id_idx": { + "name": "maintenance_window_monitors_monitor_id_idx", + "columns": [ + "monitor_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "maintenance_window_monitors_window_id_maintenance_windows_id_fk": { + "name": "maintenance_window_monitors_window_id_maintenance_windows_id_fk", + "tableFrom": "maintenance_window_monitors", + "tableTo": "maintenance_windows", + "columnsFrom": [ + "window_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "maintenance_window_monitors_monitor_id_monitors_id_fk": { + "name": "maintenance_window_monitors_monitor_id_monitors_id_fk", + "tableFrom": "maintenance_window_monitors", + "tableTo": "monitors", + "columnsFrom": [ + "monitor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "maintenance_window_monitors_window_id_monitor_id_pk": { + "columns": [ + "window_id", + "monitor_id" + ], + "name": "maintenance_window_monitors_window_id_monitor_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "maintenance_windows": { + "name": "maintenance_windows", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "start_minute": { + "name": "start_minute", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "duration_minutes": { + "name": "duration_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "timezone": { + "name": "timezone", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'UTC'" + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "maintenance_windows_enabled_idx": { + "name": "maintenance_windows_enabled_idx", + "columns": [ + "enabled" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "monitor_daily_stats": { + "name": "monitor_daily_stats", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "monitor_id": { + "name": "monitor_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "day": { + "name": "day", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total_checks": { + "name": "total_checks", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "up_checks": { + "name": "up_checks", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "avg_latency_ms": { + "name": "avg_latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "min_latency_ms": { + "name": "min_latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "max_latency_ms": { + "name": "max_latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "monitor_daily_stats_monitor_id_day_uidx": { + "name": "monitor_daily_stats_monitor_id_day_uidx", + "columns": [ + "monitor_id", + "day" + ], + "isUnique": true + } + }, + "foreignKeys": { + "monitor_daily_stats_monitor_id_monitors_id_fk": { + "name": "monitor_daily_stats_monitor_id_monitors_id_fk", + "tableFrom": "monitor_daily_stats", + "tableTo": "monitors", + "columnsFrom": [ + "monitor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "monitors": { + "name": "monitors", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "method": { + "name": "method", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'GET'" + }, + "expected_status": { + "name": "expected_status", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 200 + }, + "interval_seconds": { + "name": "interval_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 300 + }, + "timeout_ms": { + "name": "timeout_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 10000 + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "alerts_enabled": { + "name": "alerts_enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "retry_count": { + "name": "retry_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "failure_threshold": { + "name": "failure_threshold", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 2 + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_ok": { + "name": "last_ok", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_status_code": { + "name": "last_status_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_latency_ms": { + "name": "last_latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_checked_at": { + "name": "last_checked_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "monitors_enabled_last_checked_at_idx": { + "name": "monitors_enabled_last_checked_at_idx", + "columns": [ + "enabled", + "last_checked_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "notification_settings": { + "name": "notification_settings", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "webhook_url": { + "name": "webhook_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "webhook_enabled": { + "name": "webhook_enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sessions": { + "name": "sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "sessions_expires_at_idx": { + "name": "sessions_expires_at_idx", + "columns": [ + "expires_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/migrations/meta/_journal.json b/migrations/meta/_journal.json index 021f611..c533e85 100644 --- a/migrations/meta/_journal.json +++ b/migrations/meta/_journal.json @@ -50,6 +50,13 @@ "when": 1788012122846, "tag": "0006_gifted_deathstrike", "breakpoints": true + }, + { + "idx": 7, + "version": "6", + "when": 1788057318682, + "tag": "0007_true_mauler", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/client/api/monitors.ts b/src/client/api/monitors.ts index a116302..3b8737e 100644 --- a/src/client/api/monitors.ts +++ b/src/client/api/monitors.ts @@ -12,6 +12,9 @@ export type Monitor = { timeoutMs: number; enabled: boolean; alertsEnabled: boolean; + retryCount: number; + failureThreshold: number; + consecutiveFailures: number; lastOk: boolean | null; lastStatusCode: number | null; lastLatencyMs: number | null; @@ -28,6 +31,8 @@ export type MonitorInput = { expectedStatus: number; intervalSeconds: number; timeoutMs: number; + retryCount?: number; + failureThreshold?: number; enabled?: boolean; alertsEnabled?: boolean; }; @@ -37,14 +42,22 @@ export type CheckResult = { statusCode: number | null; latencyMs: number; error: string | null; + attempts: number; }; -export type Check = CheckResult & { +export type Check = { id: number; monitorId: number; + ok: boolean; + statusCode: number | null; + latencyMs: number; + error: string | null; checkedAt: string; + maintenance: boolean; }; +export type CheckTransition = 'opened' | 'pending' | 'cleared' | 'resolved' | null; + export type Incident = { id: number; title: string | null; @@ -109,5 +122,5 @@ export function deleteMonitor(id: number) { } export function runMonitorCheck(id: number) { - return postJson<{ result: CheckResult; monitor: Monitor }>(`/api/monitors/${id}/check`); + return postJson<{ result: CheckResult; transition: CheckTransition; monitor: Monitor }>(`/api/monitors/${id}/check`); } diff --git a/src/client/components/dashboard/DashboardOverview.tsx b/src/client/components/dashboard/DashboardOverview.tsx index cfa5e33..5896bd6 100644 --- a/src/client/components/dashboard/DashboardOverview.tsx +++ b/src/client/components/dashboard/DashboardOverview.tsx @@ -12,6 +12,7 @@ export function DashboardOverview({ onAddMonitor }: DashboardOverviewProps) { const monitors = monitorsQuery.data?.monitors ?? []; const up = monitors.filter((monitor) => monitor.lastOk === true).length; const down = monitors.filter((monitor) => monitor.lastOk === false).length; + const degrading = monitors.filter((monitor) => monitor.lastOk !== false && monitor.consecutiveFailures > 0).length; return ( <> @@ -42,7 +43,7 @@ export function DashboardOverview({ onAddMonitor }: DashboardOverviewProps) {

Currently down

{down} - Needs attention + {degrading > 0 ? `${degrading} degrading` : 'Needs attention'} diff --git a/src/client/components/dashboard/MonitorFormDialog.tsx b/src/client/components/dashboard/MonitorFormDialog.tsx index 5d1fbb5..a1410e0 100644 --- a/src/client/components/dashboard/MonitorFormDialog.tsx +++ b/src/client/components/dashboard/MonitorFormDialog.tsx @@ -14,6 +14,8 @@ export const DEFAULT_MONITOR_INPUT: MonitorInput = { expectedStatus: 200, intervalSeconds: 300, timeoutMs: 10_000, + retryCount: 1, + failureThreshold: 2, enabled: true, }; @@ -43,6 +45,8 @@ function monitorInput(monitor: Monitor | null): MonitorInput { expectedStatus: monitor.expectedStatus, intervalSeconds: monitor.intervalSeconds, timeoutMs: monitor.timeoutMs, + retryCount: monitor.retryCount, + failureThreshold: monitor.failureThreshold, enabled: monitor.enabled, alertsEnabled: monitor.alertsEnabled, }; @@ -87,7 +91,9 @@ export function MonitorFormDialog({ editing, onClose }: MonitorFormDialogProps)

Configuration

{editing ? `Edit ${editing.name}` : 'Add a monitor'} - Checks run on the configured schedule, with a minimum interval of five minutes. + + Checks run at least every five minutes. Failures are retried immediately and must repeat before an incident is published. +
+
+ Retries after a failure + + Retried immediately, so a sub-second blip never reaches the status page. +
+
+ Confirmations before alerting + + Confirmed after about {Math.round(((form.failureThreshold ?? 2) * form.intervalSeconds) / 60)} minutes. +
setForm({ ...form, enabled })} /> diff --git a/src/client/components/dashboard/MonitorListPanel.tsx b/src/client/components/dashboard/MonitorListPanel.tsx index 2268a2c..f280e73 100644 --- a/src/client/components/dashboard/MonitorListPanel.tsx +++ b/src/client/components/dashboard/MonitorListPanel.tsx @@ -1,9 +1,10 @@ import { useState } from 'react'; import { AlertTriangle, ArrowRight, Database, History, Pencil, Power, PowerOff, RefreshCw, Trash2 } from 'lucide-react'; -import { Badge, type BadgeVariant } from '@/components/ui/badge'; +import { Badge } from '@/components/ui/badge'; import { Button } from '@/components/ui/button'; import { Empty, EmptyContent, EmptyDescription, EmptyMedia, EmptyTitle } from '@/components/ui/empty'; import type { Monitor } from '../../api/monitors'; +import { monitorState } from '../../lib/monitor-status'; import { navigate } from '../../lib/router'; import { useDeleteMonitorMutation, useMonitorsQuery, useRunCheckMutation, useUpdateMonitorMutation } from '../../queries/monitors'; import { SiteIcon } from '../SiteIcon'; @@ -23,12 +24,6 @@ function formatCheckedAt(value: string | null) { }).format(new Date(value)); } -function monitorStatus(monitor: Monitor): { label: string; className: BadgeVariant } { - if (monitor.lastOk === true) return { label: 'Up', className: 'online' }; - if (monitor.lastOk === false) return { label: 'Down', className: 'offline' }; - return { label: 'Not checked', className: 'checking' }; -} - function errorMessage(error: unknown, fallback: string) { return error instanceof Error ? error.message : fallback; } @@ -119,7 +114,7 @@ export function MonitorListPanel({ formOpen, onAddMonitor, onEdit }: MonitorList Actions
{monitors.map((monitor) => { - const status = monitorStatus(monitor); + const status = monitorState(monitor); const checking = checkMutation.isPending && checkMutation.variables === monitor.id; const deleting = deleteMutation.isPending && deleteMutation.variables === monitor.id; const toggling = updateMutation.isPending && updateMutation.variables?.id === monitor.id; @@ -145,12 +140,16 @@ export function MonitorListPanel({ formOpen, onAddMonitor, onEdit }: MonitorList
- {checking ? 'Checking' : status.label} + + {checking ? 'Checking' : status.label} + {monitor.lastStatusCode === null ? '—' : `HTTP ${monitor.lastStatusCode}`} ·{' '} {monitor.lastLatencyMs === null ? '—' : `${monitor.lastLatencyMs} ms`} - {monitor.lastError ?? formatCheckedAt(monitor.lastCheckedAt)} + + {status.detail ?? monitor.lastError ?? formatCheckedAt(monitor.lastCheckedAt)} +
); + const status = monitorState(monitor); return (
@@ -97,7 +92,7 @@ export function MonitorDetailPage({ id }: { id: number }) {
- +

Monitor #{monitor.id}

{monitor.name}

@@ -108,7 +103,9 @@ export function MonitorDetailPage({ id }: { id: number }) {
- {status.label} + + {status.label} + {!monitor.alertsEnabled && ( Alerts muted diff --git a/src/client/styles.css b/src/client/styles.css index f4ed623..7e28a17 100644 --- a/src/client/styles.css +++ b/src/client/styles.css @@ -412,7 +412,7 @@ button { grid-column: span 3; } .form-actions.compact-actions { - grid-column: span 2; + grid-column: 3 / -1; } .form-error { grid-column: 1 / -1; @@ -630,6 +630,12 @@ button { .row-status.offline { color: #ae3d3d; } +.row-status.pending { + color: #8a5a1f; +} +.row-status.pending i { + background: #e0913f; +} .row-status.checking { color: #8b7722; } @@ -933,6 +939,10 @@ button { border-color: #c5f1df; background: var(--primary-deep); } +.status-orb.pending { + border-color: #f6e2c8; + background: #d8892f; +} .status-orb.checking { border-color: #eee5bd; background: #c6aa38; diff --git a/src/components/ui/badge.tsx b/src/components/ui/badge.tsx index c0cbee8..522c0e7 100644 --- a/src/components/ui/badge.tsx +++ b/src/components/ui/badge.tsx @@ -8,6 +8,7 @@ const badgeVariants = cva('inline-flex items-center gap-1.75 text-[12px] font-me variant: { online: 'text-[#16885b] [&>i]:bg-(--primary-deep) [&>i]:shadow-[0_0_0_3px_rgb(62_207_142/0.12)]', offline: 'text-[#ae3d3d] [&>i]:bg-[#d05a5a]', + pending: 'text-[#8a5a1f] [&>i]:bg-[#e0913f] [&>i]:shadow-[0_0_0_3px_rgb(224_145_63/0.14)]', checking: 'text-[#8b7722] [&>i]:animate-[blink_1.1s_ease-in-out_infinite] [&>i]:bg-[#d7bd53]', maintenance: 'text-[#3f6fb8] [&>i]:bg-[#5a8fd0]', }, diff --git a/src/worker/checks/persist-result.ts b/src/worker/checks/persist-result.ts index 23caa6b..45d66bf 100644 --- a/src/worker/checks/persist-result.ts +++ b/src/worker/checks/persist-result.ts @@ -4,7 +4,9 @@ import { checks, incidentMonitors, incidents, incidentUpdates, monitors } from ' import { RECOVERY_UPDATE_BODY } from '../ai/fallback-message'; import type { CheckResult, Monitor } from './run-check'; -export type IncidentTransition = 'opened' | 'resolved' | null; +/** Only opened and resolved transitions are allowed to send alerts. */ +export type CheckTransition = 'opened' | 'pending' | 'cleared' | 'resolved' | null; +export type AlertTransition = Extract; type BatchStatement = Parameters[0][number]; @@ -19,10 +21,39 @@ export function buildResultStatements(db: Database, monitor: Monitor, result: Ch checkedAt, maintenance, }), + ]; + + if (maintenance) { + statements.push( + db + .update(monitors) + .set({ + lastStatusCode: result.statusCode, + lastLatencyMs: result.latencyMs, + lastError: result.error, + lastCheckedAt: checkedAt, + updatedAt: checkedAt, + }) + .where(eq(monitors.id, monitor.id)), + ); + return { statements, transition: null as CheckTransition, consecutiveFailures: monitor.consecutiveFailures }; + } + + const threshold = Math.max(1, monitor.failureThreshold); + const previousFailures = monitor.consecutiveFailures; + // This deliberately uses the monitor snapshot. Concurrent manual and scheduled checks may lose one increment, + // which delays confirmation by one check but cannot publish a false incident. + const nextFailures = result.ok ? 0 : previousFailures + 1; + const wasDown = monitor.lastOk === false; + const isDown = !result.ok && nextFailures >= threshold; + const confirmed = result.ok ? true : isDown ? false : undefined; + + statements.push( db .update(monitors) .set({ - ...(maintenance ? {} : { lastOk: result.ok }), + ...(confirmed === undefined ? {} : { lastOk: confirmed }), + consecutiveFailures: nextFailures, lastStatusCode: result.statusCode, lastLatencyMs: result.latencyMs, lastError: result.error, @@ -30,12 +61,10 @@ export function buildResultStatements(db: Database, monitor: Monitor, result: Ch updatedAt: checkedAt, }) .where(eq(monitors.id, monitor.id)), - ]; + ); - if (maintenance) return { statements, transition: null }; - - let transition: IncidentTransition = null; - if (monitor.lastOk !== false && !result.ok) { + let transition: CheckTransition = null; + if (!wasDown && isDown) { statements.push( db.insert(incidents).values({ status: 'investigating', @@ -50,7 +79,7 @@ export function buildResultStatements(db: Database, monitor: Monitor, result: Ch db.insert(incidentMonitors).values({ incidentId: sql`last_insert_rowid()`, monitorId: monitor.id }), ); transition = 'opened'; - } else if (monitor.lastOk === false && result.ok) { + } else if (wasDown && result.ok) { const openIncidentIds = db .select({ id: incidentMonitors.incidentId }) .from(incidentMonitors) @@ -82,7 +111,8 @@ export function buildResultStatements(db: Database, monitor: Monitor, result: Ch .where(and(eq(incidents.source, 'auto'), isNull(incidents.resolvedAt), inArray(incidents.id, openIncidentIds))), ); transition = 'resolved'; - } + } else if (!wasDown && !result.ok) transition = 'pending'; + else if (!wasDown && result.ok && previousFailures > 0) transition = 'cleared'; - return { statements, transition }; + return { statements, transition, consecutiveFailures: nextFailures }; } diff --git a/src/worker/checks/run-check.ts b/src/worker/checks/run-check.ts index cd9cf7e..391c099 100644 --- a/src/worker/checks/run-check.ts +++ b/src/worker/checks/run-check.ts @@ -35,3 +35,26 @@ export async function runCheck(monitor: Monitor): Promise { }; } } + +export const MAX_RETRY_COUNT = 3; +const RETRY_BACKOFF_MS = [400, 1_200, 2_500] as const; + +export type CheckAttemptResult = CheckResult & { attempts: number }; +export type RetryBudget = { remaining: number; deadline: number }; + +export async function runCheckWithRetries(monitor: Monitor, budget?: RetryBudget): Promise { + let result = await runCheck(monitor); + if (result.ok) return { ...result, attempts: 1 }; + + const allowed = Math.min(Math.max(0, monitor.retryCount), MAX_RETRY_COUNT); + let attempts = 1; + for (let retry = 0; retry < allowed; retry += 1) { + if (budget && (budget.remaining <= 0 || Date.now() >= budget.deadline)) break; + if (budget) budget.remaining -= 1; + await new Promise((resolve) => setTimeout(resolve, RETRY_BACKOFF_MS[Math.min(retry, RETRY_BACKOFF_MS.length - 1)])); + result = await runCheck(monitor); + attempts += 1; + if (result.ok) break; + } + return { ...result, attempts }; +} diff --git a/src/worker/checks/run-due-checks.ts b/src/worker/checks/run-due-checks.ts index ab16f96..2b1d676 100644 --- a/src/worker/checks/run-due-checks.ts +++ b/src/worker/checks/run-due-checks.ts @@ -4,17 +4,22 @@ import { getDb } from '../db/client'; import { monitors } from '../db/schema'; import { loadActiveMaintenance } from '../maintenance/windows'; import { sendIncidentAlert } from '../notifications/webhook'; -import { buildResultStatements } from './persist-result'; -import { runCheck } from './run-check'; +import { type AlertTransition, buildResultStatements } from './persist-result'; +import { runCheck, runCheckWithRetries, type RetryBudget } from './run-check'; const MAX_MONITORS_PER_RUN = 40; const MAX_AI_MESSAGES_PER_RUN = 10; const CONCURRENCY = 10; +export const MAX_RETRY_ATTEMPTS_PER_RUN = 60; +const RETRY_DEADLINE_MS = 90_000; export type DueCheckSummary = { checked: number; up: number; down: number; + pending: number; + opened: number; + retries: number; }; export async function runDueChecks(env: Env, ctx?: Pick): Promise { @@ -32,32 +37,35 @@ export async function runDueChecks(env: Env, ctx?: Pick>; + result: Awaited>; checkedAt: Date; + maintenance: boolean; }> = []; for (let offset = 0; offset < due.length; offset += CONCURRENCY) { const batch = due.slice(offset, offset + CONCURRENCY); const results = await Promise.all( - batch.map(async (monitor) => ({ - monitor, - result: await runCheck(monitor), - checkedAt: new Date(), - })), + batch.map(async (monitor) => { + const maintenance = activeMaintenance.has(monitor.id); + const retryable = !maintenance && monitor.retryCount > 0 && monitor.lastOk !== false; + const result = retryable ? await runCheckWithRetries(monitor, budget) : { ...(await runCheck(monitor)), attempts: 1 }; + return { monitor, result, checkedAt: new Date(), maintenance }; + }), ); completed.push(...results); } - const persisted = completed.map(({ monitor, result, checkedAt }) => ({ + const persisted = completed.map(({ monitor, result, checkedAt, maintenance }) => ({ monitor, result, checkedAt, - ...buildResultStatements(db, monitor, result, checkedAt, activeMaintenance.has(monitor.id)), + ...buildResultStatements(db, monitor, result, checkedAt, maintenance), })); const statements = persisted.flatMap((item) => item.statements); @@ -65,11 +73,12 @@ export async function runDueChecks(env: Env, ctx?: Pick { - if (item.transition === null) return []; + if (item.transition !== 'opened' && item.transition !== 'resolved') return []; + const kind: AlertTransition = item.transition; const work: Promise[] = [ sendIncidentAlert(env, { monitor: item.monitor, - kind: item.transition, + kind, result: item.result, at: item.checkedAt, }), @@ -87,5 +96,12 @@ export async function runDueChecks(env: Env, ctx?: Pick count + Number(item.result.ok), 0); - return { checked: completed.length, up, down: completed.length - up }; + return { + checked: completed.length, + up, + down: completed.length - up, + pending: persisted.reduce((count, item) => count + Number(item.transition === 'pending'), 0), + opened: persisted.reduce((count, item) => count + Number(item.transition === 'opened'), 0), + retries: completed.reduce((count, item) => count + item.result.attempts - 1, 0), + }; } diff --git a/src/worker/db/schema.ts b/src/worker/db/schema.ts index 28deb2d..7331ddf 100644 --- a/src/worker/db/schema.ts +++ b/src/worker/db/schema.ts @@ -58,6 +58,13 @@ export const monitors = sqliteTable( timeoutMs: integer('timeout_ms').notNull().default(10_000), enabled: integer('enabled', { mode: 'boolean' }).notNull().default(true), alertsEnabled: integer('alerts_enabled', { mode: 'boolean' }).notNull().default(true), + /** Number of immediate retries after a failed attempt. Zero disables retries. */ + retryCount: integer('retry_count').notNull().default(1), + /** Consecutive failed checks required before confirming an outage. */ + failureThreshold: integer('failure_threshold').notNull().default(2), + /** Failures since the last successful check. Maintenance checks do not change this value. */ + consecutiveFailures: integer('consecutive_failures').notNull().default(0), + /** Confirmed state, not the raw latest result. */ lastOk: integer('last_ok', { mode: 'boolean' }), lastStatusCode: integer('last_status_code'), lastLatencyMs: integer('last_latency_ms'), diff --git a/src/worker/routes/monitors.ts b/src/worker/routes/monitors.ts index ec9e9be..943ec6a 100644 --- a/src/worker/routes/monitors.ts +++ b/src/worker/routes/monitors.ts @@ -2,7 +2,7 @@ import { and, desc, eq, gte, isNull, or, sql } from 'drizzle-orm'; import { Hono } from 'hono'; import { generateIncidentMessage } from '../ai/incident-message'; import { buildResultStatements } from '../checks/persist-result'; -import { runCheck } from '../checks/run-check'; +import { MAX_RETRY_COUNT, runCheckWithRetries } from '../checks/run-check'; import { getDb } from '../db/client'; import { checks, incidentMonitors, incidents, maintenanceWindowMonitors, monitors } from '../db/schema'; import { requireAuth, type AuthVariables } from '../lib/require-auth'; @@ -19,6 +19,8 @@ type ParsedMonitorInput = { expectedStatus?: number; intervalSeconds?: number; timeoutMs?: number; + retryCount?: number; + failureThreshold?: number; enabled?: boolean; alertsEnabled?: boolean; }; @@ -272,6 +274,16 @@ export function parseMonitorInput(body: unknown, partial = false): ParseResult { value[key] = parsed.value; } } + for (const [key, label, minimum, maximum] of [ + ['retryCount', 'retryCount', 0, MAX_RETRY_COUNT], + ['failureThreshold', 'failureThreshold', 1, 10], + ] as const) { + if (key in body) { + const parsed = parseInteger(body[key], label, minimum, maximum); + if (!parsed.ok) return parsed; + value[key] = parsed.value; + } + } if ('enabled' in body) { if (typeof body.enabled !== 'boolean') { @@ -494,6 +506,8 @@ monitorRoutes.post('/', async (context) => { expectedStatus: parsed.value.expectedStatus!, intervalSeconds: parsed.value.intervalSeconds!, timeoutMs: parsed.value.timeoutMs!, + retryCount: parsed.value.retryCount ?? 1, + failureThreshold: parsed.value.failureThreshold ?? 2, enabled: parsed.value.enabled ?? true, alertsEnabled: parsed.value.alertsEnabled ?? true, createdAt: now, @@ -556,12 +570,12 @@ monitorRoutes.post('/:id/check', async (context) => { const [monitor] = await db.select().from(monitors).where(eq(monitors.id, id)).limit(1); if (!monitor) return context.json({ message: 'Monitor not found' }, 404); - const result = await runCheck(monitor); + const result = await runCheckWithRetries(monitor); const checkedAt = new Date(); const activeMaintenance = await loadActiveMaintenance(db, checkedAt); const { statements, transition } = buildResultStatements(db, monitor, result, checkedAt, activeMaintenance.has(monitor.id)); await db.batch(statements as [(typeof statements)[number], ...typeof statements]); - if (transition) { + if (transition === 'opened' || transition === 'resolved') { await sendIncidentAlert(context.env, { monitor, kind: transition, result, at: checkedAt }); if (transition === 'opened') { await generateIncidentMessage(context.env, { monitor, result }); @@ -569,7 +583,7 @@ monitorRoutes.post('/:id/check', async (context) => { } const [updated] = await db.select().from(monitors).where(eq(monitors.id, monitor.id)).limit(1); - return context.json({ result, monitor: updated }); + return context.json({ result, transition, monitor: updated }); }); export default monitorRoutes; diff --git a/test/checks.spec.ts b/test/checks.spec.ts index e7d4cc1..12d7eba 100644 --- a/test/checks.spec.ts +++ b/test/checks.spec.ts @@ -1,7 +1,7 @@ import { applyD1Migrations, type D1Migration } from 'cloudflare:test'; import { env } from 'cloudflare:workers'; import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; -import { runDueChecks } from '../src/worker/checks/run-due-checks'; +import { MAX_RETRY_ATTEMPTS_PER_RUN, runDueChecks } from '../src/worker/checks/run-due-checks'; async function clearMonitoringTables() { await env.DB.batch([ @@ -12,6 +12,7 @@ async function clearMonitoringTables() { env.DB.prepare('DELETE FROM incident_monitors'), env.DB.prepare('DELETE FROM incidents'), env.DB.prepare('DELETE FROM monitor_daily_stats'), + env.DB.prepare('DELETE FROM ai_settings'), env.DB.prepare('DELETE FROM notification_settings'), env.DB.prepare('DELETE FROM monitors'), ]); @@ -26,6 +27,9 @@ async function insertMonitor(overrides: Record = {}) { expected_status: 200, interval_seconds: 300, timeout_ms: 10_000, + retry_count: 0, + failure_threshold: 2, + consecutive_failures: 0, enabled: 1, last_checked_at: null, created_at: now, @@ -35,8 +39,9 @@ async function insertMonitor(overrides: Record = {}) { const result = await env.DB.prepare( ` INSERT INTO monitors - (name, url, method, expected_status, interval_seconds, timeout_ms, enabled, alerts_enabled, last_ok, last_checked_at, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?) + (name, url, method, expected_status, interval_seconds, timeout_ms, retry_count, failure_threshold, + consecutive_failures, enabled, alerts_enabled, last_ok, last_checked_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?) `, ) .bind( @@ -46,6 +51,9 @@ async function insertMonitor(overrides: Record = {}) { values.expected_status, values.interval_seconds, values.timeout_ms, + values.retry_count, + values.failure_threshold, + values.consecutive_failures, values.enabled, overrides.last_ok ?? null, values.last_checked_at, @@ -62,7 +70,10 @@ describe('scheduled monitor checks', () => { await applyD1Migrations(testEnv.DB, testEnv.TEST_MIGRATIONS); }); beforeEach(clearMonitoringTables); - afterEach(() => vi.unstubAllGlobals()); + afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); + }); it('records a successful check and updates the monitor snapshot', async () => { const id = await insertMonitor(); @@ -72,7 +83,7 @@ describe('scheduled monitor checks', () => { ); const summary = await runDueChecks(env); - expect(summary).toEqual({ checked: 1, up: 1, down: 0 }); + expect(summary).toEqual({ checked: 1, up: 1, down: 0, pending: 0, opened: 0, retries: 0 }); const monitor = await env.DB.prepare('SELECT last_ok, last_status_code, last_latency_ms, last_checked_at FROM monitors WHERE id = ?') .bind(id) @@ -87,39 +98,180 @@ describe('scheduled monitor checks', () => { expect(checkCount?.count).toBe(1); }); - it('records a mismatched status as down with a useful error', async () => { - const id = await insertMonitor(); + it('records a first mismatched status as pending without opening an incident', async () => { + const id = await insertMonitor({ last_ok: 1 }); + const fetchMock = vi.fn(async () => new Response(null, { status: 500 })); + vi.stubGlobal('fetch', fetchMock); + + const summary = await runDueChecks(env); + expect(summary).toEqual({ checked: 1, up: 0, down: 1, pending: 1, opened: 0, retries: 0 }); + + const monitor = await env.DB.prepare('SELECT last_ok, consecutive_failures, last_status_code, last_error FROM monitors WHERE id = ?') + .bind(id) + .first<{ last_ok: number; consecutive_failures: number; last_status_code: number; last_error: string }>(); + expect(monitor).toEqual({ + last_ok: 1, + consecutive_failures: 1, + last_status_code: 500, + last_error: 'Expected HTTP 200, received 500', + }); + const incident = await env.DB.prepare('SELECT COUNT(*) AS count FROM incident_monitors WHERE monitor_id = ?') + .bind(id) + .first<{ count: number }>(); + expect(incident?.count).toBe(0); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it('opens an incident once the failure threshold is reached', async () => { + const id = await insertMonitor({ last_ok: 1 }); vi.stubGlobal( 'fetch', vi.fn(async () => new Response(null, { status: 500 })), ); + await runDueChecks(env); + await env.DB.prepare('UPDATE monitors SET last_checked_at = NULL WHERE id = ?').bind(id).run(); const summary = await runDueChecks(env); - expect(summary).toEqual({ checked: 1, up: 0, down: 1 }); - const monitor = await env.DB.prepare('SELECT last_ok, last_status_code, last_error FROM monitors WHERE id = ?') + expect(summary.opened).toBe(1); + const monitor = await env.DB.prepare('SELECT last_ok, consecutive_failures FROM monitors WHERE id = ?') .bind(id) - .first<{ last_ok: number; last_status_code: number; last_error: string }>(); - expect(monitor).toEqual({ - last_ok: 0, - last_status_code: 500, - last_error: 'Expected HTTP 200, received 500', - }); - const incident = await env.DB.prepare( - 'SELECT im.monitor_id, i.resolved_at, i.start_status_code, i.start_error FROM incidents i JOIN incident_monitors im ON im.incident_id = i.id WHERE im.monitor_id = ?', - ) + .first<{ last_ok: number; consecutive_failures: number }>(); + expect(monitor).toEqual({ last_ok: 0, consecutive_failures: 2 }); + const incident = await env.DB.prepare('SELECT COUNT(*) AS count FROM incident_monitors WHERE monitor_id = ?') .bind(id) - .first<{ monitor_id: number; resolved_at: number | null; start_status_code: number; start_error: string }>(); - expect(incident).toEqual({ - monitor_id: id, - resolved_at: null, - start_status_code: 500, - start_error: 'Expected HTTP 200, received 500', - }); + .first<{ count: number }>(); + expect(incident?.count).toBe(1); + }); + + it('resets a pending failure when the next check succeeds', async () => { + const id = await insertMonitor({ last_ok: 1, consecutive_failures: 1 }); + vi.stubGlobal( + 'fetch', + vi.fn(async () => new Response(null, { status: 200 })), + ); + await runDueChecks(env); + const monitor = await env.DB.prepare('SELECT last_ok, consecutive_failures FROM monitors WHERE id = ?') + .bind(id) + .first<{ last_ok: number; consecutive_failures: number }>(); + expect(monitor).toEqual({ last_ok: 1, consecutive_failures: 0 }); + }); + + it('opens immediately when the threshold is one', async () => { + const id = await insertMonitor({ last_ok: 1, failure_threshold: 1 }); + vi.stubGlobal( + 'fetch', + vi.fn(async () => new Response(null, { status: 503 })), + ); + const summary = await runDueChecks(env); + expect(summary.opened).toBe(1); + const incident = await env.DB.prepare('SELECT COUNT(*) AS count FROM incident_monitors WHERE monitor_id = ?') + .bind(id) + .first<{ count: number }>(); + expect(incident?.count).toBe(1); + }); + + it('retries a failed attempt and persists only the recovered result', async () => { + const id = await insertMonitor({ last_ok: 1, retry_count: 1 }); + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response(null, { status: 500 })) + .mockResolvedValueOnce(new Response(null, { status: 200 })); + vi.stubGlobal('fetch', fetchMock); + const summary = await runDueChecks(env); + expect(summary).toMatchObject({ up: 1, retries: 1, pending: 0 }); + expect(fetchMock).toHaveBeenCalledTimes(2); + const checks = await env.DB.prepare('SELECT ok FROM checks WHERE monitor_id = ?').bind(id).all<{ ok: number }>(); + expect(checks.results).toEqual([{ ok: 1 }]); + }); + + it('records one failed check after exhausting retries', async () => { + const id = await insertMonitor({ last_ok: 1, retry_count: 2 }); + const fetchMock = vi.fn(async () => new Response(null, { status: 500 })); + vi.stubGlobal('fetch', fetchMock); + await runDueChecks(env); + expect(fetchMock).toHaveBeenCalledTimes(3); + const checks = await env.DB.prepare('SELECT ok FROM checks WHERE monitor_id = ?').bind(id).all<{ ok: number }>(); + expect(checks.results).toEqual([{ ok: 0 }]); + }); + + it('does not retry a monitor that is already confirmed down', async () => { + await insertMonitor({ last_ok: 0, consecutive_failures: 2, retry_count: 3 }); + const fetchMock = vi.fn(async () => new Response(null, { status: 500 })); + vi.stubGlobal('fetch', fetchMock); + const summary = await runDueChecks(env); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(summary.retries).toBe(0); + }); + + it('caps retry attempts across a scheduled run', async () => { + vi.useFakeTimers(); + for (let index = 0; index < 40; index += 1) { + await insertMonitor({ name: `Monitor ${index}`, url: `https://monitor-${index}.example.com`, last_ok: 1, retry_count: 3 }); + } + const fetchMock = vi.fn(async () => new Response(null, { status: 500 })); + vi.stubGlobal('fetch', fetchMock); + + const run = runDueChecks(env); + await vi.runAllTimersAsync(); + const summary = await run; + + expect(summary.retries).toBe(MAX_RETRY_ATTEMPTS_PER_RUN); + expect(fetchMock).toHaveBeenCalledTimes(40 + MAX_RETRY_ATTEMPTS_PER_RUN); + }); + + it('does not send a webhook or generate AI copy until a failure is confirmed', async () => { + const id = await insertMonitor({ last_ok: 1, retry_count: 0 }); + const now = Date.now(); + await env.DB.batch([ + env.DB.prepare( + "INSERT INTO notification_settings (id, webhook_url, webhook_enabled, created_at, updated_at) VALUES (1, 'https://hooks.example.test/events', 1, ?, ?)", + ).bind(now, now), + env.DB.prepare( + "INSERT INTO ai_settings (id, enabled, base_url, api_key, model, created_at, updated_at) VALUES (1, 1, 'https://ai.example.test/v1', 'secret', 'test-model', ?, ?)", + ).bind(now, now), + ]); + const webhookBodies: Array> = []; + let aiCalls = 0; + vi.stubGlobal( + 'fetch', + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = new URL(input.toString()); + if (url.hostname === 'hooks.example.test') { + webhookBodies.push(JSON.parse(String(init?.body)) as Record); + return new Response(null, { status: 204 }); + } + if (url.hostname === 'ai.example.test') { + aiCalls += 1; + return Response.json({ + choices: [ + { + message: { + content: 'Some visitors may be unable to use the service. The team has been alerted and restoration work is underway.', + }, + }, + ], + }); + } + return new Response(null, { status: 500 }); + }), + ); + + await runDueChecks(env); + expect(webhookBodies).toEqual([]); + expect(aiCalls).toBe(0); + expect((await env.DB.prepare('SELECT COUNT(*) AS count FROM incident_updates').first<{ count: number }>())?.count).toBe(0); + + await env.DB.prepare('UPDATE monitors SET last_checked_at = NULL WHERE id = ?').bind(id).run(); + await runDueChecks(env); + expect(webhookBodies).toHaveLength(1); + expect(webhookBodies[0]).toMatchObject({ event: 'down' }); + expect(aiCalls).toBe(1); + expect((await env.DB.prepare('SELECT COUNT(*) AS count FROM incident_updates').first<{ count: number }>())?.count).toBe(1); }); it('does not open duplicate incidents while a monitor stays down', async () => { - const id = await insertMonitor({ last_ok: 0 }); + const id = await insertMonitor({ last_ok: 0, consecutive_failures: 2 }); vi.stubGlobal( 'fetch', vi.fn(async () => new Response(null, { status: 503 })), @@ -132,8 +284,18 @@ describe('scheduled monitor checks', () => { }); it('links each auto incident to the correct monitor in one scheduled batch', async () => { - const firstId = await insertMonitor({ name: 'First', url: 'https://first.example.com' }); - const secondId = await insertMonitor({ name: 'Second', url: 'https://second.example.com' }); + const firstId = await insertMonitor({ + name: 'First', + url: 'https://first.example.com', + last_ok: 1, + failure_threshold: 1, + }); + const secondId = await insertMonitor({ + name: 'Second', + url: 'https://second.example.com', + last_ok: 1, + failure_threshold: 1, + }); vi.stubGlobal( 'fetch', vi.fn(async () => new Response(null, { status: 503 })), @@ -149,7 +311,7 @@ describe('scheduled monitor checks', () => { }); it('resolves the open incident on recovery', async () => { - const id = await insertMonitor({ last_ok: 0 }); + const id = await insertMonitor({ last_ok: 0, consecutive_failures: 2 }); const startedAt = Date.now() - 60_000; const inserted = await env.DB.prepare( "INSERT INTO incidents (status, impact, source, started_at, start_status_code, start_error, created_at, updated_at) VALUES ('investigating', 'major', 'auto', ?, 500, 'Down', ?, ?)", @@ -168,6 +330,10 @@ describe('scheduled monitor checks', () => { .first<{ resolved_at: number | null; duration_ms: number | null }>(); expect(incident?.resolved_at).toEqual(expect.any(Number)); expect(incident?.duration_ms).toBeGreaterThanOrEqual(60_000); + const monitor = await env.DB.prepare('SELECT last_ok, consecutive_failures FROM monitors WHERE id = ?') + .bind(id) + .first<{ last_ok: number; consecutive_failures: number }>(); + expect(monitor).toEqual({ last_ok: 1, consecutive_failures: 0 }); }); it('skips disabled and not-yet-due monitors', async () => { @@ -177,7 +343,7 @@ describe('scheduled monitor checks', () => { vi.stubGlobal('fetch', fetchMock); const summary = await runDueChecks(env); - expect(summary).toEqual({ checked: 0, up: 0, down: 0 }); + expect(summary).toEqual({ checked: 0, up: 0, down: 0, pending: 0, opened: 0, retries: 0 }); expect(fetchMock).not.toHaveBeenCalled(); }); }); diff --git a/test/maintenance.spec.ts b/test/maintenance.spec.ts index f0d6ba6..098deaa 100644 --- a/test/maintenance.spec.ts +++ b/test/maintenance.spec.ts @@ -36,7 +36,10 @@ async function resetDatabase() { async function insertMonitor(lastOk = 1) { const now = Date.now(); const result = await env.DB.prepare( - "INSERT INTO monitors (name, url, method, expected_status, interval_seconds, timeout_ms, enabled, alerts_enabled, last_ok, created_at, updated_at) VALUES ('API', 'https://example.com', 'GET', 200, 300, 10000, 1, 1, ?, ?, ?)", + `INSERT INTO monitors + (name, url, method, expected_status, interval_seconds, timeout_ms, retry_count, failure_threshold, + consecutive_failures, enabled, alerts_enabled, last_ok, created_at, updated_at) + VALUES ('API', 'https://example.com', 'GET', 200, 300, 10000, 0, 1, 0, 1, 1, ?, ?, ?)`, ) .bind(lastOk, now, now) .run(); @@ -78,18 +81,21 @@ describe('maintenance windows', () => { it('records failed probes as maintenance without incidents or monitor state changes', async () => { const id = await insertMonitor(1); + await env.DB.prepare('UPDATE monitors SET retry_count = 3, consecutive_failures = 1 WHERE id = ?').bind(id).run(); await insertActiveWindow(id); const fetchMock = vi.fn(async () => new Response(null, { status: 503 })); vi.stubGlobal('fetch', fetchMock); await runDueChecks(env); const check = await env.DB.prepare('SELECT maintenance, ok FROM checks WHERE monitor_id = ?').bind(id).first(); - const monitor = await env.DB.prepare('SELECT last_ok, last_status_code FROM monitors WHERE id = ?').bind(id).first(); + const monitor = await env.DB.prepare('SELECT last_ok, consecutive_failures, last_status_code FROM monitors WHERE id = ?') + .bind(id) + .first(); const incident = await env.DB.prepare('SELECT COUNT(*) AS count FROM incident_monitors WHERE monitor_id = ?') .bind(id) .first<{ count: number }>(); expect(check).toMatchObject({ maintenance: 1, ok: 0 }); - expect(monitor).toMatchObject({ last_ok: 1, last_status_code: 503 }); + expect(monitor).toMatchObject({ last_ok: 1, consecutive_failures: 1, last_status_code: 503 }); expect(incident?.count).toBe(0); expect(fetchMock).toHaveBeenCalledTimes(1); }); diff --git a/test/monitors.spec.ts b/test/monitors.spec.ts index 12dc7f4..9818664 100644 --- a/test/monitors.spec.ts +++ b/test/monitors.spec.ts @@ -144,10 +144,25 @@ describe('monitor API', () => { it('creates a valid monitor and returns it in the list', async () => { const cookie = await authenticatedCookie(); const response = await createMonitor(cookie); - const created = await response.json<{ monitor: { id: number; name: string; enabled: boolean; alertsEnabled: boolean } }>(); + const created = await response.json<{ + monitor: { + id: number; + name: string; + enabled: boolean; + alertsEnabled: boolean; + retryCount: number; + failureThreshold: number; + }; + }>(); expect(response.status).toBe(200); - expect(created.monitor).toMatchObject({ name: 'Example', enabled: true, alertsEnabled: true }); + expect(created.monitor).toMatchObject({ + name: 'Example', + enabled: true, + alertsEnabled: true, + retryCount: 1, + failureThreshold: 2, + }); const listResponse = await apiFetch('/api/monitors', 'GET', cookie); const list = await listResponse.json<{ monitors: Array<{ id: number; url: string }> }>(); @@ -159,6 +174,10 @@ describe('monitor API', () => { [{ url: 'file:///etc/passwd' }, 'Enter a valid http or https URL'], [{ intervalSeconds: 60 }, 'intervalSeconds must be an integer between 300 and 86400'], [{ expectedStatus: 99 }, 'expectedStatus must be an integer between 100 and 599'], + [{ retryCount: -1 }, 'retryCount must be an integer between 0 and 3'], + [{ retryCount: 4 }, 'retryCount must be an integer between 0 and 3'], + [{ failureThreshold: 0 }, 'failureThreshold must be an integer between 1 and 10'], + [{ failureThreshold: 11 }, 'failureThreshold must be an integer between 1 and 10'], ] as const)('rejects invalid monitor input %o', async (overrides, message) => { const response = await createMonitor(await authenticatedCookie(), overrides); expect(response.status).toBe(400); @@ -171,17 +190,92 @@ describe('monitor API', () => { const response = await apiFetch(`/api/monitors/${created.monitor.id}`, 'PATCH', cookie, { name: 'Renamed endpoint', enabled: false, + retryCount: 3, + failureThreshold: 4, }); - const body = await response.json<{ monitor: { name: string; url: string; enabled: boolean } }>(); + const body = await response.json<{ + monitor: { name: string; url: string; enabled: boolean; retryCount: number; failureThreshold: number }; + }>(); expect(response.status).toBe(200); expect(body.monitor).toMatchObject({ name: 'Renamed endpoint', url: 'https://example.com/health', enabled: false, + retryCount: 3, + failureThreshold: 4, }); }); + it('does not allow clients to set the internal failure counter', async () => { + const cookie = await authenticatedCookie(); + const created = await (await createMonitor(cookie)).json<{ monitor: { id: number } }>(); + const response = await apiFetch(`/api/monitors/${created.monitor.id}`, 'PATCH', cookie, { consecutiveFailures: 99 }); + expect(response.status).toBe(400); + expect(await response.json()).toEqual({ message: 'Provide at least one field to update' }); + }); + + it('returns a pending transition without opening an incident on the first manual failure', async () => { + const cookie = await authenticatedCookie(); + const created = await (await createMonitor(cookie, { retryCount: 0 })).json<{ monitor: { id: number } }>(); + vi.stubGlobal( + 'fetch', + vi.fn(async () => new Response(null, { status: 500 })), + ); + + const response = await apiFetch(`/api/monitors/${created.monitor.id}/check`, 'POST', cookie); + const body = await response.json<{ + transition: string; + result: { attempts: number }; + monitor: { lastOk: boolean | null; consecutiveFailures: number }; + }>(); + expect(body).toMatchObject({ + transition: 'pending', + result: { attempts: 1 }, + monitor: { lastOk: null, consecutiveFailures: 1 }, + }); + const incident = await env.DB.prepare('SELECT COUNT(*) AS count FROM incidents').first<{ count: number }>(); + expect(incident?.count).toBe(0); + }); + + it('opens an incident when a manual check reaches the confirmation threshold', async () => { + const cookie = await authenticatedCookie(); + const created = await ( + await createMonitor(cookie, { retryCount: 0, failureThreshold: 2 }) + ).json<{ + monitor: { id: number }; + }>(); + vi.stubGlobal( + 'fetch', + vi.fn(async () => new Response(null, { status: 500 })), + ); + + const first = await apiFetch(`/api/monitors/${created.monitor.id}/check`, 'POST', cookie); + const second = await apiFetch(`/api/monitors/${created.monitor.id}/check`, 'POST', cookie); + + expect((await first.json<{ transition: string }>()).transition).toBe('pending'); + expect((await second.json<{ transition: string }>()).transition).toBe('opened'); + const incident = await env.DB.prepare('SELECT COUNT(*) AS count FROM incident_monitors WHERE monitor_id = ?') + .bind(created.monitor.id) + .first<{ count: number }>(); + expect(incident?.count).toBe(1); + }); + + it('retries a manual check and returns the attempt count', async () => { + const cookie = await authenticatedCookie(); + const created = await (await createMonitor(cookie, { retryCount: 1 })).json<{ monitor: { id: number } }>(); + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response(null, { status: 500 })) + .mockResolvedValueOnce(new Response(null, { status: 200 })); + vi.stubGlobal('fetch', fetchMock); + + const response = await apiFetch(`/api/monitors/${created.monitor.id}/check`, 'POST', cookie); + const body = await response.json<{ transition: null; result: { ok: boolean; attempts: number } }>(); + expect(body).toMatchObject({ transition: null, result: { ok: true, attempts: 2 } }); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + it('returns detail, checks, incidents, and raw stats', async () => { const cookie = await authenticatedCookie(); const created = await (await createMonitor(cookie)).json<{ monitor: { id: number } }>(); diff --git a/test/status.spec.ts b/test/status.spec.ts index 4ac375c..8c0bc87 100644 --- a/test/status.spec.ts +++ b/test/status.spec.ts @@ -42,21 +42,25 @@ async function insertMonitor(input: { lastStatusCode?: number | null; lastLatencyMs?: number | null; lastError?: string | null; + consecutiveFailures?: number; + failureThreshold?: number; }) { const now = Date.now(); const result = await env.DB.prepare( ` INSERT INTO monitors ( name, url, method, expected_status, interval_seconds, timeout_ms, - enabled, alerts_enabled, last_ok, last_status_code, last_latency_ms, + enabled, alerts_enabled, failure_threshold, consecutive_failures, last_ok, last_status_code, last_latency_ms, last_error, last_checked_at, created_at, updated_at - ) VALUES (?, ?, 'GET', 200, 300, 10000, ?, 1, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (?, ?, 'GET', 200, 300, 10000, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?) `, ) .bind( input.name, input.url ?? `https://${input.name.toLowerCase()}.example.com/health`, input.enabled === false ? 0 : 1, + input.failureThreshold ?? 2, + input.consecutiveFailures ?? 0, input.lastOk === null || input.lastOk === undefined ? null : input.lastOk ? 1 : 0, input.lastStatusCode ?? null, input.lastLatencyMs ?? null, @@ -103,11 +107,36 @@ describe('public status API', () => { uptime90d: null, history: [], }); - for (const privateField of ['url', 'lastError', 'lastStatusCode', 'lastLatencyMs', 'method', 'timeoutMs']) { + for (const privateField of [ + 'url', + 'lastError', + 'lastStatusCode', + 'lastLatencyMs', + 'method', + 'timeoutMs', + 'retryCount', + 'failureThreshold', + 'consecutiveFailures', + ]) { expect(body.services[0]).not.toHaveProperty(privateField); } }); + it('keeps a service operational while failures are unconfirmed', async () => { + await insertMonitor({ + name: 'Flaky API', + lastOk: true, + consecutiveFailures: 1, + failureThreshold: 2, + lastStatusCode: 500, + lastError: 'Expected HTTP 200, received 500', + }); + + const body = await (await statusFetch()).json(); + expect(body.overall).toBe('operational'); + expect(body.services[0]).toMatchObject({ status: 'up', message: null }); + }); + it('excludes disabled monitors and reports degraded health for a partial outage', async () => { await insertMonitor({ name: 'Healthy', lastOk: true }); await insertMonitor({ name: 'Unavailable', lastOk: false }); diff --git a/test/webhook.spec.ts b/test/webhook.spec.ts index c6e7fe1..551b3dc 100644 --- a/test/webhook.spec.ts +++ b/test/webhook.spec.ts @@ -30,6 +30,9 @@ describe('incident webhooks', () => { timeoutMs: 10000, enabled: true, alertsEnabled: true, + retryCount: 1, + failureThreshold: 2, + consecutiveFailures: 0, lastOk: true, lastStatusCode: 200, lastLatencyMs: 30,