nezumi/uptime-monitoring
1
0
Fork 0
mirror of https://github.com/Nezumi-2711/uptime-monitoring.git synced 2026-09-22 13:48:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: improve detect error

Browse Source
This commit is contained in:
nezumi
2026-08-30 09:54:59 +07:00
parent 7a96644c25
commit 171433a857
21 changed files with 1558 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
migrations/0007_true_mauler.sql
+4
View File
@@ -0,0 +1,4 @@
ALTER TABLE `monitors` ADD `retry_count` integer DEFAULT 1 NOT NULL;--> statement-breakpoint
ALTER TABLE `monitors` ADD `failure_threshold` integer DEFAULT 2 NOT NULL;--> statement-breakpoint
ALTER TABLE `monitors` ADD `consecutive_failures` integer DEFAULT 0 NOT NULL;--> statement-breakpoint
UPDATE `monitors` SET `consecutive_failures` = `failure_threshold` WHERE `last_ok` = 0;
migrations/meta/0007_snapshot.json
+993
View File
@@ -0,0 +1,993 @@
{
"version": "6",
"dialect": "sqlite",
"id": "3e14e2ff-5a06-45c1-83c8-91d471c8cdf7",
"prevId": "6e3277e0-8fe4-4619-8cb4-52c7c5185416",
"tables": {
"admin_credentials": {
"name": "admin_credentials",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"password_hash": {
"name": "password_hash",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"updated_at": {
"name": "updated_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"ai_settings": {
"name": "ai_settings",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"enabled": {
"name": "enabled",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": false
},
"base_url": {
"name": "base_url",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"api_key": {
"name": "api_key",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"model": {
"name": "model",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"updated_at": {
"name": "updated_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"checks": {
"name": "checks",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": true
},
"monitor_id": {
"name": "monitor_id",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"ok": {
"name": "ok",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"status_code": {
"name": "status_code",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"latency_ms": {
"name": "latency_ms",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"error": {
"name": "error",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"checked_at": {
"name": "checked_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"maintenance": {
"name": "maintenance",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": false
}
},
"indexes": {
"checks_monitor_id_checked_at_idx": {
"name": "checks_monitor_id_checked_at_idx",
"columns": [
"monitor_id",
"checked_at"
],
"isUnique": false
}
},
"foreignKeys": {
"checks_monitor_id_monitors_id_fk": {
"name": "checks_monitor_id_monitors_id_fk",
"tableFrom": "checks",
"tableTo": "monitors",
"columnsFrom": [
"monitor_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"incident_monitors": {
"name": "incident_monitors",
"columns": {
"incident_id": {
"name": "incident_id",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"monitor_id": {
"name": "monitor_id",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {
"incident_monitors_monitor_id_idx": {
"name": "incident_monitors_monitor_id_idx",
"columns": [
"monitor_id"
],
"isUnique": false
}
},
"foreignKeys": {
"incident_monitors_incident_id_incidents_id_fk": {
"name": "incident_monitors_incident_id_incidents_id_fk",
"tableFrom": "incident_monitors",
"tableTo": "incidents",
"columnsFrom": [
"incident_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
},
"incident_monitors_monitor_id_monitors_id_fk": {
"name": "incident_monitors_monitor_id_monitors_id_fk",
"tableFrom": "incident_monitors",
"tableTo": "monitors",
"columnsFrom": [
"monitor_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {
"incident_monitors_incident_id_monitor_id_pk": {
"columns": [
"incident_id",
"monitor_id"
],
"name": "incident_monitors_incident_id_monitor_id_pk"
}
},
"uniqueConstraints": {},
"checkConstraints": {}
},
"incident_updates": {
"name": "incident_updates",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": true
},
"incident_id": {
"name": "incident_id",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"status": {
"name": "status",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"body": {
"name": "body",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"note": {
"name": "note",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"source": {
"name": "source",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "'manual'"
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {
"incident_updates_incident_id_created_at_idx": {
"name": "incident_updates_incident_id_created_at_idx",
"columns": [
"incident_id",
"created_at"
],
"isUnique": false
}
},
"foreignKeys": {
"incident_updates_incident_id_incidents_id_fk": {
"name": "incident_updates_incident_id_incidents_id_fk",
"tableFrom": "incident_updates",
"tableTo": "incidents",
"columnsFrom": [
"incident_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"incidents": {
"name": "incidents",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": true
},
"title": {
"name": "title",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"status": {
"name": "status",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "'investigating'"
},
"impact": {
"name": "impact",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "'major'"
},
"source": {
"name": "source",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "'auto'"
},
"started_at": {
"name": "started_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"resolved_at": {
"name": "resolved_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"start_status_code": {
"name": "start_status_code",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"start_error": {
"name": "start_error",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"duration_ms": {
"name": "duration_ms",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"updated_at": {
"name": "updated_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {
"incidents_started_at_idx": {
"name": "incidents_started_at_idx",
"columns": [
"started_at"
],
"isUnique": false
},
"incidents_resolved_at_idx": {
"name": "incidents_resolved_at_idx",
"columns": [
"resolved_at"
],
"isUnique": false
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"login_attempts": {
"name": "login_attempts",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": true
},
"ip_address": {
"name": "ip_address",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"attempted_at": {
"name": "attempted_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {
"login_attempts_ip_attempted_at_idx": {
"name": "login_attempts_ip_attempted_at_idx",
"columns": [
"ip_address",
"attempted_at"
],
"isUnique": false
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"maintenance_window_monitors": {
"name": "maintenance_window_monitors",
"columns": {
"window_id": {
"name": "window_id",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"monitor_id": {
"name": "monitor_id",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {
"maintenance_window_monitors_monitor_id_idx": {
"name": "maintenance_window_monitors_monitor_id_idx",
"columns": [
"monitor_id"
],
"isUnique": false
}
},
"foreignKeys": {
"maintenance_window_monitors_window_id_maintenance_windows_id_fk": {
"name": "maintenance_window_monitors_window_id_maintenance_windows_id_fk",
"tableFrom": "maintenance_window_monitors",
"tableTo": "maintenance_windows",
"columnsFrom": [
"window_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
},
"maintenance_window_monitors_monitor_id_monitors_id_fk": {
"name": "maintenance_window_monitors_monitor_id_monitors_id_fk",
"tableFrom": "maintenance_window_monitors",
"tableTo": "monitors",
"columnsFrom": [
"monitor_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {
"maintenance_window_monitors_window_id_monitor_id_pk": {
"columns": [
"window_id",
"monitor_id"
],
"name": "maintenance_window_monitors_window_id_monitor_id_pk"
}
},
"uniqueConstraints": {},
"checkConstraints": {}
},
"maintenance_windows": {
"name": "maintenance_windows",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": true
},
"name": {
"name": "name",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"start_minute": {
"name": "start_minute",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"duration_minutes": {
"name": "duration_minutes",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"timezone": {
"name": "timezone",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "'UTC'"
},
"enabled": {
"name": "enabled",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": true
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"updated_at": {
"name": "updated_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {
"maintenance_windows_enabled_idx": {
"name": "maintenance_windows_enabled_idx",
"columns": [
"enabled"
],
"isUnique": false
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"monitor_daily_stats": {
"name": "monitor_daily_stats",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": true
},
"monitor_id": {
"name": "monitor_id",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"day": {
"name": "day",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"total_checks": {
"name": "total_checks",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"up_checks": {
"name": "up_checks",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"avg_latency_ms": {
"name": "avg_latency_ms",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"min_latency_ms": {
"name": "min_latency_ms",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"max_latency_ms": {
"name": "max_latency_ms",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
}
},
"indexes": {
"monitor_daily_stats_monitor_id_day_uidx": {
"name": "monitor_daily_stats_monitor_id_day_uidx",
"columns": [
"monitor_id",
"day"
],
"isUnique": true
}
},
"foreignKeys": {
"monitor_daily_stats_monitor_id_monitors_id_fk": {
"name": "monitor_daily_stats_monitor_id_monitors_id_fk",
"tableFrom": "monitor_daily_stats",
"tableTo": "monitors",
"columnsFrom": [
"monitor_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"monitors": {
"name": "monitors",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": true
},
"name": {
"name": "name",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"url": {
"name": "url",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"method": {
"name": "method",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "'GET'"
},
"expected_status": {
"name": "expected_status",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 200
},
"interval_seconds": {
"name": "interval_seconds",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 300
},
"timeout_ms": {
"name": "timeout_ms",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 10000
},
"enabled": {
"name": "enabled",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": true
},
"alerts_enabled": {
"name": "alerts_enabled",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": true
},
"retry_count": {
"name": "retry_count",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 1
},
"failure_threshold": {
"name": "failure_threshold",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 2
},
"consecutive_failures": {
"name": "consecutive_failures",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"last_ok": {
"name": "last_ok",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"last_status_code": {
"name": "last_status_code",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"last_latency_ms": {
"name": "last_latency_ms",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"last_error": {
"name": "last_error",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"last_checked_at": {
"name": "last_checked_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"updated_at": {
"name": "updated_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {
"monitors_enabled_last_checked_at_idx": {
"name": "monitors_enabled_last_checked_at_idx",
"columns": [
"enabled",
"last_checked_at"
],
"isUnique": false
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"notification_settings": {
"name": "notification_settings",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"webhook_url": {
"name": "webhook_url",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"webhook_enabled": {
"name": "webhook_enabled",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"updated_at": {
"name": "updated_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"sessions": {
"name": "sessions",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"expires_at": {
"name": "expires_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"user_agent": {
"name": "user_agent",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
}
},
"indexes": {
"sessions_expires_at_idx": {
"name": "sessions_expires_at_idx",
"columns": [
"expires_at"
],
"isUnique": false
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
}
},
"views": {},
"enums": {},
"_meta": {
"schemas": {},
"tables": {},
"columns": {}
},
"internal": {
"indexes": {}
}
}
migrations/meta/_journal.json
+7
View File
@@ -50,6 +50,13 @@
"when": 1788012122846,
"tag": "0006_gifted_deathstrike",
"breakpoints": true
},
{
"idx": 7,
"version": "6",
"when": 1788057318682,
"tag": "0007_true_mauler",
"breakpoints": true
}
]
}
src/client/api/monitors.ts
+15 -2
View File
@@ -12,6 +12,9 @@ export type Monitor = {
timeoutMs: number;
enabled: boolean;
alertsEnabled: boolean;
retryCount: number;
failureThreshold: number;
consecutiveFailures: number;
lastOk: boolean | null;
lastStatusCode: number | null;
lastLatencyMs: number | null;
@@ -28,6 +31,8 @@ export type MonitorInput = {
expectedStatus: number;
intervalSeconds: number;
timeoutMs: number;
retryCount?: number;
failureThreshold?: number;
enabled?: boolean;
alertsEnabled?: boolean;
};
@@ -37,14 +42,22 @@ export type CheckResult = {
statusCode: number | null;
latencyMs: number;
error: string | null;
attempts: number;
};
export type Check = CheckResult & {
export type Check = {
id: number;
monitorId: number;
ok: boolean;
statusCode: number | null;
latencyMs: number;
error: string | null;
checkedAt: string;
maintenance: boolean;
};
export type CheckTransition = 'opened' | 'pending' | 'cleared' | 'resolved' | null;
export type Incident = {
id: number;
title: string | null;
@@ -109,5 +122,5 @@ export function deleteMonitor(id: number) {
}
export function runMonitorCheck(id: number) {
return postJson<{ result: CheckResult; monitor: Monitor }>(`/api/monitors/${id}/check`);
return postJson<{ result: CheckResult; transition: CheckTransition; monitor: Monitor }>(`/api/monitors/${id}/check`);
}
src/client/components/dashboard/DashboardOverview.tsx
+2 -1
View File
@@ -12,6 +12,7 @@ export function DashboardOverview({ onAddMonitor }: DashboardOverviewProps) {
const monitors = monitorsQuery.data?.monitors ?? [];
const up = monitors.filter((monitor) => monitor.lastOk === true).length;
const down = monitors.filter((monitor) => monitor.lastOk === false).length;
const degrading = monitors.filter((monitor) => monitor.lastOk !== false && monitor.consecutiveFailures > 0).length;
return (
<>
@@ -42,7 +43,7 @@ export function DashboardOverview({ onAddMonitor }: DashboardOverviewProps) {
<Card className="metric-card">
<p>Currently down</p>
<strong>{down}</strong>
<span>Needs attention</span>
<span>{degrading > 0 ? `${degrading} degrading` : 'Needs attention'}</span>
</Card>
</section>
</>
src/client/components/dashboard/MonitorFormDialog.tsx
+40 -1
View File
@@ -14,6 +14,8 @@ export const DEFAULT_MONITOR_INPUT: MonitorInput = {
expectedStatus: 200,
intervalSeconds: 300,
timeoutMs: 10_000,
retryCount: 1,
failureThreshold: 2,
enabled: true,
};
@@ -43,6 +45,8 @@ function monitorInput(monitor: Monitor | null): MonitorInput {
expectedStatus: monitor.expectedStatus,
intervalSeconds: monitor.intervalSeconds,
timeoutMs: monitor.timeoutMs,
retryCount: monitor.retryCount,
failureThreshold: monitor.failureThreshold,
enabled: monitor.enabled,
alertsEnabled: monitor.alertsEnabled,
};
@@ -87,7 +91,9 @@ export function MonitorFormDialog({ editing, onClose }: MonitorFormDialogProps)
<DialogHeader>
<p className="overline">Configuration</p>
<DialogTitle>{editing ? `Edit ${editing.name}` : 'Add a monitor'}</DialogTitle>
<DialogDescription>Checks run on the configured schedule, with a minimum interval of five minutes.</DialogDescription>
<DialogDescription>
Checks run at least every five minutes. Failures are retried immediately and must repeat before an incident is published.
</DialogDescription>
</DialogHeader>
<form className="monitor-form" onSubmit={handleSubmit}>
<label className="field field-name" htmlFor="monitor-name">
@@ -166,6 +172,39 @@ export function MonitorFormDialog({ editing, onClose }: MonitorFormDialogProps)
required
/>
</label>
<div className="field">
<span id="monitor-retries-label">Retries after a failure</span>
<Select value={String(form.retryCount ?? 1)} onValueChange={(value) => setForm({ ...form, retryCount: Number(value) })}>
<SelectTrigger aria-labelledby="monitor-retries-label">
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value="0">No retry</SelectItem>
{[1, 2, 3].map((count) => (
<SelectItem key={count} value={String(count)}>{`${count} ${count === 1 ? 'retry' : 'retries'}`}</SelectItem>
))}
</SelectContent>
</Select>
<small>Retried immediately, so a sub-second blip never reaches the status page.</small>
</div>
<div className="field">
<span id="monitor-threshold-label">Confirmations before alerting</span>
<Select
value={String(form.failureThreshold ?? 2)}
onValueChange={(value) => setForm({ ...form, failureThreshold: Number(value) })}
>
<SelectTrigger aria-labelledby="monitor-threshold-label">
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value="1">1 — alert immediately</SelectItem>
{Array.from({ length: 9 }, (_, index) => index + 2).map((count) => (
<SelectItem key={count} value={String(count)}>{`${count} consecutive failed checks`}</SelectItem>
))}
</SelectContent>
</Select>
<small>Confirmed after about {Math.round(((form.failureThreshold ?? 2) * form.intervalSeconds) / 60)} minutes.</small>
</div>
<div className="toggle-field">
<Switch id="monitor-enabled" checked={form.enabled ?? true} onCheckedChange={(enabled) => setForm({ ...form, enabled })} />
<label htmlFor="monitor-enabled">Enable scheduled checks</label>
src/client/components/dashboard/MonitorListPanel.tsx
+9 -10
View File
@@ -1,9 +1,10 @@
import { useState } from 'react';
import { AlertTriangle, ArrowRight, Database, History, Pencil, Power, PowerOff, RefreshCw, Trash2 } from 'lucide-react';
import { Badge, type BadgeVariant } from '@/components/ui/badge';
import { Badge } from '@/components/ui/badge';
import { Button } from '@/components/ui/button';
import { Empty, EmptyContent, EmptyDescription, EmptyMedia, EmptyTitle } from '@/components/ui/empty';
import type { Monitor } from '../../api/monitors';
import { monitorState } from '../../lib/monitor-status';
import { navigate } from '../../lib/router';
import { useDeleteMonitorMutation, useMonitorsQuery, useRunCheckMutation, useUpdateMonitorMutation } from '../../queries/monitors';
import { SiteIcon } from '../SiteIcon';
@@ -23,12 +24,6 @@ function formatCheckedAt(value: string | null) {
}).format(new Date(value));
}
function monitorStatus(monitor: Monitor): { label: string; className: BadgeVariant } {
if (monitor.lastOk === true) return { label: 'Up', className: 'online' };
if (monitor.lastOk === false) return { label: 'Down', className: 'offline' };
return { label: 'Not checked', className: 'checking' };
}
function errorMessage(error: unknown, fallback: string) {
return error instanceof Error ? error.message : fallback;
}
@@ -119,7 +114,7 @@ export function MonitorListPanel({ formOpen, onAddMonitor, onEdit }: MonitorList
<span>Actions</span>
</div>
{monitors.map((monitor) => {
const status = monitorStatus(monitor);
const status = monitorState(monitor);
const checking = checkMutation.isPending && checkMutation.variables === monitor.id;
const deleting = deleteMutation.isPending && deleteMutation.variables === monitor.id;
const toggling = updateMutation.isPending && updateMutation.variables?.id === monitor.id;
@@ -145,12 +140,16 @@ export function MonitorListPanel({ formOpen, onAddMonitor, onEdit }: MonitorList
</div>
</div>
<div className="monitor-result">
<Badge variant={checking ? 'checking' : status.className}>{checking ? 'Checking' : status.label}</Badge>
<Badge variant={checking ? 'checking' : status.variant} title={status.detail ?? undefined}>
{checking ? 'Checking' : status.label}
</Badge>
<code>
{monitor.lastStatusCode === null ? '—' : `HTTP ${monitor.lastStatusCode}`} ·{' '}
{monitor.lastLatencyMs === null ? '—' : `${monitor.lastLatencyMs} ms`}
</code>
<small title={monitor.lastError ?? undefined}>{monitor.lastError ?? formatCheckedAt(monitor.lastCheckedAt)}</small>
<small title={status.detail ?? monitor.lastError ?? undefined}>
{status.detail ?? monitor.lastError ?? formatCheckedAt(monitor.lastCheckedAt)}
</small>
</div>
<div className="row-actions" aria-label={`Actions for ${monitor.name}`}>
<Button
src/client/lib/monitor-status.ts
+14
View File
@@ -0,0 +1,14 @@
import type { Monitor } from '../api/monitors';
export function monitorState(monitor: Pick<Monitor, 'lastOk' | 'consecutiveFailures' | 'failureThreshold'>) {
if (monitor.lastOk === false) return { label: 'Down', variant: 'offline' as const, detail: null };
if (monitor.consecutiveFailures > 0) {
return {
label: 'Degrading',
variant: 'pending' as const,
detail: `${monitor.consecutiveFailures} of ${monitor.failureThreshold} failed checks — an incident opens if the next check fails`,
};
}
if (monitor.lastOk === true) return { label: 'Up', variant: 'online' as const, detail: null };
return { label: 'Not checked', variant: 'checking' as const, detail: null };
}
src/client/pages/MonitorDetailPage.tsx
+7 -10
View File
@@ -1,6 +1,6 @@
import { useState } from 'react';
import { ArrowLeft, BellOff, CheckCircle2, Clock3, ExternalLink, Pencil, Power, PowerOff, RefreshCw, Trash2 } from 'lucide-react';
import { Badge, type BadgeVariant } from '@/components/ui/badge';
import { Badge } from '@/components/ui/badge';
import { Button } from '@/components/ui/button';
import { Card } from '@/components/ui/card';
import { Empty, EmptyContent, EmptyDescription, EmptyTitle } from '@/components/ui/empty';
@@ -10,6 +10,7 @@ import { LatencySparkline } from '../components/charts/LatencySparkline';
import { UptimeBar } from '../components/charts/UptimeBar';
import { DeleteMonitorDialog } from '../components/dashboard/DeleteMonitorDialog';
import { INTERVAL_OPTIONS, MonitorFormDialog } from '../components/dashboard/MonitorFormDialog';
import { monitorState } from '../lib/monitor-status';
import { navigate } from '../lib/router';
import {
useDeleteMonitorMutation,
@@ -33,12 +34,6 @@ function formatDuration(ms: number | null, startedAt: string) {
return `${Math.round(duration / 86_400_000)} days`;
}
function statusDetails(lastOk: boolean | null): { label: string; className: BadgeVariant } {
if (lastOk === true) return { label: 'Operational', className: 'online' };
if (lastOk === false) return { label: 'Down', className: 'offline' };
return { label: 'Awaiting first check', className: 'checking' };
}
function formatInterval(seconds: number) {
return INTERVAL_OPTIONS.find((option) => Number(option.value) === seconds)?.label ?? `${Math.round(seconds / 60)} min`;
}
@@ -56,7 +51,6 @@ export function MonitorDetailPage({ id }: { id: number }) {
const monitor = monitorQuery.data?.monitor;
const checks = checksQuery.data?.checks ?? [];
const incidents = incidentsQuery.data?.incidents ?? [];
const status = statusDetails(monitor?.lastOk ?? null);
const openIncident = incidents.find((incident) => incident.resolvedAt === null);
if (monitorQuery.isPending)
@@ -80,6 +74,7 @@ export function MonitorDetailPage({ id }: { id: number }) {
</Empty>
</div>
);
const status = monitorState(monitor);
return (
<div className="dashboard-shell">
@@ -97,7 +92,7 @@ export function MonitorDetailPage({ id }: { id: number }) {
</Button>
<section className="detail-hero">
<div className="detail-title">
<span className={`status-orb ${status.className}`} />
<span className={`status-orb ${status.variant}`} />
<div>
<p className="overline">Monitor #{monitor.id}</p>
<h1>{monitor.name}</h1>
@@ -108,7 +103,9 @@ export function MonitorDetailPage({ id }: { id: number }) {
</div>
</div>
<div className="detail-actions">
<Badge variant={status.className}>{status.label}</Badge>
<Badge variant={status.variant} title={status.detail ?? undefined}>
{status.label}
</Badge>
{!monitor.alertsEnabled && (
<span className="muted-alert">
<BellOff /> Alerts muted
src/client/styles.css
+11 -1
View File
@@ -412,7 +412,7 @@ button {
grid-column: span 3;
}
.form-actions.compact-actions {
grid-column: span 2;
grid-column: 3 / -1;
}
.form-error {
grid-column: 1 / -1;
@@ -630,6 +630,12 @@ button {
.row-status.offline {
color: #ae3d3d;
}
.row-status.pending {
color: #8a5a1f;
}
.row-status.pending i {
background: #e0913f;
}
.row-status.checking {
color: #8b7722;
}
@@ -933,6 +939,10 @@ button {
border-color: #c5f1df;
background: var(--primary-deep);
}
.status-orb.pending {
border-color: #f6e2c8;
background: #d8892f;
}
.status-orb.checking {
border-color: #eee5bd;
background: #c6aa38;
src/components/ui/badge.tsx
+1
View File
@@ -8,6 +8,7 @@ const badgeVariants = cva('inline-flex items-center gap-1.75 text-[12px] font-me
variant: {
online: 'text-[#16885b] [&>i]:bg-(--primary-deep) [&>i]:shadow-[0_0_0_3px_rgb(62_207_142/0.12)]',
offline: 'text-[#ae3d3d] [&>i]:bg-[#d05a5a]',
pending: 'text-[#8a5a1f] [&>i]:bg-[#e0913f] [&>i]:shadow-[0_0_0_3px_rgb(224_145_63/0.14)]',
checking: 'text-[#8b7722] [&>i]:animate-[blink_1.1s_ease-in-out_infinite] [&>i]:bg-[#d7bd53]',
maintenance: 'text-[#3f6fb8] [&>i]:bg-[#5a8fd0]',
},
src/worker/checks/persist-result.ts
+40 -10
View File
@@ -4,7 +4,9 @@ import { checks, incidentMonitors, incidents, incidentUpdates, monitors } from '
import { RECOVERY_UPDATE_BODY } from '../ai/fallback-message';
import type { CheckResult, Monitor } from './run-check';
export type IncidentTransition = 'opened' | 'resolved' | null;
/** Only opened and resolved transitions are allowed to send alerts. */
export type CheckTransition = 'opened' | 'pending' | 'cleared' | 'resolved' | null;
export type AlertTransition = Extract<CheckTransition, 'opened' | 'resolved'>;
type BatchStatement = Parameters<Database['batch']>[0][number];
@@ -19,10 +21,39 @@ export function buildResultStatements(db: Database, monitor: Monitor, result: Ch
checkedAt,
maintenance,
}),
];
if (maintenance) {
statements.push(
db
.update(monitors)
.set({
lastStatusCode: result.statusCode,
lastLatencyMs: result.latencyMs,
lastError: result.error,
lastCheckedAt: checkedAt,
updatedAt: checkedAt,
})
.where(eq(monitors.id, monitor.id)),
);
return { statements, transition: null as CheckTransition, consecutiveFailures: monitor.consecutiveFailures };
}
const threshold = Math.max(1, monitor.failureThreshold);
const previousFailures = monitor.consecutiveFailures;
// This deliberately uses the monitor snapshot. Concurrent manual and scheduled checks may lose one increment,
// which delays confirmation by one check but cannot publish a false incident.
const nextFailures = result.ok ? 0 : previousFailures + 1;
const wasDown = monitor.lastOk === false;
const isDown = !result.ok && nextFailures >= threshold;
const confirmed = result.ok ? true : isDown ? false : undefined;
statements.push(
db
.update(monitors)
.set({
...(maintenance ? {} : { lastOk: result.ok }),
...(confirmed === undefined ? {} : { lastOk: confirmed }),
consecutiveFailures: nextFailures,
lastStatusCode: result.statusCode,
lastLatencyMs: result.latencyMs,
lastError: result.error,
@@ -30,12 +61,10 @@ export function buildResultStatements(db: Database, monitor: Monitor, result: Ch
updatedAt: checkedAt,
})
.where(eq(monitors.id, monitor.id)),
];
);
if (maintenance) return { statements, transition: null };
let transition: IncidentTransition = null;
if (monitor.lastOk !== false && !result.ok) {
let transition: CheckTransition = null;
if (!wasDown && isDown) {
statements.push(
db.insert(incidents).values({
status: 'investigating',
@@ -50,7 +79,7 @@ export function buildResultStatements(db: Database, monitor: Monitor, result: Ch
db.insert(incidentMonitors).values({ incidentId: sql`last_insert_rowid()`, monitorId: monitor.id }),
);
transition = 'opened';
} else if (monitor.lastOk === false && result.ok) {
} else if (wasDown && result.ok) {
const openIncidentIds = db
.select({ id: incidentMonitors.incidentId })
.from(incidentMonitors)
@@ -82,7 +111,8 @@ export function buildResultStatements(db: Database, monitor: Monitor, result: Ch
.where(and(eq(incidents.source, 'auto'), isNull(incidents.resolvedAt), inArray(incidents.id, openIncidentIds))),
);
transition = 'resolved';
}
} else if (!wasDown && !result.ok) transition = 'pending';
else if (!wasDown && result.ok && previousFailures > 0) transition = 'cleared';
return { statements, transition };
return { statements, transition, consecutiveFailures: nextFailures };
}
src/worker/checks/run-check.ts
+23
View File
@@ -35,3 +35,26 @@ export async function runCheck(monitor: Monitor): Promise<CheckResult> {
};
}
}
export const MAX_RETRY_COUNT = 3;
const RETRY_BACKOFF_MS = [400, 1_200, 2_500] as const;
export type CheckAttemptResult = CheckResult & { attempts: number };
export type RetryBudget = { remaining: number; deadline: number };
export async function runCheckWithRetries(monitor: Monitor, budget?: RetryBudget): Promise<CheckAttemptResult> {
let result = await runCheck(monitor);
if (result.ok) return { ...result, attempts: 1 };
const allowed = Math.min(Math.max(0, monitor.retryCount), MAX_RETRY_COUNT);
let attempts = 1;
for (let retry = 0; retry < allowed; retry += 1) {
if (budget && (budget.remaining <= 0 || Date.now() >= budget.deadline)) break;
if (budget) budget.remaining -= 1;
await new Promise((resolve) => setTimeout(resolve, RETRY_BACKOFF_MS[Math.min(retry, RETRY_BACKOFF_MS.length - 1)]));
result = await runCheck(monitor);
attempts += 1;
if (result.ok) break;
}
return { ...result, attempts };
}
src/worker/checks/run-due-checks.ts
+30 -14
View File
@@ -4,17 +4,22 @@ import { getDb } from '../db/client';
import { monitors } from '../db/schema';
import { loadActiveMaintenance } from '../maintenance/windows';
import { sendIncidentAlert } from '../notifications/webhook';
import { buildResultStatements } from './persist-result';
import { runCheck } from './run-check';
import { type AlertTransition, buildResultStatements } from './persist-result';
import { runCheck, runCheckWithRetries, type RetryBudget } from './run-check';
const MAX_MONITORS_PER_RUN = 40;
const MAX_AI_MESSAGES_PER_RUN = 10;
const CONCURRENCY = 10;
export const MAX_RETRY_ATTEMPTS_PER_RUN = 60;
const RETRY_DEADLINE_MS = 90_000;
export type DueCheckSummary = {
checked: number;
up: number;
down: number;
pending: number;
opened: number;
retries: number;
};
export async function runDueChecks(env: Env, ctx?: Pick<ExecutionContext, 'waitUntil'>): Promise<DueCheckSummary> {
@@ -32,32 +37,35 @@ export async function runDueChecks(env: Env, ctx?: Pick<ExecutionContext, 'waitU
.orderBy(sql`${monitors.lastCheckedAt} ASC NULLS FIRST`)
.limit(MAX_MONITORS_PER_RUN);
if (due.length === 0) return { checked: 0, up: 0, down: 0 };
if (due.length === 0) return { checked: 0, up: 0, down: 0, pending: 0, opened: 0, retries: 0 };
const activeMaintenance = await loadActiveMaintenance(db, new Date());
const budget: RetryBudget = { remaining: MAX_RETRY_ATTEMPTS_PER_RUN, deadline: Date.now() + RETRY_DEADLINE_MS };
const completed: Array<{
monitor: (typeof due)[number];
result: Awaited<ReturnType<typeof runCheck>>;
result: Awaited<ReturnType<typeof runCheckWithRetries>>;
checkedAt: Date;
maintenance: boolean;
}> = [];
for (let offset = 0; offset < due.length; offset += CONCURRENCY) {
const batch = due.slice(offset, offset + CONCURRENCY);
const results = await Promise.all(
batch.map(async (monitor) => ({
monitor,
result: await runCheck(monitor),
checkedAt: new Date(),
})),
batch.map(async (monitor) => {
const maintenance = activeMaintenance.has(monitor.id);
const retryable = !maintenance && monitor.retryCount > 0 && monitor.lastOk !== false;
const result = retryable ? await runCheckWithRetries(monitor, budget) : { ...(await runCheck(monitor)), attempts: 1 };
return { monitor, result, checkedAt: new Date(), maintenance };
}),
);
completed.push(...results);
}
const persisted = completed.map(({ monitor, result, checkedAt }) => ({
const persisted = completed.map(({ monitor, result, checkedAt, maintenance }) => ({
monitor,
result,
checkedAt,
...buildResultStatements(db, monitor, result, checkedAt, activeMaintenance.has(monitor.id)),
...buildResultStatements(db, monitor, result, checkedAt, maintenance),
}));
const statements = persisted.flatMap((item) => item.statements);
@@ -65,11 +73,12 @@ export async function runDueChecks(env: Env, ctx?: Pick<ExecutionContext, 'waitU
let aiMessagesQueued = 0;
const notifications = persisted.flatMap((item) => {
if (item.transition === null) return [];
if (item.transition !== 'opened' && item.transition !== 'resolved') return [];
const kind: AlertTransition = item.transition;
const work: Promise<unknown>[] = [
sendIncidentAlert(env, {
monitor: item.monitor,
kind: item.transition,
kind,
result: item.result,
at: item.checkedAt,
}),
@@ -87,5 +96,12 @@ export async function runDueChecks(env: Env, ctx?: Pick<ExecutionContext, 'waitU
}
const up = completed.reduce((count, item) => count + Number(item.result.ok), 0);
return { checked: completed.length, up, down: completed.length - up };
return {
checked: completed.length,
up,
down: completed.length - up,
pending: persisted.reduce((count, item) => count + Number(item.transition === 'pending'), 0),
opened: persisted.reduce((count, item) => count + Number(item.transition === 'opened'), 0),
retries: completed.reduce((count, item) => count + item.result.attempts - 1, 0),
};
}
src/worker/db/schema.ts
+7
View File
@@ -58,6 +58,13 @@ export const monitors = sqliteTable(
timeoutMs: integer('timeout_ms').notNull().default(10_000),
enabled: integer('enabled', { mode: 'boolean' }).notNull().default(true),
alertsEnabled: integer('alerts_enabled', { mode: 'boolean' }).notNull().default(true),
/** Number of immediate retries after a failed attempt. Zero disables retries. */
retryCount: integer('retry_count').notNull().default(1),
/** Consecutive failed checks required before confirming an outage. */
failureThreshold: integer('failure_threshold').notNull().default(2),
/** Failures since the last successful check. Maintenance checks do not change this value. */
consecutiveFailures: integer('consecutive_failures').notNull().default(0),
/** Confirmed state, not the raw latest result. */
lastOk: integer('last_ok', { mode: 'boolean' }),
lastStatusCode: integer('last_status_code'),
lastLatencyMs: integer('last_latency_ms'),
src/worker/routes/monitors.ts
+18 -4
View File
@@ -2,7 +2,7 @@ import { and, desc, eq, gte, isNull, or, sql } from 'drizzle-orm';
import { Hono } from 'hono';
import { generateIncidentMessage } from '../ai/incident-message';
import { buildResultStatements } from '../checks/persist-result';
import { runCheck } from '../checks/run-check';
import { MAX_RETRY_COUNT, runCheckWithRetries } from '../checks/run-check';
import { getDb } from '../db/client';
import { checks, incidentMonitors, incidents, maintenanceWindowMonitors, monitors } from '../db/schema';
import { requireAuth, type AuthVariables } from '../lib/require-auth';
@@ -19,6 +19,8 @@ type ParsedMonitorInput = {
expectedStatus?: number;
intervalSeconds?: number;
timeoutMs?: number;
retryCount?: number;
failureThreshold?: number;
enabled?: boolean;
alertsEnabled?: boolean;
};
@@ -272,6 +274,16 @@ export function parseMonitorInput(body: unknown, partial = false): ParseResult {
value[key] = parsed.value;
}
}
for (const [key, label, minimum, maximum] of [
['retryCount', 'retryCount', 0, MAX_RETRY_COUNT],
['failureThreshold', 'failureThreshold', 1, 10],
] as const) {
if (key in body) {
const parsed = parseInteger(body[key], label, minimum, maximum);
if (!parsed.ok) return parsed;
value[key] = parsed.value;
}
}
if ('enabled' in body) {
if (typeof body.enabled !== 'boolean') {
@@ -494,6 +506,8 @@ monitorRoutes.post('/', async (context) => {
expectedStatus: parsed.value.expectedStatus!,
intervalSeconds: parsed.value.intervalSeconds!,
timeoutMs: parsed.value.timeoutMs!,
retryCount: parsed.value.retryCount ?? 1,
failureThreshold: parsed.value.failureThreshold ?? 2,
enabled: parsed.value.enabled ?? true,
alertsEnabled: parsed.value.alertsEnabled ?? true,
createdAt: now,
@@ -556,12 +570,12 @@ monitorRoutes.post('/:id/check', async (context) => {
const [monitor] = await db.select().from(monitors).where(eq(monitors.id, id)).limit(1);
if (!monitor) return context.json({ message: 'Monitor not found' }, 404);
const result = await runCheck(monitor);
const result = await runCheckWithRetries(monitor);
const checkedAt = new Date();
const activeMaintenance = await loadActiveMaintenance(db, checkedAt);
const { statements, transition } = buildResultStatements(db, monitor, result, checkedAt, activeMaintenance.has(monitor.id));
await db.batch(statements as [(typeof statements)[number], ...typeof statements]);
if (transition) {
if (transition === 'opened' || transition === 'resolved') {
await sendIncidentAlert(context.env, { monitor, kind: transition, result, at: checkedAt });
if (transition === 'opened') {
await generateIncidentMessage(context.env, { monitor, result });
@@ -569,7 +583,7 @@ monitorRoutes.post('/:id/check', async (context) => {
}
const [updated] = await db.select().from(monitors).where(eq(monitors.id, monitor.id)).limit(1);
return context.json({ result, monitor: updated });
return context.json({ result, transition, monitor: updated });
});
export default monitorRoutes;
test/checks.spec.ts
+196 -30
View File
@@ -1,7 +1,7 @@
import { applyD1Migrations, type D1Migration } from 'cloudflare:test';
import { env } from 'cloudflare:workers';
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { runDueChecks } from '../src/worker/checks/run-due-checks';
import { MAX_RETRY_ATTEMPTS_PER_RUN, runDueChecks } from '../src/worker/checks/run-due-checks';
async function clearMonitoringTables() {
await env.DB.batch([
@@ -12,6 +12,7 @@ async function clearMonitoringTables() {
env.DB.prepare('DELETE FROM incident_monitors'),
env.DB.prepare('DELETE FROM incidents'),
env.DB.prepare('DELETE FROM monitor_daily_stats'),
env.DB.prepare('DELETE FROM ai_settings'),
env.DB.prepare('DELETE FROM notification_settings'),
env.DB.prepare('DELETE FROM monitors'),
]);
@@ -26,6 +27,9 @@ async function insertMonitor(overrides: Record<string, unknown> = {}) {
expected_status: 200,
interval_seconds: 300,
timeout_ms: 10_000,
retry_count: 0,
failure_threshold: 2,
consecutive_failures: 0,
enabled: 1,
last_checked_at: null,
created_at: now,
@@ -35,8 +39,9 @@ async function insertMonitor(overrides: Record<string, unknown> = {}) {
const result = await env.DB.prepare(
`
INSERT INTO monitors
(name, url, method, expected_status, interval_seconds, timeout_ms, enabled, alerts_enabled, last_ok, last_checked_at, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?)
(name, url, method, expected_status, interval_seconds, timeout_ms, retry_count, failure_threshold,
consecutive_failures, enabled, alerts_enabled, last_ok, last_checked_at, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?)
`,
)
.bind(
@@ -46,6 +51,9 @@ async function insertMonitor(overrides: Record<string, unknown> = {}) {
values.expected_status,
values.interval_seconds,
values.timeout_ms,
values.retry_count,
values.failure_threshold,
values.consecutive_failures,
values.enabled,
overrides.last_ok ?? null,
values.last_checked_at,
@@ -62,7 +70,10 @@ describe('scheduled monitor checks', () => {
await applyD1Migrations(testEnv.DB, testEnv.TEST_MIGRATIONS);
});
beforeEach(clearMonitoringTables);
afterEach(() => vi.unstubAllGlobals());
afterEach(() => {
vi.useRealTimers();
vi.unstubAllGlobals();
});
it('records a successful check and updates the monitor snapshot', async () => {
const id = await insertMonitor();
@@ -72,7 +83,7 @@ describe('scheduled monitor checks', () => {
);
const summary = await runDueChecks(env);
expect(summary).toEqual({ checked: 1, up: 1, down: 0 });
expect(summary).toEqual({ checked: 1, up: 1, down: 0, pending: 0, opened: 0, retries: 0 });
const monitor = await env.DB.prepare('SELECT last_ok, last_status_code, last_latency_ms, last_checked_at FROM monitors WHERE id = ?')
.bind(id)
@@ -87,39 +98,180 @@ describe('scheduled monitor checks', () => {
expect(checkCount?.count).toBe(1);
});
it('records a mismatched status as down with a useful error', async () => {
const id = await insertMonitor();
it('records a first mismatched status as pending without opening an incident', async () => {
const id = await insertMonitor({ last_ok: 1 });
const fetchMock = vi.fn(async () => new Response(null, { status: 500 }));
vi.stubGlobal('fetch', fetchMock);
const summary = await runDueChecks(env);
expect(summary).toEqual({ checked: 1, up: 0, down: 1, pending: 1, opened: 0, retries: 0 });
const monitor = await env.DB.prepare('SELECT last_ok, consecutive_failures, last_status_code, last_error FROM monitors WHERE id = ?')
.bind(id)
.first<{ last_ok: number; consecutive_failures: number; last_status_code: number; last_error: string }>();
expect(monitor).toEqual({
last_ok: 1,
consecutive_failures: 1,
last_status_code: 500,
last_error: 'Expected HTTP 200, received 500',
});
const incident = await env.DB.prepare('SELECT COUNT(*) AS count FROM incident_monitors WHERE monitor_id = ?')
.bind(id)
.first<{ count: number }>();
expect(incident?.count).toBe(0);
expect(fetchMock).toHaveBeenCalledTimes(1);
});
it('opens an incident once the failure threshold is reached', async () => {
const id = await insertMonitor({ last_ok: 1 });
vi.stubGlobal(
'fetch',
vi.fn(async () => new Response(null, { status: 500 })),
);
await runDueChecks(env);
await env.DB.prepare('UPDATE monitors SET last_checked_at = NULL WHERE id = ?').bind(id).run();
const summary = await runDueChecks(env);
expect(summary).toEqual({ checked: 1, up: 0, down: 1 });
const monitor = await env.DB.prepare('SELECT last_ok, last_status_code, last_error FROM monitors WHERE id = ?')
expect(summary.opened).toBe(1);
const monitor = await env.DB.prepare('SELECT last_ok, consecutive_failures FROM monitors WHERE id = ?')
.bind(id)
.first<{ last_ok: number; last_status_code: number; last_error: string }>();
expect(monitor).toEqual({
last_ok: 0,
last_status_code: 500,
last_error: 'Expected HTTP 200, received 500',
});
const incident = await env.DB.prepare(
'SELECT im.monitor_id, i.resolved_at, i.start_status_code, i.start_error FROM incidents i JOIN incident_monitors im ON im.incident_id = i.id WHERE im.monitor_id = ?',
)
.first<{ last_ok: number; consecutive_failures: number }>();
expect(monitor).toEqual({ last_ok: 0, consecutive_failures: 2 });
const incident = await env.DB.prepare('SELECT COUNT(*) AS count FROM incident_monitors WHERE monitor_id = ?')
.bind(id)
.first<{ monitor_id: number; resolved_at: number | null; start_status_code: number; start_error: string }>();
expect(incident).toEqual({
monitor_id: id,
resolved_at: null,
start_status_code: 500,
start_error: 'Expected HTTP 200, received 500',
});
.first<{ count: number }>();
expect(incident?.count).toBe(1);
});
it('resets a pending failure when the next check succeeds', async () => {
const id = await insertMonitor({ last_ok: 1, consecutive_failures: 1 });
vi.stubGlobal(
'fetch',
vi.fn(async () => new Response(null, { status: 200 })),
);
await runDueChecks(env);
const monitor = await env.DB.prepare('SELECT last_ok, consecutive_failures FROM monitors WHERE id = ?')
.bind(id)
.first<{ last_ok: number; consecutive_failures: number }>();
expect(monitor).toEqual({ last_ok: 1, consecutive_failures: 0 });
});
it('opens immediately when the threshold is one', async () => {
const id = await insertMonitor({ last_ok: 1, failure_threshold: 1 });
vi.stubGlobal(
'fetch',
vi.fn(async () => new Response(null, { status: 503 })),
);
const summary = await runDueChecks(env);
expect(summary.opened).toBe(1);
const incident = await env.DB.prepare('SELECT COUNT(*) AS count FROM incident_monitors WHERE monitor_id = ?')
.bind(id)
.first<{ count: number }>();
expect(incident?.count).toBe(1);
});
it('retries a failed attempt and persists only the recovered result', async () => {
const id = await insertMonitor({ last_ok: 1, retry_count: 1 });
const fetchMock = vi
.fn()
.mockResolvedValueOnce(new Response(null, { status: 500 }))
.mockResolvedValueOnce(new Response(null, { status: 200 }));
vi.stubGlobal('fetch', fetchMock);
const summary = await runDueChecks(env);
expect(summary).toMatchObject({ up: 1, retries: 1, pending: 0 });
expect(fetchMock).toHaveBeenCalledTimes(2);
const checks = await env.DB.prepare('SELECT ok FROM checks WHERE monitor_id = ?').bind(id).all<{ ok: number }>();
expect(checks.results).toEqual([{ ok: 1 }]);
});
it('records one failed check after exhausting retries', async () => {
const id = await insertMonitor({ last_ok: 1, retry_count: 2 });
const fetchMock = vi.fn(async () => new Response(null, { status: 500 }));
vi.stubGlobal('fetch', fetchMock);
await runDueChecks(env);
expect(fetchMock).toHaveBeenCalledTimes(3);
const checks = await env.DB.prepare('SELECT ok FROM checks WHERE monitor_id = ?').bind(id).all<{ ok: number }>();
expect(checks.results).toEqual([{ ok: 0 }]);
});
it('does not retry a monitor that is already confirmed down', async () => {
await insertMonitor({ last_ok: 0, consecutive_failures: 2, retry_count: 3 });
const fetchMock = vi.fn(async () => new Response(null, { status: 500 }));
vi.stubGlobal('fetch', fetchMock);
const summary = await runDueChecks(env);
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(summary.retries).toBe(0);
});
it('caps retry attempts across a scheduled run', async () => {
vi.useFakeTimers();
for (let index = 0; index < 40; index += 1) {
await insertMonitor({ name: `Monitor ${index}`, url: `https://monitor-${index}.example.com`, last_ok: 1, retry_count: 3 });
}
const fetchMock = vi.fn(async () => new Response(null, { status: 500 }));
vi.stubGlobal('fetch', fetchMock);
const run = runDueChecks(env);
await vi.runAllTimersAsync();
const summary = await run;
expect(summary.retries).toBe(MAX_RETRY_ATTEMPTS_PER_RUN);
expect(fetchMock).toHaveBeenCalledTimes(40 + MAX_RETRY_ATTEMPTS_PER_RUN);
});
it('does not send a webhook or generate AI copy until a failure is confirmed', async () => {
const id = await insertMonitor({ last_ok: 1, retry_count: 0 });
const now = Date.now();
await env.DB.batch([
env.DB.prepare(
"INSERT INTO notification_settings (id, webhook_url, webhook_enabled, created_at, updated_at) VALUES (1, 'https://hooks.example.test/events', 1, ?, ?)",
).bind(now, now),
env.DB.prepare(
"INSERT INTO ai_settings (id, enabled, base_url, api_key, model, created_at, updated_at) VALUES (1, 1, 'https://ai.example.test/v1', 'secret', 'test-model', ?, ?)",
).bind(now, now),
]);
const webhookBodies: Array<Record<string, unknown>> = [];
let aiCalls = 0;
vi.stubGlobal(
'fetch',
vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
const url = new URL(input.toString());
if (url.hostname === 'hooks.example.test') {
webhookBodies.push(JSON.parse(String(init?.body)) as Record<string, unknown>);
return new Response(null, { status: 204 });
}
if (url.hostname === 'ai.example.test') {
aiCalls += 1;
return Response.json({
choices: [
{
message: {
content: 'Some visitors may be unable to use the service. The team has been alerted and restoration work is underway.',
},
},
],
});
}
return new Response(null, { status: 500 });
}),
);
await runDueChecks(env);
expect(webhookBodies).toEqual([]);
expect(aiCalls).toBe(0);
expect((await env.DB.prepare('SELECT COUNT(*) AS count FROM incident_updates').first<{ count: number }>())?.count).toBe(0);
await env.DB.prepare('UPDATE monitors SET last_checked_at = NULL WHERE id = ?').bind(id).run();
await runDueChecks(env);
expect(webhookBodies).toHaveLength(1);
expect(webhookBodies[0]).toMatchObject({ event: 'down' });
expect(aiCalls).toBe(1);
expect((await env.DB.prepare('SELECT COUNT(*) AS count FROM incident_updates').first<{ count: number }>())?.count).toBe(1);
});
it('does not open duplicate incidents while a monitor stays down', async () => {
const id = await insertMonitor({ last_ok: 0 });
const id = await insertMonitor({ last_ok: 0, consecutive_failures: 2 });
vi.stubGlobal(
'fetch',
vi.fn(async () => new Response(null, { status: 503 })),
@@ -132,8 +284,18 @@ describe('scheduled monitor checks', () => {
});
it('links each auto incident to the correct monitor in one scheduled batch', async () => {
const firstId = await insertMonitor({ name: 'First', url: 'https://first.example.com' });
const secondId = await insertMonitor({ name: 'Second', url: 'https://second.example.com' });
const firstId = await insertMonitor({
name: 'First',
url: 'https://first.example.com',
last_ok: 1,
failure_threshold: 1,
});
const secondId = await insertMonitor({
name: 'Second',
url: 'https://second.example.com',
last_ok: 1,
failure_threshold: 1,
});
vi.stubGlobal(
'fetch',
vi.fn(async () => new Response(null, { status: 503 })),
@@ -149,7 +311,7 @@ describe('scheduled monitor checks', () => {
});
it('resolves the open incident on recovery', async () => {
const id = await insertMonitor({ last_ok: 0 });
const id = await insertMonitor({ last_ok: 0, consecutive_failures: 2 });
const startedAt = Date.now() - 60_000;
const inserted = await env.DB.prepare(
"INSERT INTO incidents (status, impact, source, started_at, start_status_code, start_error, created_at, updated_at) VALUES ('investigating', 'major', 'auto', ?, 500, 'Down', ?, ?)",
@@ -168,6 +330,10 @@ describe('scheduled monitor checks', () => {
.first<{ resolved_at: number | null; duration_ms: number | null }>();
expect(incident?.resolved_at).toEqual(expect.any(Number));
expect(incident?.duration_ms).toBeGreaterThanOrEqual(60_000);
const monitor = await env.DB.prepare('SELECT last_ok, consecutive_failures FROM monitors WHERE id = ?')
.bind(id)
.first<{ last_ok: number; consecutive_failures: number }>();
expect(monitor).toEqual({ last_ok: 1, consecutive_failures: 0 });
});
it('skips disabled and not-yet-due monitors', async () => {
@@ -177,7 +343,7 @@ describe('scheduled monitor checks', () => {
vi.stubGlobal('fetch', fetchMock);
const summary = await runDueChecks(env);
expect(summary).toEqual({ checked: 0, up: 0, down: 0 });
expect(summary).toEqual({ checked: 0, up: 0, down: 0, pending: 0, opened: 0, retries: 0 });
expect(fetchMock).not.toHaveBeenCalled();
});
});
test/maintenance.spec.ts
+9 -3
View File
@@ -36,7 +36,10 @@ async function resetDatabase() {
async function insertMonitor(lastOk = 1) {
const now = Date.now();
const result = await env.DB.prepare(
"INSERT INTO monitors (name, url, method, expected_status, interval_seconds, timeout_ms, enabled, alerts_enabled, last_ok, created_at, updated_at) VALUES ('API', 'https://example.com', 'GET', 200, 300, 10000, 1, 1, ?, ?, ?)",
`INSERT INTO monitors
(name, url, method, expected_status, interval_seconds, timeout_ms, retry_count, failure_threshold,
consecutive_failures, enabled, alerts_enabled, last_ok, created_at, updated_at)
VALUES ('API', 'https://example.com', 'GET', 200, 300, 10000, 0, 1, 0, 1, 1, ?, ?, ?)`,
)
.bind(lastOk, now, now)
.run();
@@ -78,18 +81,21 @@ describe('maintenance windows', () => {
it('records failed probes as maintenance without incidents or monitor state changes', async () => {
const id = await insertMonitor(1);
await env.DB.prepare('UPDATE monitors SET retry_count = 3, consecutive_failures = 1 WHERE id = ?').bind(id).run();
await insertActiveWindow(id);
const fetchMock = vi.fn(async () => new Response(null, { status: 503 }));
vi.stubGlobal('fetch', fetchMock);
await runDueChecks(env);
const check = await env.DB.prepare('SELECT maintenance, ok FROM checks WHERE monitor_id = ?').bind(id).first();
const monitor = await env.DB.prepare('SELECT last_ok, last_status_code FROM monitors WHERE id = ?').bind(id).first();
const monitor = await env.DB.prepare('SELECT last_ok, consecutive_failures, last_status_code FROM monitors WHERE id = ?')
.bind(id)
.first();
const incident = await env.DB.prepare('SELECT COUNT(*) AS count FROM incident_monitors WHERE monitor_id = ?')
.bind(id)
.first<{ count: number }>();
expect(check).toMatchObject({ maintenance: 1, ok: 0 });
expect(monitor).toMatchObject({ last_ok: 1, last_status_code: 503 });
expect(monitor).toMatchObject({ last_ok: 1, consecutive_failures: 1, last_status_code: 503 });
expect(incident?.count).toBe(0);
expect(fetchMock).toHaveBeenCalledTimes(1);
});
test/monitors.spec.ts
+97 -3
View File
@@ -144,10 +144,25 @@ describe('monitor API', () => {
it('creates a valid monitor and returns it in the list', async () => {
const cookie = await authenticatedCookie();
const response = await createMonitor(cookie);
const created = await response.json<{ monitor: { id: number; name: string; enabled: boolean; alertsEnabled: boolean } }>();
const created = await response.json<{
monitor: {
id: number;
name: string;
enabled: boolean;
alertsEnabled: boolean;
retryCount: number;
failureThreshold: number;
};
}>();
expect(response.status).toBe(200);
expect(created.monitor).toMatchObject({ name: 'Example', enabled: true, alertsEnabled: true });
expect(created.monitor).toMatchObject({
name: 'Example',
enabled: true,
alertsEnabled: true,
retryCount: 1,
failureThreshold: 2,
});
const listResponse = await apiFetch('/api/monitors', 'GET', cookie);
const list = await listResponse.json<{ monitors: Array<{ id: number; url: string }> }>();
@@ -159,6 +174,10 @@ describe('monitor API', () => {
[{ url: 'file:///etc/passwd' }, 'Enter a valid http or https URL'],
[{ intervalSeconds: 60 }, 'intervalSeconds must be an integer between 300 and 86400'],
[{ expectedStatus: 99 }, 'expectedStatus must be an integer between 100 and 599'],
[{ retryCount: -1 }, 'retryCount must be an integer between 0 and 3'],
[{ retryCount: 4 }, 'retryCount must be an integer between 0 and 3'],
[{ failureThreshold: 0 }, 'failureThreshold must be an integer between 1 and 10'],
[{ failureThreshold: 11 }, 'failureThreshold must be an integer between 1 and 10'],
] as const)('rejects invalid monitor input %o', async (overrides, message) => {
const response = await createMonitor(await authenticatedCookie(), overrides);
expect(response.status).toBe(400);
@@ -171,17 +190,92 @@ describe('monitor API', () => {
const response = await apiFetch(`/api/monitors/${created.monitor.id}`, 'PATCH', cookie, {
name: 'Renamed endpoint',
enabled: false,
retryCount: 3,
failureThreshold: 4,
});
const body = await response.json<{ monitor: { name: string; url: string; enabled: boolean } }>();
const body = await response.json<{
monitor: { name: string; url: string; enabled: boolean; retryCount: number; failureThreshold: number };
}>();
expect(response.status).toBe(200);
expect(body.monitor).toMatchObject({
name: 'Renamed endpoint',
url: 'https://example.com/health',
enabled: false,
retryCount: 3,
failureThreshold: 4,
});
});
it('does not allow clients to set the internal failure counter', async () => {
const cookie = await authenticatedCookie();
const created = await (await createMonitor(cookie)).json<{ monitor: { id: number } }>();
const response = await apiFetch(`/api/monitors/${created.monitor.id}`, 'PATCH', cookie, { consecutiveFailures: 99 });
expect(response.status).toBe(400);
expect(await response.json()).toEqual({ message: 'Provide at least one field to update' });
});
it('returns a pending transition without opening an incident on the first manual failure', async () => {
const cookie = await authenticatedCookie();
const created = await (await createMonitor(cookie, { retryCount: 0 })).json<{ monitor: { id: number } }>();
vi.stubGlobal(
'fetch',
vi.fn(async () => new Response(null, { status: 500 })),
);
const response = await apiFetch(`/api/monitors/${created.monitor.id}/check`, 'POST', cookie);
const body = await response.json<{
transition: string;
result: { attempts: number };
monitor: { lastOk: boolean | null; consecutiveFailures: number };
}>();
expect(body).toMatchObject({
transition: 'pending',
result: { attempts: 1 },
monitor: { lastOk: null, consecutiveFailures: 1 },
});
const incident = await env.DB.prepare('SELECT COUNT(*) AS count FROM incidents').first<{ count: number }>();
expect(incident?.count).toBe(0);
});
it('opens an incident when a manual check reaches the confirmation threshold', async () => {
const cookie = await authenticatedCookie();
const created = await (
await createMonitor(cookie, { retryCount: 0, failureThreshold: 2 })
).json<{
monitor: { id: number };
}>();
vi.stubGlobal(
'fetch',
vi.fn(async () => new Response(null, { status: 500 })),
);
const first = await apiFetch(`/api/monitors/${created.monitor.id}/check`, 'POST', cookie);
const second = await apiFetch(`/api/monitors/${created.monitor.id}/check`, 'POST', cookie);
expect((await first.json<{ transition: string }>()).transition).toBe('pending');
expect((await second.json<{ transition: string }>()).transition).toBe('opened');
const incident = await env.DB.prepare('SELECT COUNT(*) AS count FROM incident_monitors WHERE monitor_id = ?')
.bind(created.monitor.id)
.first<{ count: number }>();
expect(incident?.count).toBe(1);
});
it('retries a manual check and returns the attempt count', async () => {
const cookie = await authenticatedCookie();
const created = await (await createMonitor(cookie, { retryCount: 1 })).json<{ monitor: { id: number } }>();
const fetchMock = vi
.fn()
.mockResolvedValueOnce(new Response(null, { status: 500 }))
.mockResolvedValueOnce(new Response(null, { status: 200 }));
vi.stubGlobal('fetch', fetchMock);
const response = await apiFetch(`/api/monitors/${created.monitor.id}/check`, 'POST', cookie);
const body = await response.json<{ transition: null; result: { ok: boolean; attempts: number } }>();
expect(body).toMatchObject({ transition: null, result: { ok: true, attempts: 2 } });
expect(fetchMock).toHaveBeenCalledTimes(2);
});
it('returns detail, checks, incidents, and raw stats', async () => {
const cookie = await authenticatedCookie();
const created = await (await createMonitor(cookie)).json<{ monitor: { id: number } }>();
test/status.spec.ts
+32 -3
View File
@@ -42,21 +42,25 @@ async function insertMonitor(input: {
lastStatusCode?: number | null;
lastLatencyMs?: number | null;
lastError?: string | null;
consecutiveFailures?: number;
failureThreshold?: number;
}) {
const now = Date.now();
const result = await env.DB.prepare(
`
INSERT INTO monitors (
name, url, method, expected_status, interval_seconds, timeout_ms,
enabled, alerts_enabled, last_ok, last_status_code, last_latency_ms,
enabled, alerts_enabled, failure_threshold, consecutive_failures, last_ok, last_status_code, last_latency_ms,
last_error, last_checked_at, created_at, updated_at
) VALUES (?, ?, 'GET', 200, 300, 10000, ?, 1, ?, ?, ?, ?, ?, ?, ?)
) VALUES (?, ?, 'GET', 200, 300, 10000, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`,
)
.bind(
input.name,
input.url ?? `https://${input.name.toLowerCase()}.example.com/health`,
input.enabled === false ? 0 : 1,
input.failureThreshold ?? 2,
input.consecutiveFailures ?? 0,
input.lastOk === null || input.lastOk === undefined ? null : input.lastOk ? 1 : 0,
input.lastStatusCode ?? null,
input.lastLatencyMs ?? null,
@@ -103,11 +107,36 @@ describe('public status API', () => {
uptime90d: null,
history: [],
});
for (const privateField of ['url', 'lastError', 'lastStatusCode', 'lastLatencyMs', 'method', 'timeoutMs']) {
for (const privateField of [
'url',
'lastError',
'lastStatusCode',
'lastLatencyMs',
'method',
'timeoutMs',
'retryCount',
'failureThreshold',
'consecutiveFailures',
]) {
expect(body.services[0]).not.toHaveProperty(privateField);
}
});
it('keeps a service operational while failures are unconfirmed', async () => {
await insertMonitor({
name: 'Flaky API',
lastOk: true,
consecutiveFailures: 1,
failureThreshold: 2,
lastStatusCode: 500,
lastError: 'Expected HTTP 200, received 500',
});
const body = await (await statusFetch()).json<PublicStatusResponse>();
expect(body.overall).toBe('operational');
expect(body.services[0]).toMatchObject({ status: 'up', message: null });
});
it('excludes disabled monitors and reports degraded health for a partial outage', async () => {
await insertMonitor({ name: 'Healthy', lastOk: true });
await insertMonitor({ name: 'Unavailable', lastOk: false });
test/webhook.spec.ts
+3
View File
@@ -30,6 +30,9 @@ describe('incident webhooks', () => {
timeoutMs: 10000,
enabled: true,
alertsEnabled: true,
retryCount: 1,
failureThreshold: 2,
consecutiveFailures: 0,
lastOk: true,
lastStatusCode: 200,
lastLatencyMs: 30,