fix: improve the quality of the app

This commit is contained in:
2026-08-31 22:25:49 +07:00
parent 3dcd04b40d
commit e90ee15b64
16 changed files with 1858 additions and 31 deletions
+4
View File
@@ -0,0 +1,4 @@
CREATE INDEX `checks_checked_at_idx` ON `checks` (`checked_at`);--> statement-breakpoint
CREATE INDEX `login_attempts_attempted_at_idx` ON `login_attempts` (`attempted_at`);--> statement-breakpoint
CREATE INDEX `monitor_daily_stats_day_idx` ON `monitor_daily_stats` (`day`);--> statement-breakpoint
CREATE INDEX `notification_deliveries_created_at_idx` ON `notification_deliveries` (`created_at`);
File diff suppressed because it is too large Load Diff
+7
View File
@@ -78,6 +78,13 @@
"when": 1788183346274, "when": 1788183346274,
"tag": "0010_slippery_mattie_franklin", "tag": "0010_slippery_mattie_franklin",
"breakpoints": true "breakpoints": true
},
{
"idx": 11,
"version": "6",
"when": 1788187390249,
"tag": "0011_hard_mystique",
"breakpoints": true
} }
] ]
} }
+4 -11
View File
@@ -1,12 +1,7 @@
import type { PublicService } from '../api/status'; import type { PublicService } from '../api/status';
import { useMediaQuery } from '../lib/useMediaQuery';
const DAY_MS = 24 * 60 * 60 * 1000; const DAY_MS = 24 * 60 * 60 * 1000;
// Desktop mirrors the API's full 90-day history. Compact screens use the most
// recent 45 days so each daily bar remains legible.
const HISTORY_DAYS = 90; const HISTORY_DAYS = 90;
const HISTORY_DAYS_COMPACT = 45;
const COMPACT_QUERY = '(max-width: 520px)';
type HistoryEntry = PublicService['history'][number]; type HistoryEntry = PublicService['history'][number];
@@ -32,25 +27,23 @@ function dayTitle(day: number, uptimePct: number | null | undefined) {
} }
export function StatusHistoryBar({ history }: { history: HistoryEntry[] }) { export function StatusHistoryBar({ history }: { history: HistoryEntry[] }) {
const compact = useMediaQuery(COMPACT_QUERY);
const windowDays = compact ? HISTORY_DAYS_COMPACT : HISTORY_DAYS;
const historyByDay = new Map(history.map((entry) => [entry.day, entry.uptimePct])); const historyByDay = new Map(history.map((entry) => [entry.day, entry.uptimePct]));
const now = new Date(); const now = new Date();
const today = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()); const today = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate());
const days = Array.from({ length: windowDays }, (_day, index) => { const days = Array.from({ length: HISTORY_DAYS }, (_day, index) => {
const day = today - (windowDays - 1 - index) * DAY_MS; const day = today - (HISTORY_DAYS - 1 - index) * DAY_MS;
return { day, uptimePct: historyByDay.get(day) }; return { day, uptimePct: historyByDay.get(day) };
}); });
return ( return (
<div className="status-history"> <div className="status-history">
<div className="uptime-days" aria-label={`Daily uptime over the last ${windowDays} days`}> <div className="uptime-days" aria-label={`Daily uptime over the last ${HISTORY_DAYS} days`}>
{days.map(({ day, uptimePct }) => ( {days.map(({ day, uptimePct }) => (
<span className={`uptime-day ${dayClass(uptimePct)}`} key={day} title={dayTitle(day, uptimePct)} /> <span className={`uptime-day ${dayClass(uptimePct)}`} key={day} title={dayTitle(day, uptimePct)} />
))} ))}
</div> </div>
<div className="uptime-days-caption" aria-hidden="true"> <div className="uptime-days-caption" aria-hidden="true">
<span>{windowDays} days ago</span> <span>{HISTORY_DAYS} days ago</span>
<i /> <i />
<span>Today</span> <span>Today</span>
</div> </div>
+6 -4
View File
@@ -14,11 +14,12 @@ import type { CheckResult, Monitor } from '../checks/run-check';
import { getDb } from '../db/client'; import { getDb } from '../db/client';
import { aiSettings, incidentUpdates, incidents } from '../db/schema'; import { aiSettings, incidentUpdates, incidents } from '../db/schema';
import { humanizeDuration } from '../lib/humanize'; import { humanizeDuration } from '../lib/humanize';
import { resolveRunLimits } from '../lib/runtime-config';
import { advanceStatus, computeImpact, nextFollowupDueAt, type AutopilotIncidentStatus } from './cadence'; import { advanceStatus, computeImpact, nextFollowupDueAt, type AutopilotIncidentStatus } from './cadence';
import { loadIncidentSignal, findLatestAutoIncidentForMonitor } from './signal'; import { loadIncidentSignal, findLatestAutoIncidentForMonitor } from './signal';
export const MAX_AI_CALLS_PER_RUN = 12; // Per-pass ceilings come from resolveRunLimits(env): AI_CALLS_PER_RUN / AI_FOLLOWUP_CALLS_PER_RUN,
export const MAX_FOLLOWUP_CALLS_PER_RUN = 6; // defaulting to DEFAULT_RUN_LIMITS. They keep one autopilot pass within the free-plan subrequest budget.
export const AUTOPILOT_CONCURRENCY = 4; export const AUTOPILOT_CONCURRENCY = 4;
export const AUTOPILOT_DEADLINE_MS = 45_000; export const AUTOPILOT_DEADLINE_MS = 45_000;
@@ -215,6 +216,7 @@ async function processFollowup(env: Env, settings: Settings, incident: typeof in
async function loadFollowupTasks(env: Env, settings: Settings, excluded: Set<number>): Promise<Task[]> { async function loadFollowupTasks(env: Env, settings: Settings, excluded: Set<number>): Promise<Task[]> {
const db = getDb(env); const db = getDb(env);
const maxFollowups = resolveRunLimits(env).aiFollowupCallsPerRun;
const rows = await db const rows = await db
.select({ .select({
incident: incidents, incident: incidents,
@@ -264,7 +266,7 @@ async function loadFollowupTasks(env: Env, settings: Settings, excluded: Set<num
monitorId: signal.monitor.id, monitorId: signal.monitor.id,
run: () => processFollowup(env, settings, row.incident), run: () => processFollowup(env, settings, row.incident),
}); });
if (tasks.length >= MAX_FOLLOWUP_CALLS_PER_RUN) break; if (tasks.length >= maxFollowups) break;
} }
return tasks; return tasks;
} }
@@ -277,7 +279,7 @@ export async function runAutopilot(
const db = getDb(env); const db = getDb(env);
const [settings] = await db.select().from(aiSettings).where(eq(aiSettings.id, 1)).limit(1); const [settings] = await db.select().from(aiSettings).where(eq(aiSettings.id, 1)).limit(1);
if (!settings?.enabled || !settings.autopilotEnabled || !settings.baseUrl || !settings.apiKey || !settings.model) return summary; if (!settings?.enabled || !settings.autopilotEnabled || !settings.baseUrl || !settings.apiKey || !settings.model) return summary;
const budget = input.budget ?? { remaining: MAX_AI_CALLS_PER_RUN, deadline: Date.now() + AUTOPILOT_DEADLINE_MS }; const budget = input.budget ?? { remaining: resolveRunLimits(env).aiCallsPerRun, deadline: Date.now() + AUTOPILOT_DEADLINE_MS };
budget.deadline ??= Date.now() + AUTOPILOT_DEADLINE_MS; budget.deadline ??= Date.now() + AUTOPILOT_DEADLINE_MS;
const tasks: Task[] = []; const tasks: Task[] = [];
const excluded = new Set<number>(); const excluded = new Set<number>();
+5 -1
View File
@@ -11,7 +11,11 @@ export type CheckResult = {
error: string | null; error: string | null;
}; };
export const MAX_BODY_MATCH_BYTES = 256 * 1024; // Keyword matching decodes and lowercases this many bytes of the response body on the hot
// path. 64 KiB covers page titles, health-check payloads and status JSON while keeping the
// per-check CPU cost small — the free plan allows only 10 ms of CPU per invocation and one
// scheduled run decodes bodies for every keyword monitor.
export const MAX_BODY_MATCH_BYTES = 64 * 1024;
function requestHeaders(monitor: Monitor) { function requestHeaders(monitor: Monitor) {
let configured: Record<string, string> = {}; let configured: Record<string, string> = {};
+7 -4
View File
@@ -2,16 +2,18 @@ import { and, eq, sql } from 'drizzle-orm';
import type { AutopilotEvent } from '../autopilot'; import type { AutopilotEvent } from '../autopilot';
import { getDb } from '../db/client'; import { getDb } from '../db/client';
import { aiSettings, monitors } from '../db/schema'; import { aiSettings, monitors } from '../db/schema';
import { DEFAULT_RUN_LIMITS, resolveRunLimits } from '../lib/runtime-config';
import { loadActiveMaintenance } from '../maintenance/windows'; import { loadActiveMaintenance } from '../maintenance/windows';
import { buildAlertEvent } from '../notifications/compose'; import { buildAlertEvent } from '../notifications/compose';
import { dispatchNotification, MAX_NOTIFICATIONS_PER_RUN, type NotificationBudget } from '../notifications/dispatch'; import { dispatchNotification, type NotificationBudget } from '../notifications/dispatch';
import { buildResultStatements } from './persist-result'; import { buildResultStatements } from './persist-result';
import { runCheck, runCheckWithRetries, type RetryBudget } from './run-check'; import { runCheck, runCheckWithRetries, type RetryBudget } from './run-check';
const MAX_MONITORS_PER_RUN = 40; const MAX_MONITORS_PER_RUN = 40;
const CONCURRENCY = 10; const CONCURRENCY = 10;
const MAX_BATCH_STATEMENTS = 100; const MAX_BATCH_STATEMENTS = 100;
export const MAX_RETRY_ATTEMPTS_PER_RUN = 60; /** Default retry ceiling for one scheduled run; override with the RETRY_ATTEMPTS_PER_RUN env var. */
export const MAX_RETRY_ATTEMPTS_PER_RUN = DEFAULT_RUN_LIMITS.retryAttemptsPerRun;
const RETRY_DEADLINE_MS = 90_000; const RETRY_DEADLINE_MS = 90_000;
export type DueCheckSummary = { export type DueCheckSummary = {
@@ -40,11 +42,12 @@ export async function runDueChecks(env: Env, ctx?: Pick<ExecutionContext, 'waitU
.limit(MAX_MONITORS_PER_RUN); .limit(MAX_MONITORS_PER_RUN);
if (due.length === 0) return { checked: 0, up: 0, down: 0, pending: 0, opened: 0, retries: 0, events: [] }; if (due.length === 0) return { checked: 0, up: 0, down: 0, pending: 0, opened: 0, retries: 0, events: [] };
const limits = resolveRunLimits(env);
const [activeMaintenance, [settings]] = await Promise.all([ const [activeMaintenance, [settings]] = await Promise.all([
loadActiveMaintenance(db, new Date()), loadActiveMaintenance(db, new Date()),
db.select().from(aiSettings).where(eq(aiSettings.id, 1)).limit(1), db.select().from(aiSettings).where(eq(aiSettings.id, 1)).limit(1),
]); ]);
const budget: RetryBudget = { remaining: MAX_RETRY_ATTEMPTS_PER_RUN, deadline: Date.now() + RETRY_DEADLINE_MS }; const budget: RetryBudget = { remaining: limits.retryAttemptsPerRun, deadline: Date.now() + RETRY_DEADLINE_MS };
const completed: Array<{ const completed: Array<{
monitor: (typeof due)[number]; monitor: (typeof due)[number];
@@ -84,7 +87,7 @@ export async function runDueChecks(env: Env, ctx?: Pick<ExecutionContext, 'waitU
} }
if (statementChunk.length > 0) await db.batch(statementChunk as [(typeof statementChunk)[number], ...typeof statementChunk]); if (statementChunk.length > 0) await db.batch(statementChunk as [(typeof statementChunk)[number], ...typeof statementChunk]);
const notificationBudget: NotificationBudget = { remaining: MAX_NOTIFICATIONS_PER_RUN }; const notificationBudget: NotificationBudget = { remaining: limits.notificationsPerRun };
const notifications = persisted.flatMap((item) => { const notifications = persisted.flatMap((item) => {
const work: Promise<unknown>[] = []; const work: Promise<unknown>[] = [];
if (item.transition === 'opened' || item.transition === 'resolved') { if (item.transition === 'opened' || item.transition === 'resolved') {
+21 -4
View File
@@ -40,7 +40,11 @@ export const loginAttempts = sqliteTable(
ipAddress: text('ip_address').notNull(), ipAddress: text('ip_address').notNull(),
attemptedAt: integer('attempted_at', { mode: 'timestamp_ms' }).notNull(), attemptedAt: integer('attempted_at', { mode: 'timestamp_ms' }).notNull(),
}, },
(table) => [index('login_attempts_ip_attempted_at_idx').on(table.ipAddress, table.attemptedAt)], (table) => [
index('login_attempts_ip_attempted_at_idx').on(table.ipAddress, table.attemptedAt),
// Retention cleanup filters on attempted_at alone; the composite index above cannot serve it.
index('login_attempts_attempted_at_idx').on(table.attemptedAt),
],
); );
export const monitors = sqliteTable( export const monitors = sqliteTable(
@@ -128,7 +132,12 @@ export const checks = sqliteTable(
maintenance: integer('maintenance', { mode: 'boolean' }).notNull().default(false), maintenance: integer('maintenance', { mode: 'boolean' }).notNull().default(false),
degraded: integer('degraded', { mode: 'boolean' }).notNull().default(false), degraded: integer('degraded', { mode: 'boolean' }).notNull().default(false),
}, },
(table) => [index('checks_monitor_id_checked_at_idx').on(table.monitorId, table.checkedAt)], (table) => [
index('checks_monitor_id_checked_at_idx').on(table.monitorId, table.checkedAt),
// Daily rollup and retention cleanup scan by checked_at across every monitor; without this
// index those become full-table scans, which dominate D1 "rows read" at scale.
index('checks_checked_at_idx').on(table.checkedAt),
],
); );
export const incidents = sqliteTable( export const incidents = sqliteTable(
@@ -247,7 +256,11 @@ export const notificationDeliveries = sqliteTable(
attempts: integer('attempts').notNull().default(1), attempts: integer('attempts').notNull().default(1),
createdAt: integer('created_at', { mode: 'timestamp_ms' }).notNull(), createdAt: integer('created_at', { mode: 'timestamp_ms' }).notNull(),
}, },
(table) => [index('notification_deliveries_channel_id_created_at_idx').on(table.channelId, table.createdAt)], (table) => [
index('notification_deliveries_channel_id_created_at_idx').on(table.channelId, table.createdAt),
// Retention cleanup filters on created_at alone; the composite index above cannot serve it.
index('notification_deliveries_created_at_idx').on(table.createdAt),
],
); );
export const monitorDailyStats = sqliteTable( export const monitorDailyStats = sqliteTable(
@@ -264,5 +277,9 @@ export const monitorDailyStats = sqliteTable(
minLatencyMs: integer('min_latency_ms'), minLatencyMs: integer('min_latency_ms'),
maxLatencyMs: integer('max_latency_ms'), maxLatencyMs: integer('max_latency_ms'),
}, },
(table) => [uniqueIndex('monitor_daily_stats_monitor_id_day_uidx').on(table.monitorId, table.day)], (table) => [
uniqueIndex('monitor_daily_stats_monitor_id_day_uidx').on(table.monitorId, table.day),
// Retention cleanup filters on day alone; the composite unique index above cannot serve it.
index('monitor_daily_stats_day_idx').on(table.day),
],
); );
+2 -1
View File
@@ -9,7 +9,7 @@ import maintenanceRoutes from './routes/maintenance';
import monitorRoutes from './routes/monitors'; import monitorRoutes from './routes/monitors';
import settingsRoutes from './routes/settings'; import settingsRoutes from './routes/settings';
import statusRoutes from './routes/status'; import statusRoutes from './routes/status';
import { cleanupExpiredAuthRecords } from './scheduled/cleanup'; import { cleanupExpiredAuthRecords, cleanupStaleData } from './scheduled/cleanup';
import { runDailyRollup } from './scheduled/rollup'; import { runDailyRollup } from './scheduled/rollup';
const app = new Hono<{ Bindings: Env }>(); const app = new Hono<{ Bindings: Env }>();
@@ -41,6 +41,7 @@ export default {
async scheduled(controller, env, ctx) { async scheduled(controller, env, ctx) {
if (controller.cron === '5 0 * * *') { if (controller.cron === '5 0 * * *') {
const result = await runDailyRollup(env, new Date(controller.scheduledTime)); const result = await runDailyRollup(env, new Date(controller.scheduledTime));
await cleanupStaleData(env);
console.log( console.log(
JSON.stringify({ JSON.stringify({
message: 'daily rollup completed', message: 'daily rollup completed',
+53
View File
@@ -0,0 +1,53 @@
/**
* Environment-tunable runtime knobs.
*
* Cloudflare's free plan caps a Worker invocation at 50 subrequests and 10 ms CPU, and the
* 5-minute cron runs checks, retries, notifications and AI autopilot inside a single
* invocation. The defaults below keep a healthy run well under that ceiling and make a large
* correlated outage shed the least-critical work first (AI, then retries) instead of failing
* mid-batch. Raise them with same-named environment variables on the Workers Paid plan, where
* the ceiling is 1,000 subrequests and 5 minutes of CPU.
*/
export type RunLimits = {
/** Immediate check retries attempted across one scheduled run, shared by every monitor. */
retryAttemptsPerRun: number;
/** Alert deliveries attempted across one scheduled run. */
notificationsPerRun: number;
/** AI completion calls attempted across one autopilot pass. */
aiCallsPerRun: number;
/** AI follow-up updates queued across one autopilot pass. */
aiFollowupCallsPerRun: number;
};
export const DEFAULT_RUN_LIMITS: RunLimits = {
retryAttemptsPerRun: 15,
notificationsPerRun: 12,
aiCallsPerRun: 4,
aiFollowupCallsPerRun: 2,
};
/** Seconds the public status responses are held in the edge cache. 0 disables edge caching. */
export const DEFAULT_STATUS_CACHE_SECONDS = 30;
type EnvVars = Record<string, string | undefined>;
function boundedInt(raw: string | undefined, fallback: number, minimum: number): number {
if (raw === undefined) return fallback;
const value = Number(raw);
return Number.isInteger(value) && value >= minimum ? value : fallback;
}
export function resolveRunLimits(env: Env): RunLimits {
const vars = env as unknown as EnvVars;
return {
retryAttemptsPerRun: boundedInt(vars.RETRY_ATTEMPTS_PER_RUN, DEFAULT_RUN_LIMITS.retryAttemptsPerRun, 1),
notificationsPerRun: boundedInt(vars.NOTIFICATIONS_PER_RUN, DEFAULT_RUN_LIMITS.notificationsPerRun, 1),
aiCallsPerRun: boundedInt(vars.AI_CALLS_PER_RUN, DEFAULT_RUN_LIMITS.aiCallsPerRun, 1),
aiFollowupCallsPerRun: boundedInt(vars.AI_FOLLOWUP_CALLS_PER_RUN, DEFAULT_RUN_LIMITS.aiFollowupCallsPerRun, 1),
};
}
export function resolveStatusCacheSeconds(env: Env): number {
return boundedInt((env as unknown as EnvVars).STATUS_CACHE_SECONDS, DEFAULT_STATUS_CACHE_SECONDS, 0);
}
+55 -3
View File
@@ -1,8 +1,9 @@
import { and, desc, eq, gte, inArray, isNull, lt, sql } from 'drizzle-orm'; import { and, desc, eq, gte, inArray, isNull, lt, sql } from 'drizzle-orm';
import { Hono } from 'hono'; import { Hono, type Context } from 'hono';
import { DEGRADED_MESSAGE, deterministicIncidentMessage } from '../ai/fallback-message'; import { DEGRADED_MESSAGE, deterministicIncidentMessage } from '../ai/fallback-message';
import { getDb } from '../db/client'; import { getDb } from '../db/client';
import { checks, incidentMonitors, incidents, incidentUpdates, monitorDailyStats, monitors } from '../db/schema'; import { checks, incidentMonitors, incidents, incidentUpdates, monitorDailyStats, monitors } from '../db/schema';
import { resolveStatusCacheSeconds } from '../lib/runtime-config';
import { loadActiveMaintenance, type ActiveMaintenance } from '../maintenance/windows'; import { loadActiveMaintenance, type ActiveMaintenance } from '../maintenance/windows';
import { resolveFavicon } from './monitors'; import { resolveFavicon } from './monitors';
@@ -111,9 +112,58 @@ const incidentSelection = {
startStatusCode: incidents.startStatusCode, startStatusCode: incidents.startStatusCode,
}; };
function edgeCache(): EdgeCache | null {
try {
return (caches as CacheStorage & { readonly default: EdgeCache }).default;
} catch {
return null;
}
}
// Key on origin + path only. These responses do not vary by query string, so ignoring it also
// stops `?x=1`, `?x=2`, … spray from bypassing the cache and hammering D1 on every request.
function statusCacheKey(context: Context<{ Bindings: Env }>): Request {
const url = new URL(context.req.url);
return new Request(`${url.origin}${url.pathname}`);
}
/**
* The public status endpoints are polled by every open status-page tab every 60s. Serving them
* from the edge cache collapses that traffic to one origin computation per window and keeps the
* `checks` / `monitor_daily_stats` scans they run off D1's free-tier read budget. Set the
* STATUS_CACHE_SECONDS env var to 0 to disable.
*/
async function cachedStatusResponse(context: Context<{ Bindings: Env }>): Promise<Response | undefined> {
if (resolveStatusCacheSeconds(context.env) <= 0) return undefined;
const cache = edgeCache();
if (!cache) return undefined;
try {
return await cache.match(statusCacheKey(context));
} catch {
return undefined;
}
}
function jsonWithEdgeCache(context: Context<{ Bindings: Env }>, body: unknown): Response {
const seconds = resolveStatusCacheSeconds(context.env);
if (seconds <= 0) return Response.json(body);
const response = Response.json(body, { headers: { 'Cache-Control': `public, max-age=${seconds}` } });
const cache = edgeCache();
if (cache) {
try {
context.executionCtx.waitUntil(cache.put(statusCacheKey(context), response.clone()).catch(() => undefined));
} catch {
// No ExecutionContext available (e.g. unit tests): serve without populating the edge cache.
}
}
return response;
}
const statusRoutes = new Hono<{ Bindings: Env }>(); const statusRoutes = new Hono<{ Bindings: Env }>();
statusRoutes.get('/', async (context) => { statusRoutes.get('/', async (context) => {
const cached = await cachedStatusResponse(context);
if (cached) return cached;
const db = getDb(context.env); const db = getDb(context.env);
const monitorRows = await db const monitorRows = await db
.select({ .select({
@@ -230,7 +280,7 @@ statusRoutes.get('/', async (context) => {
latestUpdate: latestUpdates.get(incident.id) ?? null, latestUpdate: latestUpdates.get(incident.id) ?? null,
services: incidentServices.get(incident.id) ?? [], services: incidentServices.get(incident.id) ?? [],
})); }));
return context.json({ return jsonWithEdgeCache(context, {
overall: overallStatus( overall: overallStatus(
services.map((service) => service.status), services.map((service) => service.status),
activeIncidentRows.filter((incident) => incident.source === 'manual').map((incident) => incident.impact), activeIncidentRows.filter((incident) => incident.source === 'manual').map((incident) => incident.impact),
@@ -242,6 +292,8 @@ statusRoutes.get('/', async (context) => {
}); });
statusRoutes.get('/incidents', async (context) => { statusRoutes.get('/incidents', async (context) => {
const cached = await cachedStatusResponse(context);
if (cached) return cached;
const db = getDb(context.env); const db = getDb(context.env);
const limit = parseLimit(context.req.query('limit'), 20, 20); const limit = parseLimit(context.req.query('limit'), 20, 20);
const rows = await db const rows = await db
@@ -252,7 +304,7 @@ statusRoutes.get('/incidents', async (context) => {
.limit(limit); .limit(limit);
const incidentIds = rows.map((row) => row.id); const incidentIds = rows.map((row) => row.id);
const [services, latestUpdates] = await Promise.all([loadServices(db, incidentIds), loadLatestUpdates(db, incidentIds)]); const [services, latestUpdates] = await Promise.all([loadServices(db, incidentIds), loadLatestUpdates(db, incidentIds)]);
return context.json({ return jsonWithEdgeCache(context, {
incidents: rows.map((incident) => ({ incidents: rows.map((incident) => ({
id: incident.id, id: incident.id,
title: publicIncidentTitle(incident), title: publicIncidentTitle(incident),
+22
View File
@@ -8,6 +8,11 @@ const DAILY_STATS_RETENTION_MS = 400 * 24 * 60 * 60 * 1000;
const DELIVERY_RETENTION_MS = 30 * 24 * 60 * 60 * 1000; const DELIVERY_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
export const AI_EVENT_RETENTION_MS = 30 * 24 * 60 * 60 * 1000; export const AI_EVENT_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
/**
* Cheap, high-frequency cleanup for the every-5-minute cron. Only touches the auth tables,
* which stay tiny and are indexed on the column filtered here, so each delete is a bounded
* range scan. Security-sensitive (session expiry, login rate limiting) so it must stay fresh.
*/
export async function cleanupExpiredAuthRecords(env: Env) { export async function cleanupExpiredAuthRecords(env: Env) {
const db = getDb(env); const db = getDb(env);
const now = new Date(); const now = new Date();
@@ -15,6 +20,23 @@ export async function cleanupExpiredAuthRecords(env: Env) {
await db.batch([ await db.batch([
db.delete(sessions).where(lt(sessions.expiresAt, now)), db.delete(sessions).where(lt(sessions.expiresAt, now)),
db.delete(loginAttempts).where(lt(loginAttempts.attemptedAt, new Date(now.getTime() - LOGIN_ATTEMPT_RETENTION_MS))), db.delete(loginAttempts).where(lt(loginAttempts.attemptedAt, new Date(now.getTime() - LOGIN_ATTEMPT_RETENTION_MS))),
]);
}
/**
* Retention pruning for the high-volume tables. Runs once per day from the rollup cron.
*
* At 20 monitors on 5-minute checks the `checks` table alone holds ~40k rows; running these
* deletes every 5 minutes would scan the whole table each time and blow past D1's free-tier
* "rows read" budget (~5M/day). Each column filtered below is backed by a single-column index
* so the delete only scans the expiring slice, and running daily keeps that scan to ~one day
* of rows.
*/
export async function cleanupStaleData(env: Env) {
const db = getDb(env);
const now = new Date();
await db.batch([
db.delete(checks).where(lt(checks.checkedAt, new Date(now.getTime() - CHECK_RETENTION_MS))), db.delete(checks).where(lt(checks.checkedAt, new Date(now.getTime() - CHECK_RETENTION_MS))),
db.delete(monitorDailyStats).where(lt(monitorDailyStats.day, new Date(now.getTime() - DAILY_STATS_RETENTION_MS))), db.delete(monitorDailyStats).where(lt(monitorDailyStats.day, new Date(now.getTime() - DAILY_STATS_RETENTION_MS))),
db.delete(notificationDeliveries).where(lt(notificationDeliveries.createdAt, new Date(now.getTime() - DELIVERY_RETENTION_MS))), db.delete(notificationDeliveries).where(lt(notificationDeliveries.createdAt, new Date(now.getTime() - DELIVERY_RETENTION_MS))),
+2 -2
View File
@@ -4,7 +4,7 @@ import { beforeAll, describe, expect, it } from 'vitest';
import { advanceStatus, computeImpact, nextFollowupDueAt } from '../src/worker/autopilot/cadence'; import { advanceStatus, computeImpact, nextFollowupDueAt } from '../src/worker/autopilot/cadence';
import { recordAiEvent } from '../src/worker/ai/events'; import { recordAiEvent } from '../src/worker/ai/events';
import { sanitizePublicTextWithReason } from '../src/worker/ai/sanitize'; import { sanitizePublicTextWithReason } from '../src/worker/ai/sanitize';
import { AI_EVENT_RETENTION_MS, cleanupExpiredAuthRecords } from '../src/worker/scheduled/cleanup'; import { AI_EVENT_RETENTION_MS, cleanupStaleData } from '../src/worker/scheduled/cleanup';
describe('incident autopilot', () => { describe('incident autopilot', () => {
beforeAll(async () => { beforeAll(async () => {
@@ -58,7 +58,7 @@ describe('incident autopilot', () => {
await env.DB.prepare("INSERT INTO ai_events (kind, outcome, created_at) VALUES ('settings_test', 'ok', ?)") await env.DB.prepare("INSERT INTO ai_events (kind, outcome, created_at) VALUES ('settings_test', 'ok', ?)")
.bind(Date.now() - AI_EVENT_RETENTION_MS - 1) .bind(Date.now() - AI_EVENT_RETENTION_MS - 1)
.run(); .run();
await cleanupExpiredAuthRecords(env); await cleanupStaleData(env);
expect((await env.DB.prepare('SELECT count(*) AS count FROM ai_events').first<{ count: number }>())?.count).toBe(0); expect((await env.DB.prepare('SELECT count(*) AS count FROM ai_events').first<{ count: number }>())?.count).toBe(0);
}); });
+122
View File
@@ -0,0 +1,122 @@
import { applyD1Migrations, type D1Migration } from 'cloudflare:test';
import { env } from 'cloudflare:workers';
import { beforeAll, beforeEach, describe, expect, it } from 'vitest';
import { AI_EVENT_RETENTION_MS, cleanupExpiredAuthRecords, cleanupStaleData } from '../src/worker/scheduled/cleanup';
const DAY_MS = 24 * 60 * 60 * 1000;
async function count(table: string): Promise<number> {
return (await env.DB.prepare(`SELECT count(*) AS count FROM ${table}`).first<{ count: number }>())?.count ?? 0;
}
async function insertMonitor(): Promise<number> {
const now = Date.now();
const result = await env.DB.prepare(
`INSERT INTO monitors (name, url, method, expected_status, interval_seconds, timeout_ms, enabled, alerts_enabled, created_at, updated_at)
VALUES ('Example', 'https://example.com', 'GET', 200, 300, 10000, 1, 1, ?, ?)`,
)
.bind(now, now)
.run();
return Number(result.meta.last_row_id);
}
describe('scheduled cleanup', () => {
beforeAll(async () => {
const testEnv = env as Env & { TEST_MIGRATIONS: D1Migration[] };
await applyD1Migrations(testEnv.DB, testEnv.TEST_MIGRATIONS);
});
beforeEach(async () => {
await env.DB.batch([
env.DB.prepare('DELETE FROM ai_events'),
env.DB.prepare('DELETE FROM notification_deliveries'),
env.DB.prepare('DELETE FROM notification_channels'),
env.DB.prepare('DELETE FROM monitor_daily_stats'),
env.DB.prepare('DELETE FROM checks'),
env.DB.prepare('DELETE FROM monitors'),
env.DB.prepare('DELETE FROM sessions'),
env.DB.prepare('DELETE FROM login_attempts'),
]);
});
describe('cleanupExpiredAuthRecords', () => {
it('removes expired sessions and stale login attempts but leaves fresh ones', async () => {
const now = Date.now();
await env.DB.batch([
env.DB.prepare("INSERT INTO sessions (id, expires_at, created_at) VALUES ('expired', ?, ?)").bind(now - 1000, now - DAY_MS),
env.DB.prepare("INSERT INTO sessions (id, expires_at, created_at) VALUES ('active', ?, ?)").bind(now + DAY_MS, now),
env.DB.prepare('INSERT INTO login_attempts (ip_address, attempted_at) VALUES (?, ?)').bind('1.1.1.1', now - 2 * 60 * 60 * 1000),
env.DB.prepare('INSERT INTO login_attempts (ip_address, attempted_at) VALUES (?, ?)').bind('2.2.2.2', now - 60 * 1000),
]);
await cleanupExpiredAuthRecords(env);
expect(await count('sessions')).toBe(1);
expect((await env.DB.prepare('SELECT id FROM sessions').first<{ id: string }>())?.id).toBe('active');
expect(await count('login_attempts')).toBe(1);
});
it('does not touch high-volume retention tables', async () => {
const monitorId = await insertMonitor();
await env.DB.prepare('INSERT INTO checks (monitor_id, ok, latency_ms, checked_at) VALUES (?, 1, 100, ?)')
.bind(monitorId, Date.now() - 30 * DAY_MS)
.run();
await cleanupExpiredAuthRecords(env);
expect(await count('checks')).toBe(1);
});
});
describe('cleanupStaleData', () => {
it('prunes only rows past each retention window', async () => {
const now = Date.now();
const monitorId = await insertMonitor();
const channel = await env.DB.prepare(
"INSERT INTO notification_channels (name, type, config, enabled, created_at, updated_at) VALUES ('c', 'webhook', '{}', 1, ?, ?)",
)
.bind(now, now)
.run();
const channelId = Number(channel.meta.last_row_id);
await env.DB.batch([
// checks: 7-day retention
env.DB.prepare('INSERT INTO checks (monitor_id, ok, latency_ms, checked_at) VALUES (?, 1, 100, ?)').bind(
monitorId,
now - 8 * DAY_MS,
),
env.DB.prepare('INSERT INTO checks (monitor_id, ok, latency_ms, checked_at) VALUES (?, 1, 100, ?)').bind(
monitorId,
now - 1 * DAY_MS,
),
// monitor_daily_stats: 400-day retention
env.DB.prepare('INSERT INTO monitor_daily_stats (monitor_id, day, total_checks, up_checks) VALUES (?, ?, 1, 1)').bind(
monitorId,
now - 401 * DAY_MS,
),
env.DB.prepare('INSERT INTO monitor_daily_stats (monitor_id, day, total_checks, up_checks) VALUES (?, ?, 1, 1)').bind(
monitorId,
now - 10 * DAY_MS,
),
// notification_deliveries: 30-day retention
env.DB.prepare(
"INSERT INTO notification_deliveries (channel_id, event, ok, attempts, created_at) VALUES (?, 'down', 1, 1, ?)",
).bind(channelId, now - 31 * DAY_MS),
env.DB.prepare(
"INSERT INTO notification_deliveries (channel_id, event, ok, attempts, created_at) VALUES (?, 'down', 1, 1, ?)",
).bind(channelId, now - 5 * DAY_MS),
// ai_events: 30-day retention
env.DB.prepare("INSERT INTO ai_events (kind, outcome, created_at) VALUES ('settings_test', 'ok', ?)").bind(
now - AI_EVENT_RETENTION_MS - 1,
),
env.DB.prepare("INSERT INTO ai_events (kind, outcome, created_at) VALUES ('settings_test', 'ok', ?)").bind(now - DAY_MS),
]);
await cleanupStaleData(env);
expect(await count('checks')).toBe(1);
expect(await count('monitor_daily_stats')).toBe(1);
expect(await count('notification_deliveries')).toBe(1);
expect(await count('ai_events')).toBe(1);
});
});
});
+53
View File
@@ -0,0 +1,53 @@
import { describe, expect, it } from 'vitest';
import {
DEFAULT_RUN_LIMITS,
DEFAULT_STATUS_CACHE_SECONDS,
resolveRunLimits,
resolveStatusCacheSeconds,
} from '../src/worker/lib/runtime-config';
const asEnv = (vars: Record<string, string>) => vars as unknown as Env;
describe('resolveRunLimits', () => {
it('returns the free-plan defaults when nothing is configured', () => {
expect(resolveRunLimits(asEnv({}))).toEqual(DEFAULT_RUN_LIMITS);
});
it('applies same-named environment overrides', () => {
expect(
resolveRunLimits(
asEnv({
RETRY_ATTEMPTS_PER_RUN: '60',
NOTIFICATIONS_PER_RUN: '40',
AI_CALLS_PER_RUN: '12',
AI_FOLLOWUP_CALLS_PER_RUN: '6',
}),
),
).toEqual({ retryAttemptsPerRun: 60, notificationsPerRun: 40, aiCallsPerRun: 12, aiFollowupCallsPerRun: 6 });
});
it('falls back to the default for non-positive or non-integer values', () => {
expect(resolveRunLimits(asEnv({ RETRY_ATTEMPTS_PER_RUN: '0' })).retryAttemptsPerRun).toBe(DEFAULT_RUN_LIMITS.retryAttemptsPerRun);
expect(resolveRunLimits(asEnv({ NOTIFICATIONS_PER_RUN: '-5' })).notificationsPerRun).toBe(DEFAULT_RUN_LIMITS.notificationsPerRun);
expect(resolveRunLimits(asEnv({ AI_CALLS_PER_RUN: 'lots' })).aiCallsPerRun).toBe(DEFAULT_RUN_LIMITS.aiCallsPerRun);
expect(resolveRunLimits(asEnv({ AI_FOLLOWUP_CALLS_PER_RUN: '2.5' })).aiFollowupCallsPerRun).toBe(
DEFAULT_RUN_LIMITS.aiFollowupCallsPerRun,
);
});
});
describe('resolveStatusCacheSeconds', () => {
it('defaults to DEFAULT_STATUS_CACHE_SECONDS', () => {
expect(resolveStatusCacheSeconds(asEnv({}))).toBe(DEFAULT_STATUS_CACHE_SECONDS);
});
it('accepts 0 to disable edge caching', () => {
expect(resolveStatusCacheSeconds(asEnv({ STATUS_CACHE_SECONDS: '0' }))).toBe(0);
});
it('accepts a positive override and rejects invalid values', () => {
expect(resolveStatusCacheSeconds(asEnv({ STATUS_CACHE_SECONDS: '120' }))).toBe(120);
expect(resolveStatusCacheSeconds(asEnv({ STATUS_CACHE_SECONDS: '-1' }))).toBe(DEFAULT_STATUS_CACHE_SECONDS);
expect(resolveStatusCacheSeconds(asEnv({ STATUS_CACHE_SECONDS: 'nope' }))).toBe(DEFAULT_STATUS_CACHE_SECONDS);
});
});
+3 -1
View File
@@ -8,7 +8,9 @@ export default defineConfig({
cloudflareTest({ cloudflareTest({
wrangler: { configPath: './wrangler.jsonc' }, wrangler: { configPath: './wrangler.jsonc' },
miniflare: { miniflare: {
bindings: { TEST_MIGRATIONS: migrations }, // STATUS_CACHE_SECONDS '0' disables the public-status edge cache so assertions see
// fresh D1 reads instead of a response cached by an earlier test.
bindings: { TEST_MIGRATIONS: migrations, STATUS_CACHE_SECONDS: '0' },
}, },
}), }),
], ],