mirror of
https://github.com/Nezumi-2711/9router.git
synced 2026-09-22 13:38:31 +00:00
fix: DB backup on schema change, MCP child cleanup, codex models, usage providers OOM
- Backup DB only on real SCHEMA_VERSION change, not every app version bump - Kill idle MCP stdio bridge children to prevent orphan process leaks - Add getDistinctProviders to avoid loading every row JSON blob (OOM fix) - Update codex model list (gpt-5.6 sol/terra/luna, drop 5.3 codex variants) - Reorder Claude default models (fable first) Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { getRequestDetails } from "@/lib/requestDetailsDb";
|
||||
import { getDistinctProviders } from "@/lib/requestDetailsDb";
|
||||
import { getProviderNodes } from "@/lib/localDb";
|
||||
import { AI_PROVIDERS, getProviderByAlias } from "@/shared/constants/providers";
|
||||
|
||||
@@ -9,10 +9,9 @@ import { AI_PROVIDERS, getProviderByAlias } from "@/shared/constants/providers";
|
||||
*/
|
||||
export async function GET() {
|
||||
try {
|
||||
const { details } = await getRequestDetails({ pageSize: 9999 });
|
||||
|
||||
// Extract unique providers
|
||||
const providerIds = [...new Set(details.map(r => r.provider).filter(Boolean))].sort();
|
||||
// Query DISTINCT provider column directly — avoids parsing every row's
|
||||
// full JSON blob (can be hundreds of MB), which previously caused OOM.
|
||||
const providerIds = await getDistinctProviders();
|
||||
|
||||
const providerNodes = await getProviderNodes();
|
||||
const nodeMap = {};
|
||||
|
||||
@@ -9,8 +9,10 @@ export async function GET(request) {
|
||||
try {
|
||||
const { searchParams } = new URL(request.url);
|
||||
|
||||
const page = parseInt(searchParams.get("page")) || 1;
|
||||
const pageSize = parseInt(searchParams.get("pageSize")) || 20;
|
||||
const pageRaw = parseInt(searchParams.get("page"));
|
||||
const page = Number.isNaN(pageRaw) ? 1 : pageRaw;
|
||||
const pageSizeRaw = parseInt(searchParams.get("pageSize"));
|
||||
const pageSize = Number.isNaN(pageSizeRaw) ? 20 : pageSizeRaw;
|
||||
const provider = searchParams.get("provider");
|
||||
const model = searchParams.get("model");
|
||||
const connectionId = searchParams.get("connectionId");
|
||||
|
||||
+41
-1
@@ -1,9 +1,20 @@
|
||||
// DB safety backups — taken ONLY before a schema change (see migrate.js).
|
||||
//
|
||||
// ⚠️ AGENT/DEV NOTES:
|
||||
// - Backups are a best-effort safety net before schema migrations. There is NO
|
||||
// automated restore path; recovery is manual (copy a backup file back).
|
||||
// - Backups intentionally EXCLUDE the `requestDetails` table (observability log,
|
||||
// auto-pruned, non-critical) so a multi-hundred-MB DB backs up as a few MB.
|
||||
// - Only the newest KEEP_BACKUPS are kept; older ones are pruned automatically.
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { BACKUPS_DIR, ensureDirs } from "./paths.js";
|
||||
import { timestampSlug, getAppVersion } from "./version.js";
|
||||
|
||||
const KEEP_BACKUPS = 5;
|
||||
const KEEP_BACKUPS = 3;
|
||||
|
||||
// Tables excluded from safety backups (large, non-critical, reproducible).
|
||||
const BACKUP_EXCLUDE_TABLES = ["requestDetails"];
|
||||
|
||||
export function makeBackupDir(label) {
|
||||
ensureDirs();
|
||||
@@ -22,6 +33,35 @@ export function backupFile(srcPath, destDir, destName = null) {
|
||||
return dest;
|
||||
}
|
||||
|
||||
// Lightweight DB backup via ATTACH: create an empty sqlite file, copy every
|
||||
// table EXCEPT the excluded ones into it. Avoids duplicating the huge
|
||||
// observability log, so the backup stays small regardless of DB size.
|
||||
export function backupDbLite(adapter, destDir, destName = "data.sqlite") {
|
||||
const dest = path.join(destDir, destName);
|
||||
try { fs.rmSync(dest, { force: true }); } catch {}
|
||||
const escaped = dest.replace(/'/g, "''");
|
||||
|
||||
adapter.exec(`ATTACH DATABASE '${escaped}' AS bak`);
|
||||
try {
|
||||
const excluded = new Set(BACKUP_EXCLUDE_TABLES);
|
||||
const tables = adapter
|
||||
.all(`SELECT name, sql FROM main.sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'`)
|
||||
.filter((t) => !excluded.has(t.name));
|
||||
|
||||
adapter.transaction(() => {
|
||||
for (const t of tables) {
|
||||
// Recreate table structure in backup DB, then copy rows.
|
||||
const createSql = t.sql.replace(/CREATE TABLE\s+/i, "CREATE TABLE bak.");
|
||||
adapter.exec(createSql);
|
||||
adapter.exec(`INSERT INTO bak.${t.name} SELECT * FROM main.${t.name}`);
|
||||
}
|
||||
});
|
||||
} finally {
|
||||
try { adapter.exec("DETACH DATABASE bak"); } catch {}
|
||||
}
|
||||
return dest;
|
||||
}
|
||||
|
||||
export function pruneOldBackups() {
|
||||
if (!fs.existsSync(BACKUPS_DIR)) return;
|
||||
const entries = fs.readdirSync(BACKUPS_DIR, { withFileTypes: true })
|
||||
|
||||
+1
-1
@@ -64,7 +64,7 @@ export {
|
||||
|
||||
// Request details
|
||||
export {
|
||||
saveRequestDetail, getRequestDetails, getRequestDetailById,
|
||||
saveRequestDetail, getRequestDetails, getRequestDetailById, getDistinctProviders,
|
||||
} from "./repos/requestDetailsRepo.js";
|
||||
|
||||
// Export/import full DB
|
||||
|
||||
+33
-22
@@ -1,10 +1,10 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { LEGACY_FILES, DB_DIR, DATA_FILE } from "./paths.js";
|
||||
import { TABLES, buildCreateTableSql } from "./schema.js";
|
||||
import { LEGACY_FILES, DB_DIR } from "./paths.js";
|
||||
import { TABLES, buildCreateTableSql, SCHEMA_VERSION } from "./schema.js";
|
||||
import { MIGRATIONS, latestVersion } from "./migrations/index.js";
|
||||
import { getMetaSync, setMetaSync } from "./helpers/metaStore.js";
|
||||
import { makeBackupDir, backupFile, pruneOldBackups } from "./backup.js";
|
||||
import { makeBackupDir, backupFile, backupDbLite, pruneOldBackups } from "./backup.js";
|
||||
import { getAppVersion } from "./version.js";
|
||||
import { stringifyJson } from "./helpers/jsonCol.js";
|
||||
|
||||
@@ -221,12 +221,37 @@ export async function runMigrationOnce(adapter) {
|
||||
// a brand-new DB as non-fresh once schemaVersion is written).
|
||||
const fresh = isFreshDb(adapter);
|
||||
|
||||
// Prune stale backups every boot so old oversized backups shrink to KEEP.
|
||||
pruneOldBackups();
|
||||
|
||||
// Bootstrap _meta so we can read the stored backup schema version below
|
||||
// (runVersionedMigrations also ensures this, but we need it earlier here).
|
||||
adapter.exec(buildCreateTableSql("_meta", TABLES._meta));
|
||||
|
||||
// Detect a pending schema change via the central SCHEMA_VERSION const.
|
||||
// A lightweight backup is taken BEFORE any schema mutation below.
|
||||
const storedSchemaVer = parseInt(getMetaSync(adapter, "backupSchemaVersion", "0"), 10) || 0;
|
||||
const schemaChanging = !fresh && storedSchemaVer < SCHEMA_VERSION;
|
||||
if (schemaChanging) {
|
||||
try {
|
||||
const backupDir = makeBackupDir(`schema-${storedSchemaVer}-to-${SCHEMA_VERSION}`);
|
||||
backupDbLite(adapter, backupDir);
|
||||
pruneOldBackups();
|
||||
console.log(`[DB][migrate] pre-schema backup ${storedSchemaVer} → ${SCHEMA_VERSION}: ${backupDir}`);
|
||||
} catch (e) {
|
||||
console.warn(`[DB][migrate] pre-schema backup failed (continuing): ${e.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 1. Always run versioned migrations chain (skip-version safe)
|
||||
const migInfo = runVersionedMigrations(adapter);
|
||||
|
||||
// 2. Additive sync (auto add missing columns/indexes declared in TABLES)
|
||||
syncSchemaFromTables(adapter);
|
||||
|
||||
// Stamp the schema version we just reached so future boots skip re-backup.
|
||||
setMetaSync(adapter, "backupSchemaVersion", SCHEMA_VERSION);
|
||||
|
||||
// 3. One-time legacy JSON import (only if DB was fresh on entry)
|
||||
const alreadyImported = fs.existsSync(MIGRATED_MARKER);
|
||||
const legacyMain = readJsonSafe(LEGACY_FILES.main);
|
||||
@@ -247,6 +272,7 @@ export async function runMigrationOnce(adapter) {
|
||||
importLegacyDisabled(adapter, legacyDisabled);
|
||||
importLegacyDetails(adapter, legacyDetails);
|
||||
setMetaSync(adapter, "appVersion", getAppVersion());
|
||||
setMetaSync(adapter, "backupSchemaVersion", SCHEMA_VERSION);
|
||||
setMetaSync(adapter, "migratedAt", new Date().toISOString());
|
||||
});
|
||||
} catch (err) {
|
||||
@@ -263,24 +289,9 @@ export async function runMigrationOnce(adapter) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (fresh) {
|
||||
setMetaSync(adapter, "appVersion", getAppVersion());
|
||||
return;
|
||||
}
|
||||
|
||||
// 4. App version bump → backup data.sqlite (safety net before user-side upgrade)
|
||||
const oldVer = getMetaSync(adapter, "appVersion", null);
|
||||
// Track app version for informational purposes only. App version bumps no
|
||||
// longer trigger a DB backup — only real schema changes (SCHEMA_VERSION) do.
|
||||
const newVer = getAppVersion();
|
||||
if (oldVer && oldVer !== newVer) {
|
||||
const backupDir = makeBackupDir(`upgrade-${oldVer}-to-${newVer}`);
|
||||
try { backupFile(DATA_FILE, backupDir); } catch {}
|
||||
setMetaSync(adapter, "appVersion", newVer);
|
||||
pruneOldBackups();
|
||||
console.log(`[DB][migrate] App ${oldVer} → ${newVer} | schema ${migInfo.from} → ${migInfo.to} | backup: ${backupDir}`);
|
||||
} else if (migInfo.applied > 0) {
|
||||
// Schema upgrade without app version bump — still backup
|
||||
const backupDir = makeBackupDir(`schema-${migInfo.from}-to-${migInfo.to}`);
|
||||
try { backupFile(DATA_FILE, backupDir); } catch {}
|
||||
pruneOldBackups();
|
||||
}
|
||||
const oldVer = getMetaSync(adapter, "appVersion", null);
|
||||
if (oldVer !== newVer) setMetaSync(adapter, "appVersion", newVer);
|
||||
}
|
||||
|
||||
@@ -174,6 +174,12 @@ export async function getRequestDetails(filter = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
export async function getDistinctProviders() {
|
||||
const db = await getAdapter();
|
||||
const rows = db.all(`SELECT DISTINCT provider FROM requestDetails WHERE provider IS NOT NULL ORDER BY provider ASC`);
|
||||
return rows.map((r) => r.provider);
|
||||
}
|
||||
|
||||
export async function getRequestDetailById(id) {
|
||||
const db = await getAdapter();
|
||||
const row = db.get(`SELECT data FROM requestDetails WHERE id = ?`, [id]);
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
// Latest schema version — bumped when a migration is added in ./migrations/
|
||||
// ⚠️ AGENT/DEV: Bump this by +1 EVERY TIME you change the schema below
|
||||
// (add/remove/alter a table, column, or index in TABLES). It drives the
|
||||
// pre-change safety backup in migrate.js: when the stored version is lower,
|
||||
// one lightweight DB backup is taken before applying schema changes. Forgetting
|
||||
// to bump only skips that backup — it does NOT break the additive auto-sync.
|
||||
export const SCHEMA_VERSION = 1;
|
||||
|
||||
export const PRAGMA_SQL = `
|
||||
|
||||
@@ -153,6 +153,20 @@ function unregisterSession(name, sid) {
|
||||
const entry = getStore().get(name);
|
||||
if (!entry) return;
|
||||
entry.sessions.delete(sid);
|
||||
// No sessions left → kill child to avoid idle orphan process leak.
|
||||
if (entry.sessions.size === 0) {
|
||||
try { entry.proc.kill(); } catch { /* ignore */ }
|
||||
getStore().delete(name);
|
||||
}
|
||||
}
|
||||
|
||||
// Kill all spawned MCP children — called on app shutdown to prevent orphans.
|
||||
function killAllBridges() {
|
||||
const store = getStore();
|
||||
for (const [name, entry] of store) {
|
||||
try { entry.proc.kill(); } catch { /* ignore */ }
|
||||
store.delete(name);
|
||||
}
|
||||
}
|
||||
|
||||
function sendToChild(name, jsonRpc) {
|
||||
@@ -166,4 +180,4 @@ function isRunning(name) {
|
||||
return !!(entry?.proc && !entry.proc.killed && entry.proc.exitCode === null);
|
||||
}
|
||||
|
||||
module.exports = { getOrSpawn, registerSession, unregisterSession, sendToChild, isRunning, findPlugin };
|
||||
module.exports = { getOrSpawn, registerSession, unregisterSession, sendToChild, isRunning, findPlugin, killAllBridges };
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Shim → re-export from new SQLite-based DB layer (src/lib/db/)
|
||||
export {
|
||||
saveRequestDetail, getRequestDetails, getRequestDetailById,
|
||||
saveRequestDetail, getRequestDetails, getRequestDetailById, getDistinctProviders,
|
||||
} from "@/lib/db/index.js";
|
||||
|
||||
@@ -101,9 +101,9 @@ export const CLI_TOOLS = {
|
||||
modelAliases: ["default", "sonnet", "opus", "fable", "haiku", "opusplan"],
|
||||
settingsFile: "~/.claude/settings.json",
|
||||
defaultModels: [
|
||||
{ id: "fable", name: "Claude Fable", alias: "fable", envKey: "ANTHROPIC_DEFAULT_FABLE_MODEL", defaultValue: "cc/claude-fable-5" },
|
||||
{ id: "opus", name: "Claude Opus", alias: "opus", envKey: "ANTHROPIC_DEFAULT_OPUS_MODEL", defaultValue: "cc/claude-opus-4-8" },
|
||||
{ id: "sonnet", name: "Claude Sonnet", alias: "sonnet", envKey: "ANTHROPIC_DEFAULT_SONNET_MODEL", defaultValue: "cc/claude-sonnet-5" },
|
||||
{ id: "fable", name: "Claude Fable", alias: "fable", envKey: "ANTHROPIC_DEFAULT_FABLE_MODEL", defaultValue: "cc/claude-fable-5" },
|
||||
{ id: "haiku", name: "Claude Haiku", alias: "haiku", envKey: "ANTHROPIC_DEFAULT_HAIKU_MODEL", defaultValue: "cc/claude-haiku-4-5-20251001" },
|
||||
],
|
||||
},
|
||||
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
import { getMitmStatus, startMitm, loadEncryptedPassword, initDbHooks, restoreToolDNS, removeAllDNSEntriesSync } from "@/mitm/manager";
|
||||
import { startQuotaAutoPing } from "@/shared/services/quotaAutoPing";
|
||||
import { syncToJson as syncMitmAliasCache } from "@/lib/mitmAliasCache";
|
||||
import { killAllBridges } from "@/lib/mcp/stdioSseBridge";
|
||||
|
||||
// Inject correct paths and DB hooks into manager.js (CJS) from ESM context
|
||||
(function bootstrapMitm() {
|
||||
@@ -56,6 +57,7 @@ export async function initializeApp() {
|
||||
if (!g.signalHandlersRegistered) {
|
||||
const cleanup = () => {
|
||||
try { removeAllDNSEntriesSync(); } catch { /* best effort */ }
|
||||
try { killAllBridges(); } catch { /* best effort */ }
|
||||
killCloudflared();
|
||||
process.exit();
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user