mirror of
https://github.com/Nezumi-2711/9router.git
synced 2026-09-22 13:38:31 +00:00
fix(headroom): clarify token diagnostics vs provider billing
Distinguish Headroom-reported token deltas from outbound payload size, scrub credentials in logs, and warn on phantom savings when compressed JSON barely shrinks. Refs #1998 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Sutarto Jordan Chrisfivo
committed by
decolua
decolua
co-authored by
Cursor
Cursor
parent
c7933de79c
commit
fb543a1f39
+136
-20
@@ -3,39 +3,135 @@ import { openaiToClaudeRequest } from "../translator/request/openai-to-claude.js
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 3000;
|
||||
|
||||
function jsonBytes(value) {
|
||||
try {
|
||||
return new TextEncoder().encode(JSON.stringify(value) || "").length;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
function messagePayload(body) {
|
||||
if (Array.isArray(body?.messages)) return body.messages;
|
||||
if (Array.isArray(body?.input)) return body.input;
|
||||
return null;
|
||||
}
|
||||
|
||||
function captureSizeSnapshot(body) {
|
||||
const messages = messagePayload(body);
|
||||
return {
|
||||
bodyBytes: jsonBytes(body),
|
||||
messageBytes: messages ? jsonBytes(messages) : 0,
|
||||
};
|
||||
}
|
||||
|
||||
function setDiagnostic(diagnostics, reason) {
|
||||
if (diagnostics && !diagnostics.reason) diagnostics.reason = reason;
|
||||
}
|
||||
|
||||
function scrubSensitiveUrlText(text) {
|
||||
return String(text)
|
||||
.replace(/\/\/[^/@\s]+@/g, "//")
|
||||
.replace(/(https?:\/\/[^\s?#]+)[?#][^\s)]*/g, "$1");
|
||||
}
|
||||
|
||||
function describeFetchError(error) {
|
||||
const cause = error?.cause;
|
||||
const code = cause?.code || error?.code;
|
||||
const message = scrubSensitiveUrlText(cause?.message || error?.message || String(error));
|
||||
return code ? `${code}: ${message}` : message;
|
||||
}
|
||||
|
||||
function buildCompressEndpoint(url) {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
parsed.pathname = `${parsed.pathname.replace(/\/$/, "")}/v1/compress`;
|
||||
parsed.hash = "";
|
||||
return parsed.toString();
|
||||
} catch {
|
||||
const raw = String(url).replace(/#.*$/, "");
|
||||
const [base, query = ""] = raw.split("?", 2);
|
||||
const endpoint = `${base.replace(/\/$/, "")}/v1/compress`;
|
||||
return query ? `${endpoint}?${query}` : endpoint;
|
||||
}
|
||||
}
|
||||
|
||||
function maskEndpoint(endpoint) {
|
||||
try {
|
||||
const parsed = new URL(endpoint);
|
||||
parsed.username = "";
|
||||
parsed.password = "";
|
||||
parsed.search = "";
|
||||
parsed.hash = "";
|
||||
return parsed.toString();
|
||||
} catch {
|
||||
return String(endpoint).replace(/\/\/[^/@\s]+@/, "//").replace(/[?#].*$/, "");
|
||||
}
|
||||
}
|
||||
|
||||
// POST messages to Headroom /v1/compress; returns compressed messages + stats or null.
|
||||
async function callCompress(url, messages, model, timeoutMs, compressUserMessages) {
|
||||
const endpoint = `${String(url).replace(/\/$/, "")}/v1/compress`;
|
||||
async function callCompress(url, messages, model, timeoutMs, compressUserMessages, diagnostics) {
|
||||
const endpoint = buildCompressEndpoint(url);
|
||||
diagnostics.endpoint = maskEndpoint(endpoint);
|
||||
const payload = { messages, model };
|
||||
if (compressUserMessages) payload.config = { compress_user_messages: true };
|
||||
const res = await fetch(endpoint, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(payload),
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
});
|
||||
if (!res.ok) return null;
|
||||
let res;
|
||||
try {
|
||||
res = await fetch(endpoint, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(payload),
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
});
|
||||
} catch (error) {
|
||||
setDiagnostic(diagnostics, `request failed: ${describeFetchError(error)}`);
|
||||
return null;
|
||||
}
|
||||
if (!res.ok) {
|
||||
setDiagnostic(diagnostics, `proxy returned HTTP ${res.status}`);
|
||||
return null;
|
||||
}
|
||||
const data = await res.json();
|
||||
if (!Array.isArray(data?.messages)) return null;
|
||||
if (!Array.isArray(data?.messages)) {
|
||||
setDiagnostic(diagnostics, "proxy response missing messages[]");
|
||||
return null;
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
// Compress request body via Headroom proxy. Fail-open: returns null on any error.
|
||||
// /v1/compress only understands OpenAI shape, so Claude bodies are translated
|
||||
// to OpenAI, compressed, then translated back using 9Router's own translators.
|
||||
export async function compressWithHeadroom(body, { enabled, url, model, format, compressUserMessages, timeoutMs = DEFAULT_TIMEOUT_MS } = {}) {
|
||||
if (!enabled || !url || !body) return null;
|
||||
export async function compressWithHeadroom(body, { enabled, url, model, format, compressUserMessages, timeoutMs = DEFAULT_TIMEOUT_MS, diagnostics = null } = {}) {
|
||||
if (!enabled) {
|
||||
setDiagnostic(diagnostics, "disabled");
|
||||
return null;
|
||||
}
|
||||
if (!url) {
|
||||
setDiagnostic(diagnostics, "missing proxy URL");
|
||||
return null;
|
||||
}
|
||||
if (!body) {
|
||||
setDiagnostic(diagnostics, "missing request body");
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
if (diagnostics) diagnostics.before = captureSizeSnapshot(body);
|
||||
|
||||
// Claude shape: translate → OpenAI → compress → translate back.
|
||||
if (format === "claude") {
|
||||
const oai = claudeToOpenAIRequest(model, body, false);
|
||||
if (!Array.isArray(oai?.messages)) return null;
|
||||
const data = await callCompress(url, oai.messages, model, timeoutMs, compressUserMessages);
|
||||
if (!Array.isArray(oai?.messages)) {
|
||||
setDiagnostic(diagnostics, "Claude request did not translate to messages[]");
|
||||
return null;
|
||||
}
|
||||
const data = await callCompress(url, oai.messages, model, timeoutMs, compressUserMessages, diagnostics || {});
|
||||
if (!data) return null;
|
||||
const claudeBody = openaiToClaudeRequest(model, { ...oai, messages: data.messages }, false);
|
||||
if (Array.isArray(claudeBody?.messages)) body.messages = claudeBody.messages;
|
||||
if (claudeBody?.system !== undefined) body.system = claudeBody.system;
|
||||
if (diagnostics) diagnostics.after = captureSizeSnapshot(body);
|
||||
return data;
|
||||
}
|
||||
|
||||
@@ -43,12 +139,17 @@ export async function compressWithHeadroom(body, { enabled, url, model, format,
|
||||
const key = Array.isArray(body.messages) ? "messages"
|
||||
: Array.isArray(body.input) ? "input"
|
||||
: null;
|
||||
if (!key) return null;
|
||||
const data = await callCompress(url, body[key], model, timeoutMs, compressUserMessages);
|
||||
if (!key) {
|
||||
setDiagnostic(diagnostics, `unsupported ${format || "unknown"} request shape`);
|
||||
return null;
|
||||
}
|
||||
const data = await callCompress(url, body[key], model, timeoutMs, compressUserMessages, diagnostics || {});
|
||||
if (!data) return null;
|
||||
body[key] = data.messages;
|
||||
if (diagnostics) diagnostics.after = captureSizeSnapshot(body);
|
||||
return data;
|
||||
} catch {
|
||||
} catch (error) {
|
||||
setDiagnostic(diagnostics, `unexpected error: ${error?.message || String(error)}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -57,7 +158,22 @@ export function formatHeadroomLog(stats) {
|
||||
if (!stats) return null;
|
||||
const before = stats.tokens_before || 0;
|
||||
const after = stats.tokens_after || 0;
|
||||
const saved = stats.tokens_saved || 0;
|
||||
const pct = before > 0 ? ((saved / before) * 100).toFixed(1) : "0";
|
||||
return `saved ${saved} tokens / ${before} (${pct}%) ${after ? `after=${after}` : ""}`.trim();
|
||||
const delta = stats.tokens_saved || 0;
|
||||
const pct = before > 0 ? ((delta / before) * 100).toFixed(1) : "0";
|
||||
return `reported token delta=${delta} before=${before}${after ? ` after=${after}` : ""} (${pct}%)`.trim();
|
||||
}
|
||||
|
||||
export function formatHeadroomSizeLog(diagnostics) {
|
||||
const before = diagnostics?.before;
|
||||
const after = diagnostics?.after;
|
||||
if (!before || !after) return "";
|
||||
return `body=${before.bodyBytes}B→${after.bodyBytes}B messages=${before.messageBytes}B→${after.messageBytes}B`;
|
||||
}
|
||||
|
||||
export function isHeadroomPhantomSavings(stats, diagnostics, minShrinkRatio = 0.05) {
|
||||
if (!stats?.tokens_saved || stats.tokens_saved <= 0) return false;
|
||||
const before = diagnostics?.before?.bodyBytes || 0;
|
||||
const after = diagnostics?.after?.bodyBytes || 0;
|
||||
if (before <= 0 || after <= 0) return false;
|
||||
return after >= before * (1 - minShrinkRatio);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user