From 54e3245ace948e7218a12068a71294a948a28377 Mon Sep 17 00:00:00 2001 From: hodtien Date: Fri, 3 Jul 2026 15:07:37 +0700 Subject: [PATCH] feat(usage): track cached tokens + correct input/output/cache cost (#2209) Normalize every provider to one cache-inclusive convention via canonicalizeUsage() before persist, and price cached + cache_creation as subsets of prompt_tokens in calculateCostFromTokens() to stop double-counting. usageRepo now delegates cost math to a single source. Surface Cached tokens/cost across dashboard (overview, tokens, cost, details). Merge Claude message_start cache with message_delta output so cache counts survive. Compatible LLM nodes now allow multiple API-key connections (key pool). Co-authored-by: Cursor --- open-sse/executors/kiro.js | 11 + open-sse/handlers/chatCore/requestDetail.js | 9 +- open-sse/providers/pricing.js | 6 +- open-sse/translator/concerns/usage.js | 11 +- .../translator/response/claude-to-openai.js | 40 +++- open-sse/utils/bypassHandler.js | 21 +- open-sse/utils/stream.js | 6 +- open-sse/utils/usageTracking.js | 96 +++++++++ .../usage/components/OverviewCards.js | 6 +- .../usage/components/RequestDetailsTab.js | 37 +++- .../dashboard/usage/components/UsageTable.js | 8 + src/app/api/providers/route.js | 2 + src/lib/db/repos/usageRepo.js | 81 ++++---- src/shared/components/UsageStats.js | 15 +- tests/unit/cached-token-e2e.test.js | 84 ++++++++ tests/unit/cached-token-usage.test.js | 188 ++++++++++++++++++ .../compatible-provider-connections.test.js | 8 +- 17 files changed, 558 insertions(+), 71 deletions(-) create mode 100644 tests/unit/cached-token-e2e.test.js create mode 100644 tests/unit/cached-token-usage.test.js diff --git a/open-sse/executors/kiro.js b/open-sse/executors/kiro.js index e6dde985..556e0c72 100644 --- a/open-sse/executors/kiro.js +++ b/open-sse/executors/kiro.js @@ -391,6 +391,11 @@ export class KiroExecutor extends BaseExecutor { if (metrics && typeof metrics === 'object') { const inputTokens = metrics.inputTokens || 0; const outputTokens = metrics.outputTokens || 0; + // ponytail: Amazon Q upstream does not expose cache fields today, + // but pick up cache_read_input_tokens / cache_creation_input_tokens + // if the event shape grows them so cost tracking stays accurate. + const cachedTokens = metrics.cacheReadInputTokens || metrics.cache_read_input_tokens || 0; + const cacheCreationInputTokens = metrics.cacheCreationInputTokens || metrics.cache_creation_input_tokens || 0; if (inputTokens > 0 || outputTokens > 0) { state.usage = { @@ -398,6 +403,12 @@ export class KiroExecutor extends BaseExecutor { completion_tokens: outputTokens, total_tokens: inputTokens + outputTokens }; + // Kiro is Claude-backed: inputTokens EXCLUDES cache (Claude convention), + // not inclusive like OpenAI's cached_tokens. Emit cache_read_input_tokens + // (not cached_tokens) so canonicalizeUsage takes the Claude fold path and + // correctly adds cache back into prompt_tokens instead of undercharging. + if (cachedTokens > 0) state.usage.cache_read_input_tokens = cachedTokens; + if (cacheCreationInputTokens > 0) state.usage.cache_creation_input_tokens = cacheCreationInputTokens; } } } diff --git a/open-sse/handlers/chatCore/requestDetail.js b/open-sse/handlers/chatCore/requestDetail.js index d9dde1a3..451111ad 100644 --- a/open-sse/handlers/chatCore/requestDetail.js +++ b/open-sse/handlers/chatCore/requestDetail.js @@ -1,5 +1,6 @@ import { saveRequestUsage, appendRequestLog, saveRequestDetail } from "@/lib/usageDb.js"; import { COLORS } from "../../utils/stream.js"; +import { canonicalizeUsage } from "../../utils/usageTracking.js"; const OPTIONAL_PARAMS = [ "temperature", "top_p", "top_k", @@ -48,7 +49,8 @@ export function extractUsageFromResponse(responseBody) { return { prompt_tokens: responseBody.usageMetadata.promptTokenCount || 0, completion_tokens: responseBody.usageMetadata.candidatesTokenCount || 0, - reasoning_tokens: responseBody.usageMetadata.thoughtsTokenCount + cached_tokens: responseBody.usageMetadata.cachedContentTokenCount || 0, + reasoning_tokens: responseBody.usageMetadata.thoughtsTokenCount || 0 }; } @@ -84,8 +86,9 @@ export function saveUsageStats({ provider, model, tokens, connectionId, apiKey, const accountSuffix = connectionId ? ` | account=${connectionId.slice(0, 8)}...` : ""; console.log(`${COLORS.green}[${time}] 📊 [${label}] ${provider.toUpperCase()} | in=${inTokens} | out=${outTokens}${accountSuffix}${COLORS.reset}`); - // Normalize to OpenAI token shape for storage - const normalized = { + // Canonicalize to one storage convention (prompt_tokens cache-inclusive) so + // cached/cache-creation tokens survive to cost calc + stats. See canonicalizeUsage. + const normalized = canonicalizeUsage(tokens) || { prompt_tokens: tokens.prompt_tokens ?? tokens.input_tokens ?? 0, completion_tokens: tokens.completion_tokens ?? tokens.output_tokens ?? 0 }; diff --git a/open-sse/providers/pricing.js b/open-sse/providers/pricing.js index 9e767a80..2fb0cfc9 100644 --- a/open-sse/providers/pricing.js +++ b/open-sse/providers/pricing.js @@ -279,7 +279,10 @@ export function calculateCostFromTokens(tokens, pricing) { const inputTokens = tokens.prompt_tokens || tokens.input_tokens || 0; const cachedTokens = tokens.cached_tokens || tokens.cache_read_input_tokens || 0; - const nonCachedInput = Math.max(0, inputTokens - cachedTokens); + const cacheCreationTokens = tokens.cache_creation_input_tokens || 0; + // prompt_tokens is cache-inclusive (see canonicalizeUsage): cached + cache_creation + // are subsets, so subtract both to avoid charging them at the full input rate. + const nonCachedInput = Math.max(0, inputTokens - cachedTokens - cacheCreationTokens); cost += nonCachedInput * (pricing.input / 1000000); @@ -295,7 +298,6 @@ export function calculateCostFromTokens(tokens, pricing) { cost += reasoningTokens * ((pricing.reasoning || pricing.output) / 1000000); } - const cacheCreationTokens = tokens.cache_creation_input_tokens || 0; if (cacheCreationTokens > 0) { cost += cacheCreationTokens * ((pricing.cache_creation || pricing.input) / 1000000); } diff --git a/open-sse/translator/concerns/usage.js b/open-sse/translator/concerns/usage.js index 44622901..3ace3062 100644 --- a/open-sse/translator/concerns/usage.js +++ b/open-sse/translator/concerns/usage.js @@ -39,7 +39,16 @@ const USAGE_EXTRACTORS = { }, kiro(raw) { const input = n(raw.inputTokens), output = n(raw.outputTokens); - return { promptTokens: input, completionTokens: output, totalTokens: input + output }; + // ponytail: Amazon Q (Kiro upstream) does not expose cache fields today, + // but pass through any cache_read/cache_creation/cached_tokens if the + // event shape grows them later so cost tracking keeps working without + // a second pass. + const cached = n(raw.cache_read_input_tokens) || n(raw.cachedTokens) || n(raw.cached_tokens); + const cacheCreation = n(raw.cache_creation_input_tokens); + const out = { promptTokens: input, completionTokens: output, totalTokens: input + output }; + if (cached > 0) out.cachedTokens = cached; + if (cacheCreation > 0) out.cacheCreationTokens = cacheCreation; + return out; }, ollama(raw) { const input = n(raw.prompt_eval_count), output = n(raw.eval_count); diff --git a/open-sse/translator/response/claude-to-openai.js b/open-sse/translator/response/claude-to-openai.js index 9dfd74d0..4651d3cf 100644 --- a/open-sse/translator/response/claude-to-openai.js +++ b/open-sse/translator/response/claude-to-openai.js @@ -27,6 +27,25 @@ export function claudeToOpenAIResponse(chunk, state) { state.messageId = chunk.message?.id || `msg_${Date.now()}`; state.model = chunk.message?.model; state.toolCallIndex = 0; + // Claude sends input_tokens + cache_read + cache_creation here; message_delta + // later carries only the final output_tokens. Capture cache now so the + // delta (output-only) doesn't reset it to zero. + const startUsage = chunk.message?.usage; + if (startUsage && typeof startUsage === "object") { + const inputTokens = typeof startUsage.input_tokens === "number" ? startUsage.input_tokens : 0; + const cacheReadTokens = typeof startUsage.cache_read_input_tokens === "number" ? startUsage.cache_read_input_tokens : 0; + const cacheCreationTokens = typeof startUsage.cache_creation_input_tokens === "number" ? startUsage.cache_creation_input_tokens : 0; + const promptTokens = inputTokens + cacheReadTokens + cacheCreationTokens; + state.usage = { + prompt_tokens: promptTokens, + completion_tokens: 0, + total_tokens: promptTokens, + input_tokens: inputTokens, + output_tokens: 0 + }; + if (cacheReadTokens > 0) state.usage.cache_read_input_tokens = cacheReadTokens; + if (cacheCreationTokens > 0) state.usage.cache_creation_input_tokens = cacheCreationTokens; + } results.push(createChunk(state, { role: ROLE.ASSISTANT })); break; } @@ -103,13 +122,15 @@ export function claudeToOpenAIResponse(chunk, state) { } case "message_delta": { - // Extract usage from message_delta event (Claude native format) - // Normalize to OpenAI format (prompt_tokens/completion_tokens) for consistent logging + // Extract usage from message_delta event (Claude native format). + // Anthropic sends input/cache in message_start and only output here, so + // fall back to cache captured in message_start when the delta omits it. if (chunk.usage && typeof chunk.usage === "object") { - const inputTokens = typeof chunk.usage.input_tokens === "number" ? chunk.usage.input_tokens : 0; + const prev = state.usage || {}; + const inputTokens = typeof chunk.usage.input_tokens === "number" ? chunk.usage.input_tokens : (prev.input_tokens || 0); const outputTokens = typeof chunk.usage.output_tokens === "number" ? chunk.usage.output_tokens : 0; - const cacheReadTokens = typeof chunk.usage.cache_read_input_tokens === "number" ? chunk.usage.cache_read_input_tokens : 0; - const cacheCreationTokens = typeof chunk.usage.cache_creation_input_tokens === "number" ? chunk.usage.cache_creation_input_tokens : 0; + const cacheReadTokens = typeof chunk.usage.cache_read_input_tokens === "number" ? chunk.usage.cache_read_input_tokens : (prev.cache_read_input_tokens || 0); + const cacheCreationTokens = typeof chunk.usage.cache_creation_input_tokens === "number" ? chunk.usage.cache_creation_input_tokens : (prev.cache_creation_input_tokens || 0); // prompt_tokens = input_tokens + cache_read + cache_creation (all prompt-side tokens) const promptTokens = inputTokens + cacheReadTokens + cacheCreationTokens; @@ -131,7 +152,14 @@ export function claudeToOpenAIResponse(chunk, state) { const finalChunk = createChunk(state, {}, state.finishReason); if (state.usage) { - finalChunk.usage = toOpenAIUsage(chunk.usage, "claude"); + // Build OpenAI usage from the merged state (cache from message_start + + // output from message_delta), not the delta chunk alone. + finalChunk.usage = toOpenAIUsage({ + input_tokens: state.usage.input_tokens || 0, + output_tokens: state.usage.output_tokens || 0, + cache_read_input_tokens: state.usage.cache_read_input_tokens, + cache_creation_input_tokens: state.usage.cache_creation_input_tokens + }, "claude"); } results.push(finalChunk); diff --git a/open-sse/utils/bypassHandler.js b/open-sse/utils/bypassHandler.js index 57fa2ff2..906ce724 100644 --- a/open-sse/utils/bypassHandler.js +++ b/open-sse/utils/bypassHandler.js @@ -247,9 +247,24 @@ function mergeChunksToResponse(chunks, sourceFormat) { if (messageStart?.message) { finalChunk = messageStart.message; - // Merge usage if available - if (messageDelta?.usage) { - finalChunk.usage = messageDelta.usage; + // message_start.usage has input + cache; message_delta.usage has the + // final output_tokens. Merge so cache survives (delta omits it). + const startUsage = messageStart.message.usage; + const deltaUsage = messageDelta?.usage; + if (startUsage || deltaUsage) { + finalChunk.usage = { + ...(startUsage || {}), + ...(deltaUsage || {}), + ...(startUsage?.cache_read_input_tokens !== undefined + ? { cache_read_input_tokens: startUsage.cache_read_input_tokens } + : {}), + ...(startUsage?.cache_creation_input_tokens !== undefined + ? { cache_creation_input_tokens: startUsage.cache_creation_input_tokens } + : {}), + ...(startUsage?.input_tokens !== undefined + ? { input_tokens: startUsage.input_tokens } + : {}) + }; } } } diff --git a/open-sse/utils/stream.js b/open-sse/utils/stream.js index 7ce372db..464acde4 100644 --- a/open-sse/utils/stream.js +++ b/open-sse/utils/stream.js @@ -1,7 +1,7 @@ import { translateResponse, initState } from "../translator/index.js"; import { FORMATS } from "../translator/formats.js"; import { trackPendingRequest, appendRequestLog } from "@/lib/usageDb.js"; -import { extractUsage, hasValidUsage, estimateUsage, logUsage, addBufferToUsage, filterUsageForFormat, COLORS } from "./usageTracking.js"; +import { extractUsage, mergeUsage, hasValidUsage, estimateUsage, logUsage, addBufferToUsage, filterUsageForFormat, COLORS } from "./usageTracking.js"; import { parseSSELine, hasValuableContent, fixInvalidId, formatSSE } from "./streamHelpers.js"; import { getOpenAIResponsesEventName, isOpenAIResponsesTerminalEvent, formatIncompleteOpenAIResponsesStreamFailure } from "./responsesStreamHelpers.js"; import { dbg, isDebugEnabled } from "./debugLog.js"; @@ -162,7 +162,7 @@ export function createSSEStream(options = {}) { const extracted = extractUsage(parsed); if (extracted) { - usage = extracted; + usage = mergeUsage(usage, extracted); } const isFinishChunk = parsed.choices?.[0]?.finish_reason; @@ -280,7 +280,7 @@ export function createSSEStream(options = {}) { // Extract usage const extracted = extractUsage(parsed); - if (extracted) state.usage = extracted; // Keep original usage for logging + if (extracted) state.usage = mergeUsage(state.usage, extracted); // Keep original usage for logging // Responses same-format passthrough: re-emit with original event framing if (keepsOpenAIResponsesFormat && openAIResponsesEventName) { diff --git a/open-sse/utils/usageTracking.js b/open-sse/utils/usageTracking.js index f42fcebb..663f2eaf 100644 --- a/open-sse/utils/usageTracking.js +++ b/open-sse/utils/usageTracking.js @@ -141,6 +141,68 @@ export function normalizeUsage(usage) { return normalized; } +/** + * Canonicalize usage into ONE storage/cost convention so token counts and cost + * are consistent across providers: + * prompt_tokens = total input INCLUDING cache read + cache creation + * cached_tokens = cache-read portion (subset of prompt_tokens) + * cache_creation_input_tokens = cache-write portion (subset of prompt_tokens) + * completion_tokens, reasoning_tokens, total_tokens + * + * Discriminator: Claude reports cache_read_input_tokens with a prompt that + * EXCLUDES cache, so we fold cache into prompt. OpenAI/Gemini report + * cached_tokens already counted inside prompt, so we pass through. Idempotent: + * once folded the output carries cached_tokens (not cache_read_input_tokens), + * so re-running takes the passthrough branch and does not double-add. + * + * @param {object} usage - a normalizeUsage()-shaped object + * @returns {object|null} canonical token object, or null for invalid input + */ +export function canonicalizeUsage(usage) { + if (!usage || typeof usage !== "object" || Array.isArray(usage)) return null; + + const num = (v) => (Number.isFinite(Number(v)) ? Number(v) : 0); + const completion = num(usage.completion_tokens ?? usage.output_tokens); + const reasoning = num(usage.reasoning_tokens); + // Fall back to the nested prompt_tokens_details.cache_creation_tokens shape + // (buildUsage()'s OpenAI-forwarding format) when the top-level field is + // absent, so callers that pass a buildUsage() object through don't silently + // drop cache_creation. + const cacheCreation = num(usage.cache_creation_input_tokens ?? usage.prompt_tokens_details?.cache_creation_tokens); + + let prompt = num(usage.prompt_tokens ?? usage.input_tokens); + let cached; + + // Claude path: prompt excludes cache; cache_read_input_tokens and/or + // cache_creation_input_tokens are separate. A cache-miss "first write" only + // carries cache_creation_input_tokens (no cache_read_input_tokens yet), so + // check both fields — otherwise a first-write request falls through to the + // OpenAI passthrough branch below and cache_creation never gets folded in. + // Guard on the absence of `cached_tokens`: our own canonical output always + // sets that key (even to 0), so re-running canonicalizeUsage on an already- + // folded result takes the passthrough branch instead of folding again. + if (usage.cached_tokens === undefined && + (usage.cache_read_input_tokens !== undefined || usage.cache_creation_input_tokens !== undefined)) { + cached = num(usage.cache_read_input_tokens); + prompt = prompt + cached + cacheCreation; + } else { + // OpenAI/Gemini path (or already-canonical input): prompt already includes cached_tokens. + cached = num(usage.cached_tokens); + } + + const result = { + prompt_tokens: prompt, + completion_tokens: completion, + // Recompute rather than pass through: when the fold branch ran above, + // an upstream total_tokens (cache-exclusive) would otherwise be stale. + total_tokens: prompt + completion, + cached_tokens: cached, + cache_creation_input_tokens: cacheCreation, + }; + if (reasoning > 0) result.reasoning_tokens = reasoning; + return result; +} + /** * Check if usage has valid token data * Valid = has at least one token field with value > 0 @@ -171,6 +233,19 @@ export function hasValidUsage(usage) { export function extractUsage(chunk) { if (!chunk || typeof chunk !== "object") return null; + // Claude format (message_start event): carries input_tokens + cache_read + + // cache_creation. message_delta later carries only the final output_tokens, + // so callers must MERGE (mergeUsage), not overwrite, to keep cache counts. + if (chunk.type === "message_start" && chunk.message?.usage && typeof chunk.message.usage === "object") { + const u = chunk.message.usage; + return normalizeUsage({ + prompt_tokens: u.input_tokens || 0, + completion_tokens: u.output_tokens || 0, + cache_read_input_tokens: u.cache_read_input_tokens, + cache_creation_input_tokens: u.cache_creation_input_tokens + }); + } + // Claude format (message_delta event) if (chunk.type === "message_delta" && chunk.usage && typeof chunk.usage === "object") { return normalizeUsage({ @@ -232,6 +307,27 @@ export function extractUsage(chunk) { return null; } +// Field-wise max-merge of two usage objects. Anthropic splits usage across +// events: message_start has real input+cache (output is a placeholder 1), +// message_delta has the real cumulative output (input/cache absent). Max keeps +// the meaningful value from each without clobbering. Idempotent for other +// providers that emit a single complete usage object. +export function mergeUsage(prev, next) { + if (!prev) return next || null; + if (!next) return prev; + const merged = { ...prev }; + for (const [k, v] of Object.entries(next)) { + // typeof NaN === "number" — guard with Number.isFinite so one malformed + // chunk can't poison the whole accumulation (Math.max(x, NaN) is NaN). + if (typeof v === "number" && Number.isFinite(v)) { + merged[k] = Math.max(typeof merged[k] === "number" ? merged[k] : 0, v); + } else if (v && typeof v === "object") { + merged[k] = v; // nested details objects: take latest + } + } + return merged; +} + /** * Estimate input tokens from request body * Calculate total body size for more accurate estimation diff --git a/src/app/(dashboard)/dashboard/usage/components/OverviewCards.js b/src/app/(dashboard)/dashboard/usage/components/OverviewCards.js index 5d08933d..22020ff2 100644 --- a/src/app/(dashboard)/dashboard/usage/components/OverviewCards.js +++ b/src/app/(dashboard)/dashboard/usage/components/OverviewCards.js @@ -8,7 +8,7 @@ const fmtCost = (n) => `$${(n || 0).toFixed(2)}`; export default function OverviewCards({ stats }) { return ( -
+
Total Requests {fmt(stats.totalRequests)} @@ -17,6 +17,10 @@ export default function OverviewCards({ stats }) { Total Input Tokens {fmt(stats.totalPromptTokens)} + + Cached Tokens + {fmt(stats.totalCachedTokens)} + Output Tokens {fmt(stats.totalCompletionTokens)} diff --git a/src/app/(dashboard)/dashboard/usage/components/RequestDetailsTab.js b/src/app/(dashboard)/dashboard/usage/components/RequestDetailsTab.js index e450dd94..b82b8286 100644 --- a/src/app/(dashboard)/dashboard/usage/components/RequestDetailsTab.js +++ b/src/app/(dashboard)/dashboard/usage/components/RequestDetailsTab.js @@ -82,9 +82,20 @@ function CollapsibleSection({ title, children, defaultOpen = false, icon = null ); } +function getCachedTokens(tokens) { + return tokens?.cached_tokens || tokens?.cache_read_input_tokens || 0; +} + +function getCacheCreationTokens(tokens) { + return tokens?.cache_creation_input_tokens || 0; +} + function getInputTokens(tokens) { const prompt = tokens?.prompt_tokens || tokens?.input_tokens || 0; - const cache = tokens?.cached_tokens || tokens?.cache_read_input_tokens || 0; + // Canonical storage keeps prompt cache-inclusive. Legacy Claude rows may have + // stored prompt cache-exclusive; fall back to cache when it's larger so old + // rows don't under-report input. + const cache = getCachedTokens(tokens); return prompt < cache ? cache : prompt; } @@ -245,6 +256,8 @@ export default function RequestDetailsTab() { Model Provider Input Tokens + Cached + Cache Creation Output Tokens Latency Action @@ -286,6 +299,12 @@ export default function RequestDetailsTab() { {getInputTokens(detail.tokens).toLocaleString()} + + {getCachedTokens(detail.tokens) > 0 ? getCachedTokens(detail.tokens).toLocaleString() : "—"} + + + {getCacheCreationTokens(detail.tokens) > 0 ? getCacheCreationTokens(detail.tokens).toLocaleString() : "—"} + {detail.tokens?.completion_tokens?.toLocaleString() || 0} @@ -370,6 +389,22 @@ export default function RequestDetailsTab() { {getInputTokens(selectedDetail.tokens).toLocaleString()}
+ {getCachedTokens(selectedDetail.tokens) > 0 && ( +
+ Cached Tokens:{" "} + + {getCachedTokens(selectedDetail.tokens).toLocaleString()} + +
+ )} + {getCacheCreationTokens(selectedDetail.tokens) > 0 && ( +
+ Cache Creation:{" "} + + {getCacheCreationTokens(selectedDetail.tokens).toLocaleString()} + +
+ )}
Output Tokens:{" "} diff --git a/src/app/(dashboard)/dashboard/usage/components/UsageTable.js b/src/app/(dashboard)/dashboard/usage/components/UsageTable.js index 9f3d3099..9da3c3e6 100644 --- a/src/app/(dashboard)/dashboard/usage/components/UsageTable.js +++ b/src/app/(dashboard)/dashboard/usage/components/UsageTable.js @@ -38,6 +38,9 @@ function ValueCells({ item, viewMode, isSummary = false }) { {isSummary && item.promptTokens === undefined ? "—" : fmt(item.promptTokens)} + + {item.cachedTokens ? fmt(item.cachedTokens) : "—"} + {isSummary && item.completionTokens === undefined ? "—" : fmt(item.completionTokens)} @@ -52,6 +55,9 @@ function ValueCells({ item, viewMode, isSummary = false }) { {isSummary && item.inputCost === undefined ? "—" : fmtCost(item.inputCost)} + + {item.cachedCost ? fmtCost(item.cachedCost) : "—"} + {isSummary && item.outputCost === undefined ? "—" : fmtCost(item.outputCost)} @@ -133,12 +139,14 @@ export default function UsageTable({ if (viewMode === "tokens") { return [ { field: "promptTokens", label: "Input Tokens" }, + { field: "cachedTokens", label: "Cached" }, { field: "completionTokens", label: "Output Tokens" }, { field: "totalTokens", label: "Total Tokens" }, ]; } return [ { field: "promptTokens", label: "Input Cost" }, + { field: "cachedCost", label: "Cached Cost" }, { field: "completionTokens", label: "Output Cost" }, { field: "cost", label: "Total Cost" }, ]; diff --git a/src/app/api/providers/route.js b/src/app/api/providers/route.js index 7e1842c8..5885472b 100644 --- a/src/app/api/providers/route.js +++ b/src/app/api/providers/route.js @@ -126,6 +126,8 @@ export async function POST(request) { let providerSpecificData = normalizeProviderSpecificData(provider, body, body.providerSpecificData); + // Compatible LLM nodes support multiple API-key connections (key pool); runtime + // rotates/fails over via getProviderCredentials. Embedding nodes stay single-connection. if (isOpenAICompatibleProvider(provider)) { const node = await getProviderNodeById(provider); if (!node) { diff --git a/src/lib/db/repos/usageRepo.js b/src/lib/db/repos/usageRepo.js index b0d6bff0..ce6c4761 100644 --- a/src/lib/db/repos/usageRepo.js +++ b/src/lib/db/repos/usageRepo.js @@ -51,10 +51,11 @@ function getLocalDateKey(timestamp) { } function addToCounter(target, key, values) { - if (!target[key]) target[key] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0 }; + if (!target[key]) target[key] = { requests: 0, promptTokens: 0, completionTokens: 0, cachedTokens: 0, cost: 0 }; target[key].requests += values.requests || 1; target[key].promptTokens += values.promptTokens || 0; target[key].completionTokens += values.completionTokens || 0; + target[key].cachedTokens += values.cachedTokens || 0; target[key].cost += values.cost || 0; if (values.meta) Object.assign(target[key], values.meta); } @@ -62,12 +63,14 @@ function addToCounter(target, key, values) { function aggregateEntryToDay(day, entry) { const promptTokens = entry.tokens?.prompt_tokens || entry.tokens?.input_tokens || 0; const completionTokens = entry.tokens?.completion_tokens || entry.tokens?.output_tokens || 0; + const cachedTokens = entry.tokens?.cached_tokens || entry.tokens?.cache_read_input_tokens || 0; const cost = entry.cost || 0; - const vals = { promptTokens, completionTokens, cost }; + const vals = { promptTokens, completionTokens, cachedTokens, cost }; day.requests = (day.requests || 0) + 1; day.promptTokens = (day.promptTokens || 0) + promptTokens; day.completionTokens = (day.completionTokens || 0) + completionTokens; + day.cachedTokens = (day.cachedTokens || 0) + cachedTokens; day.cost = (day.cost || 0) + cost; day.byProvider ||= {}; @@ -135,33 +138,11 @@ async function calculateCost(provider, model, tokens) { const pricing = await getPricingForModel(provider, model); if (!pricing) return 0; - let cost = 0; - const inputTokens = tokens.prompt_tokens || tokens.input_tokens || 0; - const cachedTokens = tokens.cached_tokens || tokens.cache_read_input_tokens || 0; - const nonCachedInput = Math.max(0, inputTokens - cachedTokens); - cost += nonCachedInput * (pricing.input / 1000000); - - if (cachedTokens > 0) { - const cachedRate = pricing.cached || pricing.input; - cost += cachedTokens * (cachedRate / 1000000); - } - - const outputTokens = tokens.completion_tokens || tokens.output_tokens || 0; - cost += outputTokens * (pricing.output / 1000000); - - const reasoningTokens = tokens.reasoning_tokens || 0; - if (reasoningTokens > 0) { - const rate = pricing.reasoning || pricing.output; - cost += reasoningTokens * (rate / 1000000); - } - - const cacheCreationTokens = tokens.cache_creation_input_tokens || 0; - if (cacheCreationTokens > 0) { - const rate = pricing.cache_creation || pricing.input; - cost += cacheCreationTokens * (rate / 1000000); - } - - return cost; + // Delegate the actual math to the single source of truth (avoids the two + // copies drifting apart — see open-sse/providers/pricing.js for the + // cache-inclusive prompt_tokens convention this assumes). + const { calculateCostFromTokens } = await import("open-sse/providers/pricing.js"); + return calculateCostFromTokens(tokens, pricing); } catch (e) { console.error("Error calculating cost:", e); return 0; @@ -398,6 +379,7 @@ export async function getUsageStats(period = "all") { timestamp: r.timestamp, model: r.model, provider: r.provider || "", promptTokens: t.prompt_tokens || t.input_tokens || 0, completionTokens: t.completion_tokens || t.output_tokens || 0, + cachedTokens: t.cached_tokens || t.cache_read_input_tokens || 0, status: r.status || "ok", }; }) @@ -413,7 +395,7 @@ export async function getUsageStats(period = "all") { const stats = { totalRequests: 0, - totalPromptTokens: 0, totalCompletionTokens: 0, totalCost: 0, + totalPromptTokens: 0, totalCompletionTokens: 0, totalCachedTokens: 0, totalCost: 0, byProvider: {}, byModel: {}, byAccount: {}, byApiKey: {}, byEndpoint: {}, last10Minutes: [], pending: pendingRequests, @@ -474,13 +456,15 @@ export async function getUsageStats(period = "all") { const day = parseJson(dr.data, {}); stats.totalPromptTokens += day.promptTokens || 0; stats.totalCompletionTokens += day.completionTokens || 0; + stats.totalCachedTokens += day.cachedTokens || 0; stats.totalCost += day.cost || 0; for (const [prov, p] of Object.entries(day.byProvider || {})) { - if (!stats.byProvider[prov]) stats.byProvider[prov] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0 }; + if (!stats.byProvider[prov]) stats.byProvider[prov] = { requests: 0, promptTokens: 0, completionTokens: 0, cachedTokens: 0, cost: 0 }; stats.byProvider[prov].requests += p.requests || 0; stats.byProvider[prov].promptTokens += p.promptTokens || 0; stats.byProvider[prov].completionTokens += p.completionTokens || 0; + stats.byProvider[prov].cachedTokens += p.cachedTokens || 0; stats.byProvider[prov].cost += p.cost || 0; } @@ -490,11 +474,12 @@ export async function getUsageStats(period = "all") { const statsKey = provider ? `${rawModel} (${provider})` : rawModel; const providerDisplayName = providerNodeNameMap[provider] || provider; if (!stats.byModel[statsKey]) { - stats.byModel[statsKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel, provider: providerDisplayName, lastUsed: dateKey }; + stats.byModel[statsKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cachedTokens: 0, cost: 0, rawModel, provider: providerDisplayName, lastUsed: dateKey }; } stats.byModel[statsKey].requests += m.requests || 0; stats.byModel[statsKey].promptTokens += m.promptTokens || 0; stats.byModel[statsKey].completionTokens += m.completionTokens || 0; + stats.byModel[statsKey].cachedTokens += m.cachedTokens || 0; stats.byModel[statsKey].cost += m.cost || 0; if (dateKey > (stats.byModel[statsKey].lastUsed || "")) stats.byModel[statsKey].lastUsed = dateKey; } @@ -506,11 +491,12 @@ export async function getUsageStats(period = "all") { const providerDisplayName = providerNodeNameMap[provider] || provider; const accountKey = `${rawModel} (${provider} - ${accountName})`; if (!stats.byAccount[accountKey]) { - stats.byAccount[accountKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel, provider: providerDisplayName, connectionId: connId, accountName, lastUsed: dateKey }; + stats.byAccount[accountKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cachedTokens: 0, cost: 0, rawModel, provider: providerDisplayName, connectionId: connId, accountName, lastUsed: dateKey }; } stats.byAccount[accountKey].requests += a.requests || 0; stats.byAccount[accountKey].promptTokens += a.promptTokens || 0; stats.byAccount[accountKey].completionTokens += a.completionTokens || 0; + stats.byAccount[accountKey].cachedTokens += a.cachedTokens || 0; stats.byAccount[accountKey].cost += a.cost || 0; if (dateKey > (stats.byAccount[accountKey].lastUsed || "")) stats.byAccount[accountKey].lastUsed = dateKey; } @@ -525,11 +511,12 @@ export async function getUsageStats(period = "all") { const apiKeyMasked = maskApiKey(apiKeyVal); const apiKeyKey = apiKeyMasked || "local-no-key"; if (!stats.byApiKey[akKey]) { - stats.byApiKey[akKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel, provider: providerDisplayName, apiKeyMasked, keyName, apiKeyKey, lastUsed: dateKey }; + stats.byApiKey[akKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cachedTokens: 0, cost: 0, rawModel, provider: providerDisplayName, apiKeyMasked, keyName, apiKeyKey, lastUsed: dateKey }; } stats.byApiKey[akKey].requests += ak.requests || 0; stats.byApiKey[akKey].promptTokens += ak.promptTokens || 0; stats.byApiKey[akKey].completionTokens += ak.completionTokens || 0; + stats.byApiKey[akKey].cachedTokens += ak.cachedTokens || 0; stats.byApiKey[akKey].cost += ak.cost || 0; if (dateKey > (stats.byApiKey[akKey].lastUsed || "")) stats.byApiKey[akKey].lastUsed = dateKey; } @@ -540,11 +527,12 @@ export async function getUsageStats(period = "all") { const provider = ep.provider || ""; const providerDisplayName = providerNodeNameMap[provider] || provider; if (!stats.byEndpoint[epKey]) { - stats.byEndpoint[epKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, endpoint, rawModel, provider: providerDisplayName, lastUsed: dateKey }; + stats.byEndpoint[epKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cachedTokens: 0, cost: 0, endpoint, rawModel, provider: providerDisplayName, lastUsed: dateKey }; } stats.byEndpoint[epKey].requests += ep.requests || 0; stats.byEndpoint[epKey].promptTokens += ep.promptTokens || 0; stats.byEndpoint[epKey].completionTokens += ep.completionTokens || 0; + stats.byEndpoint[epKey].cachedTokens += ep.cachedTokens || 0; stats.byEndpoint[epKey].cost += ep.cost || 0; if (dateKey > (stats.byEndpoint[epKey].lastUsed || "")) stats.byEndpoint[epKey].lastUsed = dateKey; } @@ -595,26 +583,30 @@ export async function getUsageStats(period = "all") { const tokens = parseJson(r.tokens, {}) || {}; const promptTokens = tokens.prompt_tokens || 0; const completionTokens = tokens.completion_tokens || 0; + const cachedTokens = tokens.cached_tokens || tokens.cache_read_input_tokens || 0; const entryCost = r.cost || 0; const providerDisplayName = providerNodeNameMap[r.provider] || r.provider; stats.totalPromptTokens += promptTokens; stats.totalCompletionTokens += completionTokens; + stats.totalCachedTokens += cachedTokens; stats.totalCost += entryCost; - if (!stats.byProvider[r.provider]) stats.byProvider[r.provider] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0 }; + if (!stats.byProvider[r.provider]) stats.byProvider[r.provider] = { requests: 0, promptTokens: 0, completionTokens: 0, cachedTokens: 0, cost: 0 }; stats.byProvider[r.provider].requests++; stats.byProvider[r.provider].promptTokens += promptTokens; stats.byProvider[r.provider].completionTokens += completionTokens; + stats.byProvider[r.provider].cachedTokens += cachedTokens; stats.byProvider[r.provider].cost += entryCost; const modelKey = r.provider ? `${r.model} (${r.provider})` : r.model; if (!stats.byModel[modelKey]) { - stats.byModel[modelKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel: r.model, provider: providerDisplayName, lastUsed: r.timestamp }; + stats.byModel[modelKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cachedTokens: 0, cost: 0, rawModel: r.model, provider: providerDisplayName, lastUsed: r.timestamp }; } stats.byModel[modelKey].requests++; stats.byModel[modelKey].promptTokens += promptTokens; stats.byModel[modelKey].completionTokens += completionTokens; + stats.byModel[modelKey].cachedTokens += cachedTokens; stats.byModel[modelKey].cost += entryCost; if (new Date(r.timestamp) > new Date(stats.byModel[modelKey].lastUsed)) stats.byModel[modelKey].lastUsed = r.timestamp; @@ -622,11 +614,12 @@ export async function getUsageStats(period = "all") { const accountName = connectionMap[r.connectionId] || `Account ${r.connectionId.slice(0, 8)}...`; const accountKey = `${r.model} (${r.provider} - ${accountName})`; if (!stats.byAccount[accountKey]) { - stats.byAccount[accountKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel: r.model, provider: providerDisplayName, connectionId: r.connectionId, accountName, lastUsed: r.timestamp }; + stats.byAccount[accountKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cachedTokens: 0, cost: 0, rawModel: r.model, provider: providerDisplayName, connectionId: r.connectionId, accountName, lastUsed: r.timestamp }; } stats.byAccount[accountKey].requests++; stats.byAccount[accountKey].promptTokens += promptTokens; stats.byAccount[accountKey].completionTokens += completionTokens; + stats.byAccount[accountKey].cachedTokens += cachedTokens; stats.byAccount[accountKey].cost += entryCost; if (new Date(r.timestamp) > new Date(stats.byAccount[accountKey].lastUsed)) stats.byAccount[accountKey].lastUsed = r.timestamp; } @@ -637,27 +630,27 @@ export async function getUsageStats(period = "all") { const apiKeyMasked = maskApiKey(r.apiKey); const akKey = `${apiKeyMasked}|${r.model}|${r.provider || "unknown"}`; if (!stats.byApiKey[akKey]) { - stats.byApiKey[akKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel: r.model, provider: providerDisplayName, apiKeyMasked, keyName, apiKeyKey: apiKeyMasked, lastUsed: r.timestamp }; + stats.byApiKey[akKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cachedTokens: 0, cost: 0, rawModel: r.model, provider: providerDisplayName, apiKeyMasked, keyName, apiKeyKey: apiKeyMasked, lastUsed: r.timestamp }; } const ake = stats.byApiKey[akKey]; - ake.requests++; ake.promptTokens += promptTokens; ake.completionTokens += completionTokens; ake.cost += entryCost; + ake.requests++; ake.promptTokens += promptTokens; ake.completionTokens += completionTokens; ake.cachedTokens += cachedTokens; ake.cost += entryCost; if (new Date(r.timestamp) > new Date(ake.lastUsed)) ake.lastUsed = r.timestamp; } else { if (!stats.byApiKey["local-no-key"]) { - stats.byApiKey["local-no-key"] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel: r.model, provider: providerDisplayName, apiKeyMasked: null, keyName: "Local (No API Key)", apiKeyKey: "local-no-key", lastUsed: r.timestamp }; + stats.byApiKey["local-no-key"] = { requests: 0, promptTokens: 0, completionTokens: 0, cachedTokens: 0, cost: 0, rawModel: r.model, provider: providerDisplayName, apiKeyMasked: null, keyName: "Local (No API Key)", apiKeyKey: "local-no-key", lastUsed: r.timestamp }; } const ake = stats.byApiKey["local-no-key"]; - ake.requests++; ake.promptTokens += promptTokens; ake.completionTokens += completionTokens; ake.cost += entryCost; + ake.requests++; ake.promptTokens += promptTokens; ake.completionTokens += completionTokens; ake.cachedTokens += cachedTokens; ake.cost += entryCost; if (new Date(r.timestamp) > new Date(ake.lastUsed)) ake.lastUsed = r.timestamp; } const endpoint = r.endpoint || "Unknown"; const epKey = `${endpoint}|${r.model}|${r.provider || "unknown"}`; if (!stats.byEndpoint[epKey]) { - stats.byEndpoint[epKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, endpoint, rawModel: r.model, provider: providerDisplayName, lastUsed: r.timestamp }; + stats.byEndpoint[epKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cachedTokens: 0, cost: 0, endpoint, rawModel: r.model, provider: providerDisplayName, lastUsed: r.timestamp }; } const epe = stats.byEndpoint[epKey]; - epe.requests++; epe.promptTokens += promptTokens; epe.completionTokens += completionTokens; epe.cost += entryCost; + epe.requests++; epe.promptTokens += promptTokens; epe.completionTokens += completionTokens; epe.cachedTokens += cachedTokens; epe.cost += entryCost; if (new Date(r.timestamp) > new Date(epe.lastUsed)) epe.lastUsed = r.timestamp; } } diff --git a/src/shared/components/UsageStats.js b/src/shared/components/UsageStats.js index 950a7af9..607cbbc0 100644 --- a/src/shared/components/UsageStats.js +++ b/src/shared/components/UsageStats.js @@ -89,9 +89,16 @@ function sortData(dataMap, pendingMap = {}, sortBy, sortOrder) { .map(([key, data]) => { const totalTokens = (data.promptTokens || 0) + (data.completionTokens || 0); const totalCost = data.cost || 0; - const inputCost = totalTokens > 0 ? (data.promptTokens || 0) * (totalCost / totalTokens) : 0; + // ponytail: cost split is a token-share allocation of the (rate-accurate) + // server total, not a per-rate recompute. cached is a subset of prompt, so + // peel it out of the input share. Upgrade to a stored per-component cost + // breakdown if exact cached-rate cost display is needed. + const cachedTokens = data.cachedTokens || 0; + const nonCachedInput = Math.max(0, (data.promptTokens || 0) - cachedTokens); + const inputCost = totalTokens > 0 ? nonCachedInput * (totalCost / totalTokens) : 0; + const cachedCost = totalTokens > 0 ? cachedTokens * (totalCost / totalTokens) : 0; const outputCost = totalTokens > 0 ? (data.completionTokens || 0) * (totalCost / totalTokens) : 0; - return { ...data, key, totalTokens, totalCost, inputCost, outputCost, pending: pendingMap[key] || 0 }; + return { ...data, key, totalTokens, totalCost, inputCost, cachedCost, outputCost, pending: pendingMap[key] || 0 }; }) .sort((a, b) => { let valA = a[sortBy]; @@ -122,7 +129,7 @@ function groupDataByKey(data, keyField) { if (!groups[gk]) { groups[gk] = { groupKey: gk, - summary: { requests: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0, cost: 0, inputCost: 0, outputCost: 0, lastUsed: null, pending: 0 }, + summary: { requests: 0, promptTokens: 0, completionTokens: 0, cachedTokens: 0, totalTokens: 0, cost: 0, inputCost: 0, cachedCost: 0, outputCost: 0, lastUsed: null, pending: 0 }, items: [], }; } @@ -130,9 +137,11 @@ function groupDataByKey(data, keyField) { s.requests += item.requests || 0; s.promptTokens += item.promptTokens || 0; s.completionTokens += item.completionTokens || 0; + s.cachedTokens += item.cachedTokens || 0; s.totalTokens += item.totalTokens || 0; s.cost += item.cost || 0; s.inputCost += item.inputCost || 0; + s.cachedCost += item.cachedCost || 0; s.outputCost += item.outputCost || 0; s.pending += item.pending || 0; if (item.lastUsed && (!s.lastUsed || new Date(item.lastUsed) > new Date(s.lastUsed))) { diff --git a/tests/unit/cached-token-e2e.test.js b/tests/unit/cached-token-e2e.test.js new file mode 100644 index 00000000..f32f6a3e --- /dev/null +++ b/tests/unit/cached-token-e2e.test.js @@ -0,0 +1,84 @@ +// End-to-end: a cache-bearing request flows through canonicalizeUsage → +// saveRequestUsage → getUsageStats, proving cached tokens are persisted, +// aggregated, and cost is computed correctly (the bug this branch fixes). +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, it, expect, beforeAll, afterAll, vi } from "vitest"; +import { canonicalizeUsage } from "../../open-sse/utils/usageTracking.js"; + +const originalDataDir = process.env.DATA_DIR; +let tempDir; +let db; + +beforeAll(async () => { + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "9router-cached-e2e-")); + process.env.DATA_DIR = tempDir; + vi.resetModules(); + db = await import("@/lib/db/index.js"); + await db.initDb(); +}); + +afterAll(() => { + if (tempDir) fs.rmSync(tempDir, { recursive: true, force: true }); + if (originalDataDir === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = originalDataDir; +}); + +describe("cached-token end-to-end (persist + aggregate + cost)", () => { + it("Claude cache usage: canonical prompt is inclusive, cached persisted, cost correct", async () => { + // Raw Claude usage (cache-EXCLUSIVE prompt): input 100, cache_read 200, cache_creation 30, output 50 + const canonical = canonicalizeUsage({ + prompt_tokens: 100, + completion_tokens: 50, + cache_read_input_tokens: 200, + cache_creation_input_tokens: 30, + }); + expect(canonical.prompt_tokens).toBe(330); // inclusive + + await db.saveRequestUsage({ + provider: "anthropic", + model: "claude-sonnet-4-6", + connectionId: "c-cache", + tokens: canonical, + endpoint: "/v1/messages", + status: "ok", + }); + + const stats = await db.getUsageStats("24h"); + expect(stats.totalCachedTokens).toBe(200); + expect(stats.totalPromptTokens).toBe(330); + expect(stats.byProvider.anthropic.cachedTokens).toBe(200); + + // Cost: nonCached=330-200-30=100 @3 + cached 200 @0.30 + creation 30 @3.75 + output 50 @15 + const expected = (100 * 3 + 200 * 0.3 + 30 * 3.75 + 50 * 15) / 1_000_000; + const hist = await db.getUsageHistory({ provider: "anthropic" }); + expect(hist.length).toBe(1); + expect(hist[0].cost).toBeCloseTo(expected, 12); + expect(hist[0].tokens.cached_tokens).toBe(200); + expect(hist[0].tokens.cache_creation_input_tokens).toBe(30); + }); + + it("OpenAI cache usage: inclusive prompt passes through, cached counted once", async () => { + const canonical = canonicalizeUsage({ + prompt_tokens: 1000, // already includes cached + completion_tokens: 200, + cached_tokens: 600, + }); + expect(canonical.prompt_tokens).toBe(1000); + expect(canonical.cached_tokens).toBe(600); + + await db.saveRequestUsage({ + provider: "openai", + model: "gpt-4o", + connectionId: "c-oai", + tokens: canonical, + endpoint: "/v1/chat/completions", + status: "ok", + }); + + const hist = await db.getUsageHistory({ provider: "openai" }); + expect(hist[0].tokens.prompt_tokens).toBe(1000); + expect(hist[0].tokens.cached_tokens).toBe(600); + }); +}); diff --git a/tests/unit/cached-token-usage.test.js b/tests/unit/cached-token-usage.test.js new file mode 100644 index 00000000..878110d0 --- /dev/null +++ b/tests/unit/cached-token-usage.test.js @@ -0,0 +1,188 @@ +import { describe, it, expect } from "vitest"; +import { canonicalizeUsage, extractUsage, mergeUsage } from "../../open-sse/utils/usageTracking.js"; +import { calculateCostFromTokens } from "../../open-sse/providers/pricing.js"; +import { toOpenAIUsage } from "../../open-sse/translator/concerns/usage.js"; + +// Canonical convention (single source of truth for storage + cost): +// prompt_tokens = total input INCLUDING cache read + cache creation +// cached_tokens = cache-read portion (subset of prompt_tokens) +// cache_creation_input_tokens = cache-write portion (subset of prompt_tokens) +// completion_tokens = output +// Discriminator: Claude reports cache separately (prompt EXCLUDES cache); +// OpenAI/Gemini report prompt INCLUDING cached_tokens. +describe("canonicalizeUsage", () => { + it("folds Claude exclusive cache into an inclusive prompt count", () => { + // Claude: input_tokens excludes cache; cache_read + cache_creation are separate + const out = canonicalizeUsage({ + prompt_tokens: 100, + completion_tokens: 50, + cache_read_input_tokens: 200, + cache_creation_input_tokens: 30, + }); + expect(out.prompt_tokens).toBe(330); // 100 + 200 + 30 + expect(out.completion_tokens).toBe(50); + expect(out.cached_tokens).toBe(200); + expect(out.cache_creation_input_tokens).toBe(30); + }); + + it("passes through OpenAI inclusive prompt unchanged", () => { + // OpenAI: prompt_tokens already includes cached_tokens (a subset) + const out = canonicalizeUsage({ + prompt_tokens: 330, + completion_tokens: 50, + cached_tokens: 200, + }); + expect(out.prompt_tokens).toBe(330); + expect(out.cached_tokens).toBe(200); + expect(out.cache_creation_input_tokens).toBe(0); + }); + + it("passes through Gemini inclusive prompt (cachedContent already counted)", () => { + const out = canonicalizeUsage({ + prompt_tokens: 500, + completion_tokens: 80, + cached_tokens: 120, + reasoning_tokens: 40, + }); + expect(out.prompt_tokens).toBe(500); + expect(out.cached_tokens).toBe(120); + expect(out.reasoning_tokens).toBe(40); + }); + + it("handles no-cache usage", () => { + const out = canonicalizeUsage({ prompt_tokens: 100, completion_tokens: 50 }); + expect(out.prompt_tokens).toBe(100); + expect(out.cached_tokens).toBe(0); + expect(out.cache_creation_input_tokens).toBe(0); + }); + + it("is idempotent (running twice yields the same canonical shape)", () => { + const once = canonicalizeUsage({ + prompt_tokens: 100, + completion_tokens: 50, + cache_read_input_tokens: 200, + cache_creation_input_tokens: 30, + }); + const twice = canonicalizeUsage(once); + expect(twice.prompt_tokens).toBe(330); + expect(twice.cached_tokens).toBe(200); + expect(twice.cache_creation_input_tokens).toBe(30); + expect(twice.completion_tokens).toBe(50); + }); + + it("returns null for invalid input", () => { + expect(canonicalizeUsage(null)).toBeNull(); + expect(canonicalizeUsage(undefined)).toBeNull(); + }); + + it("folds a Claude cache-miss first write (cache_creation only, no cache_read yet)", () => { + // Cache-miss on first write: upstream emits cache_creation_input_tokens but + // no cache_read_input_tokens at all (not even 0). Must still fold into prompt + // instead of falling through to the OpenAI passthrough branch. + const out = canonicalizeUsage({ + prompt_tokens: 100, + completion_tokens: 20, + cache_creation_input_tokens: 500, + }); + expect(out.prompt_tokens).toBe(600); // 100 + 0 (no read) + 500 + expect(out.cached_tokens).toBe(0); + expect(out.cache_creation_input_tokens).toBe(500); + }); +}); + +describe("calculateCostFromTokens (canonical inclusive convention)", () => { + const pricing = { input: 3, output: 15, cached: 0.3, cache_creation: 3.75 }; + + it("prices cached + cache_creation as subsets of an inclusive prompt without double-counting", () => { + // prompt=330 includes 200 cached + 30 cache_creation → 100 full-price input + const cost = calculateCostFromTokens( + { prompt_tokens: 330, completion_tokens: 50, cached_tokens: 200, cache_creation_input_tokens: 30 }, + pricing + ); + const expected = + (100 * 3 + 200 * 0.3 + 30 * 3.75 + 50 * 15) / 1_000_000; + expect(cost).toBeCloseTo(expected, 12); + }); + + it("does not let cache_creation drive nonCached negative", () => { + // pathological: cached + creation exceeds prompt → nonCached clamps at 0 + const cost = calculateCostFromTokens( + { prompt_tokens: 100, completion_tokens: 0, cached_tokens: 80, cache_creation_input_tokens: 40 }, + pricing + ); + const expected = (0 * 3 + 80 * 0.3 + 40 * 3.75) / 1_000_000; + expect(cost).toBeCloseTo(expected, 12); + }); + + it("matches plain input pricing when no cache present", () => { + const cost = calculateCostFromTokens({ prompt_tokens: 100, completion_tokens: 50 }, pricing); + expect(cost).toBeCloseTo((100 * 3 + 50 * 15) / 1_000_000, 12); + }); +}); + +describe("Anthropic streaming usage (message_start carries cache, message_delta output-only)", () => { + it("extractUsage reads input + cache from message_start", () => { + const u = extractUsage({ + type: "message_start", + message: { usage: { input_tokens: 100, output_tokens: 1, cache_read_input_tokens: 200, cache_creation_input_tokens: 30 } }, + }); + expect(u.prompt_tokens).toBe(100); + expect(u.cache_read_input_tokens).toBe(200); + expect(u.cache_creation_input_tokens).toBe(30); + }); + + it("merges message_start cache with message_delta output without clobbering", () => { + // Real Anthropic SSE: cache only in message_start, real output only in message_delta. + const start = extractUsage({ + type: "message_start", + message: { usage: { input_tokens: 100, output_tokens: 1, cache_read_input_tokens: 200, cache_creation_input_tokens: 30 } }, + }); + const delta = extractUsage({ type: "message_delta", usage: { output_tokens: 50 } }); + const merged = mergeUsage(start, delta); + expect(merged.prompt_tokens).toBe(100); + expect(merged.cache_read_input_tokens).toBe(200); + expect(merged.cache_creation_input_tokens).toBe(30); + expect(merged.completion_tokens).toBe(50); + + // And it canonicalizes to a cache-inclusive prompt for storage/cost. + const canon = canonicalizeUsage(merged); + expect(canon.prompt_tokens).toBe(330); // 100 + 200 + 30 + expect(canon.cached_tokens).toBe(200); + expect(canon.cache_creation_input_tokens).toBe(30); + expect(canon.completion_tokens).toBe(50); + }); + + it("does not let a NaN field poison the running max-merge", () => { + // typeof NaN === "number", so a naive Math.max(prev, NaN) is NaN — one + // malformed chunk must not wipe out an already-accumulated good value. + const prev = { prompt_tokens: 100, cache_read_input_tokens: 200 }; + const bad = { prompt_tokens: NaN, completion_tokens: 50 }; + const merged = mergeUsage(prev, bad); + expect(merged.prompt_tokens).toBe(100); + expect(merged.cache_read_input_tokens).toBe(200); + expect(merged.completion_tokens).toBe(50); + }); +}); + +describe("Kiro usage pass-through", () => { + it("passes through plain input/output when no cache fields are present", () => { + const out = toOpenAIUsage({ inputTokens: 100, outputTokens: 50 }, "kiro"); + expect(out.prompt_tokens).toBe(100); + expect(out.completion_tokens).toBe(50); + expect(out.total_tokens).toBe(150); + expect(out.prompt_tokens_details).toBeUndefined(); + }); + + it("forward-compat: surfaces cache fields if Kiro event shape grows them", () => { + // ponytail: Amazon Q upstream doesn't expose cache today, but if it starts + // sending cache_read_input_tokens / cache_creation_input_tokens / cachedTokens, + // cost tracking should pick them up automatically without another change. + const out = toOpenAIUsage( + { inputTokens: 500, outputTokens: 100, cache_read_input_tokens: 200, cache_creation_input_tokens: 50 }, + "kiro" + ); + expect(out.prompt_tokens_details).toBeDefined(); + expect(out.prompt_tokens_details.cached_tokens).toBe(200); + expect(out.prompt_tokens_details.cache_creation_tokens).toBe(50); + }); +}); diff --git a/tests/unit/compatible-provider-connections.test.js b/tests/unit/compatible-provider-connections.test.js index d41d4657..0fe146f0 100644 --- a/tests/unit/compatible-provider-connections.test.js +++ b/tests/unit/compatible-provider-connections.test.js @@ -38,14 +38,14 @@ async function setupTestContext(nodeData) { }; } -function makeRequest(provider) { +function makeRequest(provider, name = "Test Connection") { return new Request("https://9router.local/api/providers", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ provider, apiKey: "test-key", - name: "Test Connection", + name, defaultModel: "test-model", }), }); @@ -156,8 +156,8 @@ describe("compatible provider connections API", () => { }); cleanup = ctx.cleanup; - const firstResponse = await ctx.POST(makeRequest(ctx.node.id)); - const secondResponse = await ctx.POST(makeRequest(ctx.node.id)); + const firstResponse = await ctx.POST(makeRequest(ctx.node.id, "Key A")); + const secondResponse = await ctx.POST(makeRequest(ctx.node.id, "Key B")); const storedConnections = await ctx.getProviderConnections({ provider: ctx.node.id }); expect(firstResponse.status).toBe(201);