mirror of
https://github.com/Nezumi-2711/9router.git
synced 2026-09-22 13:38:31 +00:00
feat(usage): track cached tokens + correct input/output/cache cost (#2209)
Normalize every provider to one cache-inclusive convention via canonicalizeUsage() before persist, and price cached + cache_creation as subsets of prompt_tokens in calculateCostFromTokens() to stop double-counting. usageRepo now delegates cost math to a single source. Surface Cached tokens/cost across dashboard (overview, tokens, cost, details). Merge Claude message_start cache with message_delta output so cache counts survive. Compatible LLM nodes now allow multiple API-key connections (key pool). Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
hodtien
committed by
decolua
decolua
co-authored by
Cursor
Cursor
parent
960f8a0379
commit
54e3245ace
@@ -391,6 +391,11 @@ export class KiroExecutor extends BaseExecutor {
|
||||
if (metrics && typeof metrics === 'object') {
|
||||
const inputTokens = metrics.inputTokens || 0;
|
||||
const outputTokens = metrics.outputTokens || 0;
|
||||
// ponytail: Amazon Q upstream does not expose cache fields today,
|
||||
// but pick up cache_read_input_tokens / cache_creation_input_tokens
|
||||
// if the event shape grows them so cost tracking stays accurate.
|
||||
const cachedTokens = metrics.cacheReadInputTokens || metrics.cache_read_input_tokens || 0;
|
||||
const cacheCreationInputTokens = metrics.cacheCreationInputTokens || metrics.cache_creation_input_tokens || 0;
|
||||
|
||||
if (inputTokens > 0 || outputTokens > 0) {
|
||||
state.usage = {
|
||||
@@ -398,6 +403,12 @@ export class KiroExecutor extends BaseExecutor {
|
||||
completion_tokens: outputTokens,
|
||||
total_tokens: inputTokens + outputTokens
|
||||
};
|
||||
// Kiro is Claude-backed: inputTokens EXCLUDES cache (Claude convention),
|
||||
// not inclusive like OpenAI's cached_tokens. Emit cache_read_input_tokens
|
||||
// (not cached_tokens) so canonicalizeUsage takes the Claude fold path and
|
||||
// correctly adds cache back into prompt_tokens instead of undercharging.
|
||||
if (cachedTokens > 0) state.usage.cache_read_input_tokens = cachedTokens;
|
||||
if (cacheCreationInputTokens > 0) state.usage.cache_creation_input_tokens = cacheCreationInputTokens;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { saveRequestUsage, appendRequestLog, saveRequestDetail } from "@/lib/usageDb.js";
|
||||
import { COLORS } from "../../utils/stream.js";
|
||||
import { canonicalizeUsage } from "../../utils/usageTracking.js";
|
||||
|
||||
const OPTIONAL_PARAMS = [
|
||||
"temperature", "top_p", "top_k",
|
||||
@@ -48,7 +49,8 @@ export function extractUsageFromResponse(responseBody) {
|
||||
return {
|
||||
prompt_tokens: responseBody.usageMetadata.promptTokenCount || 0,
|
||||
completion_tokens: responseBody.usageMetadata.candidatesTokenCount || 0,
|
||||
reasoning_tokens: responseBody.usageMetadata.thoughtsTokenCount
|
||||
cached_tokens: responseBody.usageMetadata.cachedContentTokenCount || 0,
|
||||
reasoning_tokens: responseBody.usageMetadata.thoughtsTokenCount || 0
|
||||
};
|
||||
}
|
||||
|
||||
@@ -84,8 +86,9 @@ export function saveUsageStats({ provider, model, tokens, connectionId, apiKey,
|
||||
const accountSuffix = connectionId ? ` | account=${connectionId.slice(0, 8)}...` : "";
|
||||
console.log(`${COLORS.green}[${time}] 📊 [${label}] ${provider.toUpperCase()} | in=${inTokens} | out=${outTokens}${accountSuffix}${COLORS.reset}`);
|
||||
|
||||
// Normalize to OpenAI token shape for storage
|
||||
const normalized = {
|
||||
// Canonicalize to one storage convention (prompt_tokens cache-inclusive) so
|
||||
// cached/cache-creation tokens survive to cost calc + stats. See canonicalizeUsage.
|
||||
const normalized = canonicalizeUsage(tokens) || {
|
||||
prompt_tokens: tokens.prompt_tokens ?? tokens.input_tokens ?? 0,
|
||||
completion_tokens: tokens.completion_tokens ?? tokens.output_tokens ?? 0
|
||||
};
|
||||
|
||||
@@ -279,7 +279,10 @@ export function calculateCostFromTokens(tokens, pricing) {
|
||||
|
||||
const inputTokens = tokens.prompt_tokens || tokens.input_tokens || 0;
|
||||
const cachedTokens = tokens.cached_tokens || tokens.cache_read_input_tokens || 0;
|
||||
const nonCachedInput = Math.max(0, inputTokens - cachedTokens);
|
||||
const cacheCreationTokens = tokens.cache_creation_input_tokens || 0;
|
||||
// prompt_tokens is cache-inclusive (see canonicalizeUsage): cached + cache_creation
|
||||
// are subsets, so subtract both to avoid charging them at the full input rate.
|
||||
const nonCachedInput = Math.max(0, inputTokens - cachedTokens - cacheCreationTokens);
|
||||
|
||||
cost += nonCachedInput * (pricing.input / 1000000);
|
||||
|
||||
@@ -295,7 +298,6 @@ export function calculateCostFromTokens(tokens, pricing) {
|
||||
cost += reasoningTokens * ((pricing.reasoning || pricing.output) / 1000000);
|
||||
}
|
||||
|
||||
const cacheCreationTokens = tokens.cache_creation_input_tokens || 0;
|
||||
if (cacheCreationTokens > 0) {
|
||||
cost += cacheCreationTokens * ((pricing.cache_creation || pricing.input) / 1000000);
|
||||
}
|
||||
|
||||
@@ -39,7 +39,16 @@ const USAGE_EXTRACTORS = {
|
||||
},
|
||||
kiro(raw) {
|
||||
const input = n(raw.inputTokens), output = n(raw.outputTokens);
|
||||
return { promptTokens: input, completionTokens: output, totalTokens: input + output };
|
||||
// ponytail: Amazon Q (Kiro upstream) does not expose cache fields today,
|
||||
// but pass through any cache_read/cache_creation/cached_tokens if the
|
||||
// event shape grows them later so cost tracking keeps working without
|
||||
// a second pass.
|
||||
const cached = n(raw.cache_read_input_tokens) || n(raw.cachedTokens) || n(raw.cached_tokens);
|
||||
const cacheCreation = n(raw.cache_creation_input_tokens);
|
||||
const out = { promptTokens: input, completionTokens: output, totalTokens: input + output };
|
||||
if (cached > 0) out.cachedTokens = cached;
|
||||
if (cacheCreation > 0) out.cacheCreationTokens = cacheCreation;
|
||||
return out;
|
||||
},
|
||||
ollama(raw) {
|
||||
const input = n(raw.prompt_eval_count), output = n(raw.eval_count);
|
||||
|
||||
@@ -27,6 +27,25 @@ export function claudeToOpenAIResponse(chunk, state) {
|
||||
state.messageId = chunk.message?.id || `msg_${Date.now()}`;
|
||||
state.model = chunk.message?.model;
|
||||
state.toolCallIndex = 0;
|
||||
// Claude sends input_tokens + cache_read + cache_creation here; message_delta
|
||||
// later carries only the final output_tokens. Capture cache now so the
|
||||
// delta (output-only) doesn't reset it to zero.
|
||||
const startUsage = chunk.message?.usage;
|
||||
if (startUsage && typeof startUsage === "object") {
|
||||
const inputTokens = typeof startUsage.input_tokens === "number" ? startUsage.input_tokens : 0;
|
||||
const cacheReadTokens = typeof startUsage.cache_read_input_tokens === "number" ? startUsage.cache_read_input_tokens : 0;
|
||||
const cacheCreationTokens = typeof startUsage.cache_creation_input_tokens === "number" ? startUsage.cache_creation_input_tokens : 0;
|
||||
const promptTokens = inputTokens + cacheReadTokens + cacheCreationTokens;
|
||||
state.usage = {
|
||||
prompt_tokens: promptTokens,
|
||||
completion_tokens: 0,
|
||||
total_tokens: promptTokens,
|
||||
input_tokens: inputTokens,
|
||||
output_tokens: 0
|
||||
};
|
||||
if (cacheReadTokens > 0) state.usage.cache_read_input_tokens = cacheReadTokens;
|
||||
if (cacheCreationTokens > 0) state.usage.cache_creation_input_tokens = cacheCreationTokens;
|
||||
}
|
||||
results.push(createChunk(state, { role: ROLE.ASSISTANT }));
|
||||
break;
|
||||
}
|
||||
@@ -103,13 +122,15 @@ export function claudeToOpenAIResponse(chunk, state) {
|
||||
}
|
||||
|
||||
case "message_delta": {
|
||||
// Extract usage from message_delta event (Claude native format)
|
||||
// Normalize to OpenAI format (prompt_tokens/completion_tokens) for consistent logging
|
||||
// Extract usage from message_delta event (Claude native format).
|
||||
// Anthropic sends input/cache in message_start and only output here, so
|
||||
// fall back to cache captured in message_start when the delta omits it.
|
||||
if (chunk.usage && typeof chunk.usage === "object") {
|
||||
const inputTokens = typeof chunk.usage.input_tokens === "number" ? chunk.usage.input_tokens : 0;
|
||||
const prev = state.usage || {};
|
||||
const inputTokens = typeof chunk.usage.input_tokens === "number" ? chunk.usage.input_tokens : (prev.input_tokens || 0);
|
||||
const outputTokens = typeof chunk.usage.output_tokens === "number" ? chunk.usage.output_tokens : 0;
|
||||
const cacheReadTokens = typeof chunk.usage.cache_read_input_tokens === "number" ? chunk.usage.cache_read_input_tokens : 0;
|
||||
const cacheCreationTokens = typeof chunk.usage.cache_creation_input_tokens === "number" ? chunk.usage.cache_creation_input_tokens : 0;
|
||||
const cacheReadTokens = typeof chunk.usage.cache_read_input_tokens === "number" ? chunk.usage.cache_read_input_tokens : (prev.cache_read_input_tokens || 0);
|
||||
const cacheCreationTokens = typeof chunk.usage.cache_creation_input_tokens === "number" ? chunk.usage.cache_creation_input_tokens : (prev.cache_creation_input_tokens || 0);
|
||||
|
||||
// prompt_tokens = input_tokens + cache_read + cache_creation (all prompt-side tokens)
|
||||
const promptTokens = inputTokens + cacheReadTokens + cacheCreationTokens;
|
||||
@@ -131,7 +152,14 @@ export function claudeToOpenAIResponse(chunk, state) {
|
||||
const finalChunk = createChunk(state, {}, state.finishReason);
|
||||
|
||||
if (state.usage) {
|
||||
finalChunk.usage = toOpenAIUsage(chunk.usage, "claude");
|
||||
// Build OpenAI usage from the merged state (cache from message_start +
|
||||
// output from message_delta), not the delta chunk alone.
|
||||
finalChunk.usage = toOpenAIUsage({
|
||||
input_tokens: state.usage.input_tokens || 0,
|
||||
output_tokens: state.usage.output_tokens || 0,
|
||||
cache_read_input_tokens: state.usage.cache_read_input_tokens,
|
||||
cache_creation_input_tokens: state.usage.cache_creation_input_tokens
|
||||
}, "claude");
|
||||
}
|
||||
|
||||
results.push(finalChunk);
|
||||
|
||||
@@ -247,9 +247,24 @@ function mergeChunksToResponse(chunks, sourceFormat) {
|
||||
|
||||
if (messageStart?.message) {
|
||||
finalChunk = messageStart.message;
|
||||
// Merge usage if available
|
||||
if (messageDelta?.usage) {
|
||||
finalChunk.usage = messageDelta.usage;
|
||||
// message_start.usage has input + cache; message_delta.usage has the
|
||||
// final output_tokens. Merge so cache survives (delta omits it).
|
||||
const startUsage = messageStart.message.usage;
|
||||
const deltaUsage = messageDelta?.usage;
|
||||
if (startUsage || deltaUsage) {
|
||||
finalChunk.usage = {
|
||||
...(startUsage || {}),
|
||||
...(deltaUsage || {}),
|
||||
...(startUsage?.cache_read_input_tokens !== undefined
|
||||
? { cache_read_input_tokens: startUsage.cache_read_input_tokens }
|
||||
: {}),
|
||||
...(startUsage?.cache_creation_input_tokens !== undefined
|
||||
? { cache_creation_input_tokens: startUsage.cache_creation_input_tokens }
|
||||
: {}),
|
||||
...(startUsage?.input_tokens !== undefined
|
||||
? { input_tokens: startUsage.input_tokens }
|
||||
: {})
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { translateResponse, initState } from "../translator/index.js";
|
||||
import { FORMATS } from "../translator/formats.js";
|
||||
import { trackPendingRequest, appendRequestLog } from "@/lib/usageDb.js";
|
||||
import { extractUsage, hasValidUsage, estimateUsage, logUsage, addBufferToUsage, filterUsageForFormat, COLORS } from "./usageTracking.js";
|
||||
import { extractUsage, mergeUsage, hasValidUsage, estimateUsage, logUsage, addBufferToUsage, filterUsageForFormat, COLORS } from "./usageTracking.js";
|
||||
import { parseSSELine, hasValuableContent, fixInvalidId, formatSSE } from "./streamHelpers.js";
|
||||
import { getOpenAIResponsesEventName, isOpenAIResponsesTerminalEvent, formatIncompleteOpenAIResponsesStreamFailure } from "./responsesStreamHelpers.js";
|
||||
import { dbg, isDebugEnabled } from "./debugLog.js";
|
||||
@@ -162,7 +162,7 @@ export function createSSEStream(options = {}) {
|
||||
|
||||
const extracted = extractUsage(parsed);
|
||||
if (extracted) {
|
||||
usage = extracted;
|
||||
usage = mergeUsage(usage, extracted);
|
||||
}
|
||||
|
||||
const isFinishChunk = parsed.choices?.[0]?.finish_reason;
|
||||
@@ -280,7 +280,7 @@ export function createSSEStream(options = {}) {
|
||||
|
||||
// Extract usage
|
||||
const extracted = extractUsage(parsed);
|
||||
if (extracted) state.usage = extracted; // Keep original usage for logging
|
||||
if (extracted) state.usage = mergeUsage(state.usage, extracted); // Keep original usage for logging
|
||||
|
||||
// Responses same-format passthrough: re-emit with original event framing
|
||||
if (keepsOpenAIResponsesFormat && openAIResponsesEventName) {
|
||||
|
||||
@@ -141,6 +141,68 @@ export function normalizeUsage(usage) {
|
||||
return normalized;
|
||||
}
|
||||
|
||||
/**
|
||||
* Canonicalize usage into ONE storage/cost convention so token counts and cost
|
||||
* are consistent across providers:
|
||||
* prompt_tokens = total input INCLUDING cache read + cache creation
|
||||
* cached_tokens = cache-read portion (subset of prompt_tokens)
|
||||
* cache_creation_input_tokens = cache-write portion (subset of prompt_tokens)
|
||||
* completion_tokens, reasoning_tokens, total_tokens
|
||||
*
|
||||
* Discriminator: Claude reports cache_read_input_tokens with a prompt that
|
||||
* EXCLUDES cache, so we fold cache into prompt. OpenAI/Gemini report
|
||||
* cached_tokens already counted inside prompt, so we pass through. Idempotent:
|
||||
* once folded the output carries cached_tokens (not cache_read_input_tokens),
|
||||
* so re-running takes the passthrough branch and does not double-add.
|
||||
*
|
||||
* @param {object} usage - a normalizeUsage()-shaped object
|
||||
* @returns {object|null} canonical token object, or null for invalid input
|
||||
*/
|
||||
export function canonicalizeUsage(usage) {
|
||||
if (!usage || typeof usage !== "object" || Array.isArray(usage)) return null;
|
||||
|
||||
const num = (v) => (Number.isFinite(Number(v)) ? Number(v) : 0);
|
||||
const completion = num(usage.completion_tokens ?? usage.output_tokens);
|
||||
const reasoning = num(usage.reasoning_tokens);
|
||||
// Fall back to the nested prompt_tokens_details.cache_creation_tokens shape
|
||||
// (buildUsage()'s OpenAI-forwarding format) when the top-level field is
|
||||
// absent, so callers that pass a buildUsage() object through don't silently
|
||||
// drop cache_creation.
|
||||
const cacheCreation = num(usage.cache_creation_input_tokens ?? usage.prompt_tokens_details?.cache_creation_tokens);
|
||||
|
||||
let prompt = num(usage.prompt_tokens ?? usage.input_tokens);
|
||||
let cached;
|
||||
|
||||
// Claude path: prompt excludes cache; cache_read_input_tokens and/or
|
||||
// cache_creation_input_tokens are separate. A cache-miss "first write" only
|
||||
// carries cache_creation_input_tokens (no cache_read_input_tokens yet), so
|
||||
// check both fields — otherwise a first-write request falls through to the
|
||||
// OpenAI passthrough branch below and cache_creation never gets folded in.
|
||||
// Guard on the absence of `cached_tokens`: our own canonical output always
|
||||
// sets that key (even to 0), so re-running canonicalizeUsage on an already-
|
||||
// folded result takes the passthrough branch instead of folding again.
|
||||
if (usage.cached_tokens === undefined &&
|
||||
(usage.cache_read_input_tokens !== undefined || usage.cache_creation_input_tokens !== undefined)) {
|
||||
cached = num(usage.cache_read_input_tokens);
|
||||
prompt = prompt + cached + cacheCreation;
|
||||
} else {
|
||||
// OpenAI/Gemini path (or already-canonical input): prompt already includes cached_tokens.
|
||||
cached = num(usage.cached_tokens);
|
||||
}
|
||||
|
||||
const result = {
|
||||
prompt_tokens: prompt,
|
||||
completion_tokens: completion,
|
||||
// Recompute rather than pass through: when the fold branch ran above,
|
||||
// an upstream total_tokens (cache-exclusive) would otherwise be stale.
|
||||
total_tokens: prompt + completion,
|
||||
cached_tokens: cached,
|
||||
cache_creation_input_tokens: cacheCreation,
|
||||
};
|
||||
if (reasoning > 0) result.reasoning_tokens = reasoning;
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if usage has valid token data
|
||||
* Valid = has at least one token field with value > 0
|
||||
@@ -171,6 +233,19 @@ export function hasValidUsage(usage) {
|
||||
export function extractUsage(chunk) {
|
||||
if (!chunk || typeof chunk !== "object") return null;
|
||||
|
||||
// Claude format (message_start event): carries input_tokens + cache_read +
|
||||
// cache_creation. message_delta later carries only the final output_tokens,
|
||||
// so callers must MERGE (mergeUsage), not overwrite, to keep cache counts.
|
||||
if (chunk.type === "message_start" && chunk.message?.usage && typeof chunk.message.usage === "object") {
|
||||
const u = chunk.message.usage;
|
||||
return normalizeUsage({
|
||||
prompt_tokens: u.input_tokens || 0,
|
||||
completion_tokens: u.output_tokens || 0,
|
||||
cache_read_input_tokens: u.cache_read_input_tokens,
|
||||
cache_creation_input_tokens: u.cache_creation_input_tokens
|
||||
});
|
||||
}
|
||||
|
||||
// Claude format (message_delta event)
|
||||
if (chunk.type === "message_delta" && chunk.usage && typeof chunk.usage === "object") {
|
||||
return normalizeUsage({
|
||||
@@ -232,6 +307,27 @@ export function extractUsage(chunk) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Field-wise max-merge of two usage objects. Anthropic splits usage across
|
||||
// events: message_start has real input+cache (output is a placeholder 1),
|
||||
// message_delta has the real cumulative output (input/cache absent). Max keeps
|
||||
// the meaningful value from each without clobbering. Idempotent for other
|
||||
// providers that emit a single complete usage object.
|
||||
export function mergeUsage(prev, next) {
|
||||
if (!prev) return next || null;
|
||||
if (!next) return prev;
|
||||
const merged = { ...prev };
|
||||
for (const [k, v] of Object.entries(next)) {
|
||||
// typeof NaN === "number" — guard with Number.isFinite so one malformed
|
||||
// chunk can't poison the whole accumulation (Math.max(x, NaN) is NaN).
|
||||
if (typeof v === "number" && Number.isFinite(v)) {
|
||||
merged[k] = Math.max(typeof merged[k] === "number" ? merged[k] : 0, v);
|
||||
} else if (v && typeof v === "object") {
|
||||
merged[k] = v; // nested details objects: take latest
|
||||
}
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
/**
|
||||
* Estimate input tokens from request body
|
||||
* Calculate total body size for more accurate estimation
|
||||
|
||||
@@ -8,7 +8,7 @@ const fmtCost = (n) => `$${(n || 0).toFixed(2)}`;
|
||||
|
||||
export default function OverviewCards({ stats }) {
|
||||
return (
|
||||
<div className="grid min-w-0 grid-cols-1 gap-3 sm:grid-cols-2 md:grid-cols-4 sm:gap-4">
|
||||
<div className="grid min-w-0 grid-cols-1 gap-3 sm:grid-cols-2 md:grid-cols-3 lg:grid-cols-5 sm:gap-4">
|
||||
<Card className="flex min-w-0 flex-col gap-1 px-4 py-3">
|
||||
<span className="text-text-muted text-sm uppercase font-semibold">Total Requests</span>
|
||||
<span className="truncate text-2xl font-bold">{fmt(stats.totalRequests)}</span>
|
||||
@@ -17,6 +17,10 @@ export default function OverviewCards({ stats }) {
|
||||
<span className="text-text-muted text-sm uppercase font-semibold">Total Input Tokens</span>
|
||||
<span className="truncate text-2xl font-bold text-primary">{fmt(stats.totalPromptTokens)}</span>
|
||||
</Card>
|
||||
<Card className="flex min-w-0 flex-col gap-1 px-4 py-3">
|
||||
<span className="text-text-muted text-sm uppercase font-semibold">Cached Tokens</span>
|
||||
<span className="truncate text-2xl font-bold text-info">{fmt(stats.totalCachedTokens)}</span>
|
||||
</Card>
|
||||
<Card className="flex min-w-0 flex-col gap-1 px-4 py-3">
|
||||
<span className="text-text-muted text-sm uppercase font-semibold">Output Tokens</span>
|
||||
<span className="truncate text-2xl font-bold text-success">{fmt(stats.totalCompletionTokens)}</span>
|
||||
|
||||
@@ -82,9 +82,20 @@ function CollapsibleSection({ title, children, defaultOpen = false, icon = null
|
||||
);
|
||||
}
|
||||
|
||||
function getCachedTokens(tokens) {
|
||||
return tokens?.cached_tokens || tokens?.cache_read_input_tokens || 0;
|
||||
}
|
||||
|
||||
function getCacheCreationTokens(tokens) {
|
||||
return tokens?.cache_creation_input_tokens || 0;
|
||||
}
|
||||
|
||||
function getInputTokens(tokens) {
|
||||
const prompt = tokens?.prompt_tokens || tokens?.input_tokens || 0;
|
||||
const cache = tokens?.cached_tokens || tokens?.cache_read_input_tokens || 0;
|
||||
// Canonical storage keeps prompt cache-inclusive. Legacy Claude rows may have
|
||||
// stored prompt cache-exclusive; fall back to cache when it's larger so old
|
||||
// rows don't under-report input.
|
||||
const cache = getCachedTokens(tokens);
|
||||
return prompt < cache ? cache : prompt;
|
||||
}
|
||||
|
||||
@@ -245,6 +256,8 @@ export default function RequestDetailsTab() {
|
||||
<th className="text-left p-4 text-sm font-semibold text-text-main">Model</th>
|
||||
<th className="text-left p-4 text-sm font-semibold text-text-main">Provider</th>
|
||||
<th className="text-right p-4 text-sm font-semibold text-text-main">Input Tokens</th>
|
||||
<th className="text-right p-4 text-sm font-semibold text-text-main">Cached</th>
|
||||
<th className="text-right p-4 text-sm font-semibold text-text-main">Cache Creation</th>
|
||||
<th className="text-right p-4 text-sm font-semibold text-text-main">Output Tokens</th>
|
||||
<th className="text-left p-4 text-sm font-semibold text-text-main">Latency</th>
|
||||
<th className="text-center p-4 text-sm font-semibold text-text-main">Action</th>
|
||||
@@ -286,6 +299,12 @@ export default function RequestDetailsTab() {
|
||||
<td className="p-4 text-sm text-text-main text-right font-mono">
|
||||
{getInputTokens(detail.tokens).toLocaleString()}
|
||||
</td>
|
||||
<td className="p-4 text-sm text-text-main text-right font-mono">
|
||||
{getCachedTokens(detail.tokens) > 0 ? getCachedTokens(detail.tokens).toLocaleString() : "—"}
|
||||
</td>
|
||||
<td className="p-4 text-sm text-text-main text-right font-mono">
|
||||
{getCacheCreationTokens(detail.tokens) > 0 ? getCacheCreationTokens(detail.tokens).toLocaleString() : "—"}
|
||||
</td>
|
||||
<td className="p-4 text-sm text-text-main text-right font-mono">
|
||||
{detail.tokens?.completion_tokens?.toLocaleString() || 0}
|
||||
</td>
|
||||
@@ -370,6 +389,22 @@ export default function RequestDetailsTab() {
|
||||
{getInputTokens(selectedDetail.tokens).toLocaleString()}
|
||||
</span>
|
||||
</div>
|
||||
{getCachedTokens(selectedDetail.tokens) > 0 && (
|
||||
<div>
|
||||
<span className="text-text-muted">Cached Tokens:</span>{" "}
|
||||
<span className="text-text-main font-mono">
|
||||
{getCachedTokens(selectedDetail.tokens).toLocaleString()}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
{getCacheCreationTokens(selectedDetail.tokens) > 0 && (
|
||||
<div>
|
||||
<span className="text-text-muted">Cache Creation:</span>{" "}
|
||||
<span className="text-text-main font-mono">
|
||||
{getCacheCreationTokens(selectedDetail.tokens).toLocaleString()}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
<div>
|
||||
<span className="text-text-muted">Output Tokens:</span>{" "}
|
||||
<span className="text-text-main font-mono">
|
||||
|
||||
@@ -38,6 +38,9 @@ function ValueCells({ item, viewMode, isSummary = false }) {
|
||||
<td className="px-6 py-3 text-right text-text-muted">
|
||||
{isSummary && item.promptTokens === undefined ? "—" : fmt(item.promptTokens)}
|
||||
</td>
|
||||
<td className="px-6 py-3 text-right text-text-muted">
|
||||
{item.cachedTokens ? fmt(item.cachedTokens) : "—"}
|
||||
</td>
|
||||
<td className="px-6 py-3 text-right text-text-muted">
|
||||
{isSummary && item.completionTokens === undefined ? "—" : fmt(item.completionTokens)}
|
||||
</td>
|
||||
@@ -52,6 +55,9 @@ function ValueCells({ item, viewMode, isSummary = false }) {
|
||||
<td className="px-6 py-3 text-right text-text-muted">
|
||||
{isSummary && item.inputCost === undefined ? "—" : fmtCost(item.inputCost)}
|
||||
</td>
|
||||
<td className="px-6 py-3 text-right text-text-muted">
|
||||
{item.cachedCost ? fmtCost(item.cachedCost) : "—"}
|
||||
</td>
|
||||
<td className="px-6 py-3 text-right text-text-muted">
|
||||
{isSummary && item.outputCost === undefined ? "—" : fmtCost(item.outputCost)}
|
||||
</td>
|
||||
@@ -133,12 +139,14 @@ export default function UsageTable({
|
||||
if (viewMode === "tokens") {
|
||||
return [
|
||||
{ field: "promptTokens", label: "Input Tokens" },
|
||||
{ field: "cachedTokens", label: "Cached" },
|
||||
{ field: "completionTokens", label: "Output Tokens" },
|
||||
{ field: "totalTokens", label: "Total Tokens" },
|
||||
];
|
||||
}
|
||||
return [
|
||||
{ field: "promptTokens", label: "Input Cost" },
|
||||
{ field: "cachedCost", label: "Cached Cost" },
|
||||
{ field: "completionTokens", label: "Output Cost" },
|
||||
{ field: "cost", label: "Total Cost" },
|
||||
];
|
||||
|
||||
@@ -126,6 +126,8 @@ export async function POST(request) {
|
||||
|
||||
let providerSpecificData = normalizeProviderSpecificData(provider, body, body.providerSpecificData);
|
||||
|
||||
// Compatible LLM nodes support multiple API-key connections (key pool); runtime
|
||||
// rotates/fails over via getProviderCredentials. Embedding nodes stay single-connection.
|
||||
if (isOpenAICompatibleProvider(provider)) {
|
||||
const node = await getProviderNodeById(provider);
|
||||
if (!node) {
|
||||
|
||||
@@ -51,10 +51,11 @@ function getLocalDateKey(timestamp) {
|
||||
}
|
||||
|
||||
function addToCounter(target, key, values) {
|
||||
if (!target[key]) target[key] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0 };
|
||||
if (!target[key]) target[key] = { requests: 0, promptTokens: 0, completionTokens: 0, cachedTokens: 0, cost: 0 };
|
||||
target[key].requests += values.requests || 1;
|
||||
target[key].promptTokens += values.promptTokens || 0;
|
||||
target[key].completionTokens += values.completionTokens || 0;
|
||||
target[key].cachedTokens += values.cachedTokens || 0;
|
||||
target[key].cost += values.cost || 0;
|
||||
if (values.meta) Object.assign(target[key], values.meta);
|
||||
}
|
||||
@@ -62,12 +63,14 @@ function addToCounter(target, key, values) {
|
||||
function aggregateEntryToDay(day, entry) {
|
||||
const promptTokens = entry.tokens?.prompt_tokens || entry.tokens?.input_tokens || 0;
|
||||
const completionTokens = entry.tokens?.completion_tokens || entry.tokens?.output_tokens || 0;
|
||||
const cachedTokens = entry.tokens?.cached_tokens || entry.tokens?.cache_read_input_tokens || 0;
|
||||
const cost = entry.cost || 0;
|
||||
const vals = { promptTokens, completionTokens, cost };
|
||||
const vals = { promptTokens, completionTokens, cachedTokens, cost };
|
||||
|
||||
day.requests = (day.requests || 0) + 1;
|
||||
day.promptTokens = (day.promptTokens || 0) + promptTokens;
|
||||
day.completionTokens = (day.completionTokens || 0) + completionTokens;
|
||||
day.cachedTokens = (day.cachedTokens || 0) + cachedTokens;
|
||||
day.cost = (day.cost || 0) + cost;
|
||||
|
||||
day.byProvider ||= {};
|
||||
@@ -135,33 +138,11 @@ async function calculateCost(provider, model, tokens) {
|
||||
const pricing = await getPricingForModel(provider, model);
|
||||
if (!pricing) return 0;
|
||||
|
||||
let cost = 0;
|
||||
const inputTokens = tokens.prompt_tokens || tokens.input_tokens || 0;
|
||||
const cachedTokens = tokens.cached_tokens || tokens.cache_read_input_tokens || 0;
|
||||
const nonCachedInput = Math.max(0, inputTokens - cachedTokens);
|
||||
cost += nonCachedInput * (pricing.input / 1000000);
|
||||
|
||||
if (cachedTokens > 0) {
|
||||
const cachedRate = pricing.cached || pricing.input;
|
||||
cost += cachedTokens * (cachedRate / 1000000);
|
||||
}
|
||||
|
||||
const outputTokens = tokens.completion_tokens || tokens.output_tokens || 0;
|
||||
cost += outputTokens * (pricing.output / 1000000);
|
||||
|
||||
const reasoningTokens = tokens.reasoning_tokens || 0;
|
||||
if (reasoningTokens > 0) {
|
||||
const rate = pricing.reasoning || pricing.output;
|
||||
cost += reasoningTokens * (rate / 1000000);
|
||||
}
|
||||
|
||||
const cacheCreationTokens = tokens.cache_creation_input_tokens || 0;
|
||||
if (cacheCreationTokens > 0) {
|
||||
const rate = pricing.cache_creation || pricing.input;
|
||||
cost += cacheCreationTokens * (rate / 1000000);
|
||||
}
|
||||
|
||||
return cost;
|
||||
// Delegate the actual math to the single source of truth (avoids the two
|
||||
// copies drifting apart — see open-sse/providers/pricing.js for the
|
||||
// cache-inclusive prompt_tokens convention this assumes).
|
||||
const { calculateCostFromTokens } = await import("open-sse/providers/pricing.js");
|
||||
return calculateCostFromTokens(tokens, pricing);
|
||||
} catch (e) {
|
||||
console.error("Error calculating cost:", e);
|
||||
return 0;
|
||||
@@ -398,6 +379,7 @@ export async function getUsageStats(period = "all") {
|
||||
timestamp: r.timestamp, model: r.model, provider: r.provider || "",
|
||||
promptTokens: t.prompt_tokens || t.input_tokens || 0,
|
||||
completionTokens: t.completion_tokens || t.output_tokens || 0,
|
||||
cachedTokens: t.cached_tokens || t.cache_read_input_tokens || 0,
|
||||
status: r.status || "ok",
|
||||
};
|
||||
})
|
||||
@@ -413,7 +395,7 @@ export async function getUsageStats(period = "all") {
|
||||
|
||||
const stats = {
|
||||
totalRequests: 0,
|
||||
totalPromptTokens: 0, totalCompletionTokens: 0, totalCost: 0,
|
||||
totalPromptTokens: 0, totalCompletionTokens: 0, totalCachedTokens: 0, totalCost: 0,
|
||||
byProvider: {}, byModel: {}, byAccount: {}, byApiKey: {}, byEndpoint: {},
|
||||
last10Minutes: [],
|
||||
pending: pendingRequests,
|
||||
@@ -474,13 +456,15 @@ export async function getUsageStats(period = "all") {
|
||||
const day = parseJson(dr.data, {});
|
||||
stats.totalPromptTokens += day.promptTokens || 0;
|
||||
stats.totalCompletionTokens += day.completionTokens || 0;
|
||||
stats.totalCachedTokens += day.cachedTokens || 0;
|
||||
stats.totalCost += day.cost || 0;
|
||||
|
||||
for (const [prov, p] of Object.entries(day.byProvider || {})) {
|
||||
if (!stats.byProvider[prov]) stats.byProvider[prov] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0 };
|
||||
if (!stats.byProvider[prov]) stats.byProvider[prov] = { requests: 0, promptTokens: 0, completionTokens: 0, cachedTokens: 0, cost: 0 };
|
||||
stats.byProvider[prov].requests += p.requests || 0;
|
||||
stats.byProvider[prov].promptTokens += p.promptTokens || 0;
|
||||
stats.byProvider[prov].completionTokens += p.completionTokens || 0;
|
||||
stats.byProvider[prov].cachedTokens += p.cachedTokens || 0;
|
||||
stats.byProvider[prov].cost += p.cost || 0;
|
||||
}
|
||||
|
||||
@@ -490,11 +474,12 @@ export async function getUsageStats(period = "all") {
|
||||
const statsKey = provider ? `${rawModel} (${provider})` : rawModel;
|
||||
const providerDisplayName = providerNodeNameMap[provider] || provider;
|
||||
if (!stats.byModel[statsKey]) {
|
||||
stats.byModel[statsKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel, provider: providerDisplayName, lastUsed: dateKey };
|
||||
stats.byModel[statsKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cachedTokens: 0, cost: 0, rawModel, provider: providerDisplayName, lastUsed: dateKey };
|
||||
}
|
||||
stats.byModel[statsKey].requests += m.requests || 0;
|
||||
stats.byModel[statsKey].promptTokens += m.promptTokens || 0;
|
||||
stats.byModel[statsKey].completionTokens += m.completionTokens || 0;
|
||||
stats.byModel[statsKey].cachedTokens += m.cachedTokens || 0;
|
||||
stats.byModel[statsKey].cost += m.cost || 0;
|
||||
if (dateKey > (stats.byModel[statsKey].lastUsed || "")) stats.byModel[statsKey].lastUsed = dateKey;
|
||||
}
|
||||
@@ -506,11 +491,12 @@ export async function getUsageStats(period = "all") {
|
||||
const providerDisplayName = providerNodeNameMap[provider] || provider;
|
||||
const accountKey = `${rawModel} (${provider} - ${accountName})`;
|
||||
if (!stats.byAccount[accountKey]) {
|
||||
stats.byAccount[accountKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel, provider: providerDisplayName, connectionId: connId, accountName, lastUsed: dateKey };
|
||||
stats.byAccount[accountKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cachedTokens: 0, cost: 0, rawModel, provider: providerDisplayName, connectionId: connId, accountName, lastUsed: dateKey };
|
||||
}
|
||||
stats.byAccount[accountKey].requests += a.requests || 0;
|
||||
stats.byAccount[accountKey].promptTokens += a.promptTokens || 0;
|
||||
stats.byAccount[accountKey].completionTokens += a.completionTokens || 0;
|
||||
stats.byAccount[accountKey].cachedTokens += a.cachedTokens || 0;
|
||||
stats.byAccount[accountKey].cost += a.cost || 0;
|
||||
if (dateKey > (stats.byAccount[accountKey].lastUsed || "")) stats.byAccount[accountKey].lastUsed = dateKey;
|
||||
}
|
||||
@@ -525,11 +511,12 @@ export async function getUsageStats(period = "all") {
|
||||
const apiKeyMasked = maskApiKey(apiKeyVal);
|
||||
const apiKeyKey = apiKeyMasked || "local-no-key";
|
||||
if (!stats.byApiKey[akKey]) {
|
||||
stats.byApiKey[akKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel, provider: providerDisplayName, apiKeyMasked, keyName, apiKeyKey, lastUsed: dateKey };
|
||||
stats.byApiKey[akKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cachedTokens: 0, cost: 0, rawModel, provider: providerDisplayName, apiKeyMasked, keyName, apiKeyKey, lastUsed: dateKey };
|
||||
}
|
||||
stats.byApiKey[akKey].requests += ak.requests || 0;
|
||||
stats.byApiKey[akKey].promptTokens += ak.promptTokens || 0;
|
||||
stats.byApiKey[akKey].completionTokens += ak.completionTokens || 0;
|
||||
stats.byApiKey[akKey].cachedTokens += ak.cachedTokens || 0;
|
||||
stats.byApiKey[akKey].cost += ak.cost || 0;
|
||||
if (dateKey > (stats.byApiKey[akKey].lastUsed || "")) stats.byApiKey[akKey].lastUsed = dateKey;
|
||||
}
|
||||
@@ -540,11 +527,12 @@ export async function getUsageStats(period = "all") {
|
||||
const provider = ep.provider || "";
|
||||
const providerDisplayName = providerNodeNameMap[provider] || provider;
|
||||
if (!stats.byEndpoint[epKey]) {
|
||||
stats.byEndpoint[epKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, endpoint, rawModel, provider: providerDisplayName, lastUsed: dateKey };
|
||||
stats.byEndpoint[epKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cachedTokens: 0, cost: 0, endpoint, rawModel, provider: providerDisplayName, lastUsed: dateKey };
|
||||
}
|
||||
stats.byEndpoint[epKey].requests += ep.requests || 0;
|
||||
stats.byEndpoint[epKey].promptTokens += ep.promptTokens || 0;
|
||||
stats.byEndpoint[epKey].completionTokens += ep.completionTokens || 0;
|
||||
stats.byEndpoint[epKey].cachedTokens += ep.cachedTokens || 0;
|
||||
stats.byEndpoint[epKey].cost += ep.cost || 0;
|
||||
if (dateKey > (stats.byEndpoint[epKey].lastUsed || "")) stats.byEndpoint[epKey].lastUsed = dateKey;
|
||||
}
|
||||
@@ -595,26 +583,30 @@ export async function getUsageStats(period = "all") {
|
||||
const tokens = parseJson(r.tokens, {}) || {};
|
||||
const promptTokens = tokens.prompt_tokens || 0;
|
||||
const completionTokens = tokens.completion_tokens || 0;
|
||||
const cachedTokens = tokens.cached_tokens || tokens.cache_read_input_tokens || 0;
|
||||
const entryCost = r.cost || 0;
|
||||
const providerDisplayName = providerNodeNameMap[r.provider] || r.provider;
|
||||
|
||||
stats.totalPromptTokens += promptTokens;
|
||||
stats.totalCompletionTokens += completionTokens;
|
||||
stats.totalCachedTokens += cachedTokens;
|
||||
stats.totalCost += entryCost;
|
||||
|
||||
if (!stats.byProvider[r.provider]) stats.byProvider[r.provider] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0 };
|
||||
if (!stats.byProvider[r.provider]) stats.byProvider[r.provider] = { requests: 0, promptTokens: 0, completionTokens: 0, cachedTokens: 0, cost: 0 };
|
||||
stats.byProvider[r.provider].requests++;
|
||||
stats.byProvider[r.provider].promptTokens += promptTokens;
|
||||
stats.byProvider[r.provider].completionTokens += completionTokens;
|
||||
stats.byProvider[r.provider].cachedTokens += cachedTokens;
|
||||
stats.byProvider[r.provider].cost += entryCost;
|
||||
|
||||
const modelKey = r.provider ? `${r.model} (${r.provider})` : r.model;
|
||||
if (!stats.byModel[modelKey]) {
|
||||
stats.byModel[modelKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel: r.model, provider: providerDisplayName, lastUsed: r.timestamp };
|
||||
stats.byModel[modelKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cachedTokens: 0, cost: 0, rawModel: r.model, provider: providerDisplayName, lastUsed: r.timestamp };
|
||||
}
|
||||
stats.byModel[modelKey].requests++;
|
||||
stats.byModel[modelKey].promptTokens += promptTokens;
|
||||
stats.byModel[modelKey].completionTokens += completionTokens;
|
||||
stats.byModel[modelKey].cachedTokens += cachedTokens;
|
||||
stats.byModel[modelKey].cost += entryCost;
|
||||
if (new Date(r.timestamp) > new Date(stats.byModel[modelKey].lastUsed)) stats.byModel[modelKey].lastUsed = r.timestamp;
|
||||
|
||||
@@ -622,11 +614,12 @@ export async function getUsageStats(period = "all") {
|
||||
const accountName = connectionMap[r.connectionId] || `Account ${r.connectionId.slice(0, 8)}...`;
|
||||
const accountKey = `${r.model} (${r.provider} - ${accountName})`;
|
||||
if (!stats.byAccount[accountKey]) {
|
||||
stats.byAccount[accountKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel: r.model, provider: providerDisplayName, connectionId: r.connectionId, accountName, lastUsed: r.timestamp };
|
||||
stats.byAccount[accountKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cachedTokens: 0, cost: 0, rawModel: r.model, provider: providerDisplayName, connectionId: r.connectionId, accountName, lastUsed: r.timestamp };
|
||||
}
|
||||
stats.byAccount[accountKey].requests++;
|
||||
stats.byAccount[accountKey].promptTokens += promptTokens;
|
||||
stats.byAccount[accountKey].completionTokens += completionTokens;
|
||||
stats.byAccount[accountKey].cachedTokens += cachedTokens;
|
||||
stats.byAccount[accountKey].cost += entryCost;
|
||||
if (new Date(r.timestamp) > new Date(stats.byAccount[accountKey].lastUsed)) stats.byAccount[accountKey].lastUsed = r.timestamp;
|
||||
}
|
||||
@@ -637,27 +630,27 @@ export async function getUsageStats(period = "all") {
|
||||
const apiKeyMasked = maskApiKey(r.apiKey);
|
||||
const akKey = `${apiKeyMasked}|${r.model}|${r.provider || "unknown"}`;
|
||||
if (!stats.byApiKey[akKey]) {
|
||||
stats.byApiKey[akKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel: r.model, provider: providerDisplayName, apiKeyMasked, keyName, apiKeyKey: apiKeyMasked, lastUsed: r.timestamp };
|
||||
stats.byApiKey[akKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cachedTokens: 0, cost: 0, rawModel: r.model, provider: providerDisplayName, apiKeyMasked, keyName, apiKeyKey: apiKeyMasked, lastUsed: r.timestamp };
|
||||
}
|
||||
const ake = stats.byApiKey[akKey];
|
||||
ake.requests++; ake.promptTokens += promptTokens; ake.completionTokens += completionTokens; ake.cost += entryCost;
|
||||
ake.requests++; ake.promptTokens += promptTokens; ake.completionTokens += completionTokens; ake.cachedTokens += cachedTokens; ake.cost += entryCost;
|
||||
if (new Date(r.timestamp) > new Date(ake.lastUsed)) ake.lastUsed = r.timestamp;
|
||||
} else {
|
||||
if (!stats.byApiKey["local-no-key"]) {
|
||||
stats.byApiKey["local-no-key"] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel: r.model, provider: providerDisplayName, apiKeyMasked: null, keyName: "Local (No API Key)", apiKeyKey: "local-no-key", lastUsed: r.timestamp };
|
||||
stats.byApiKey["local-no-key"] = { requests: 0, promptTokens: 0, completionTokens: 0, cachedTokens: 0, cost: 0, rawModel: r.model, provider: providerDisplayName, apiKeyMasked: null, keyName: "Local (No API Key)", apiKeyKey: "local-no-key", lastUsed: r.timestamp };
|
||||
}
|
||||
const ake = stats.byApiKey["local-no-key"];
|
||||
ake.requests++; ake.promptTokens += promptTokens; ake.completionTokens += completionTokens; ake.cost += entryCost;
|
||||
ake.requests++; ake.promptTokens += promptTokens; ake.completionTokens += completionTokens; ake.cachedTokens += cachedTokens; ake.cost += entryCost;
|
||||
if (new Date(r.timestamp) > new Date(ake.lastUsed)) ake.lastUsed = r.timestamp;
|
||||
}
|
||||
|
||||
const endpoint = r.endpoint || "Unknown";
|
||||
const epKey = `${endpoint}|${r.model}|${r.provider || "unknown"}`;
|
||||
if (!stats.byEndpoint[epKey]) {
|
||||
stats.byEndpoint[epKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, endpoint, rawModel: r.model, provider: providerDisplayName, lastUsed: r.timestamp };
|
||||
stats.byEndpoint[epKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cachedTokens: 0, cost: 0, endpoint, rawModel: r.model, provider: providerDisplayName, lastUsed: r.timestamp };
|
||||
}
|
||||
const epe = stats.byEndpoint[epKey];
|
||||
epe.requests++; epe.promptTokens += promptTokens; epe.completionTokens += completionTokens; epe.cost += entryCost;
|
||||
epe.requests++; epe.promptTokens += promptTokens; epe.completionTokens += completionTokens; epe.cachedTokens += cachedTokens; epe.cost += entryCost;
|
||||
if (new Date(r.timestamp) > new Date(epe.lastUsed)) epe.lastUsed = r.timestamp;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -89,9 +89,16 @@ function sortData(dataMap, pendingMap = {}, sortBy, sortOrder) {
|
||||
.map(([key, data]) => {
|
||||
const totalTokens = (data.promptTokens || 0) + (data.completionTokens || 0);
|
||||
const totalCost = data.cost || 0;
|
||||
const inputCost = totalTokens > 0 ? (data.promptTokens || 0) * (totalCost / totalTokens) : 0;
|
||||
// ponytail: cost split is a token-share allocation of the (rate-accurate)
|
||||
// server total, not a per-rate recompute. cached is a subset of prompt, so
|
||||
// peel it out of the input share. Upgrade to a stored per-component cost
|
||||
// breakdown if exact cached-rate cost display is needed.
|
||||
const cachedTokens = data.cachedTokens || 0;
|
||||
const nonCachedInput = Math.max(0, (data.promptTokens || 0) - cachedTokens);
|
||||
const inputCost = totalTokens > 0 ? nonCachedInput * (totalCost / totalTokens) : 0;
|
||||
const cachedCost = totalTokens > 0 ? cachedTokens * (totalCost / totalTokens) : 0;
|
||||
const outputCost = totalTokens > 0 ? (data.completionTokens || 0) * (totalCost / totalTokens) : 0;
|
||||
return { ...data, key, totalTokens, totalCost, inputCost, outputCost, pending: pendingMap[key] || 0 };
|
||||
return { ...data, key, totalTokens, totalCost, inputCost, cachedCost, outputCost, pending: pendingMap[key] || 0 };
|
||||
})
|
||||
.sort((a, b) => {
|
||||
let valA = a[sortBy];
|
||||
@@ -122,7 +129,7 @@ function groupDataByKey(data, keyField) {
|
||||
if (!groups[gk]) {
|
||||
groups[gk] = {
|
||||
groupKey: gk,
|
||||
summary: { requests: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0, cost: 0, inputCost: 0, outputCost: 0, lastUsed: null, pending: 0 },
|
||||
summary: { requests: 0, promptTokens: 0, completionTokens: 0, cachedTokens: 0, totalTokens: 0, cost: 0, inputCost: 0, cachedCost: 0, outputCost: 0, lastUsed: null, pending: 0 },
|
||||
items: [],
|
||||
};
|
||||
}
|
||||
@@ -130,9 +137,11 @@ function groupDataByKey(data, keyField) {
|
||||
s.requests += item.requests || 0;
|
||||
s.promptTokens += item.promptTokens || 0;
|
||||
s.completionTokens += item.completionTokens || 0;
|
||||
s.cachedTokens += item.cachedTokens || 0;
|
||||
s.totalTokens += item.totalTokens || 0;
|
||||
s.cost += item.cost || 0;
|
||||
s.inputCost += item.inputCost || 0;
|
||||
s.cachedCost += item.cachedCost || 0;
|
||||
s.outputCost += item.outputCost || 0;
|
||||
s.pending += item.pending || 0;
|
||||
if (item.lastUsed && (!s.lastUsed || new Date(item.lastUsed) > new Date(s.lastUsed))) {
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
// End-to-end: a cache-bearing request flows through canonicalizeUsage →
|
||||
// saveRequestUsage → getUsageStats, proving cached tokens are persisted,
|
||||
// aggregated, and cost is computed correctly (the bug this branch fixes).
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { describe, it, expect, beforeAll, afterAll, vi } from "vitest";
|
||||
import { canonicalizeUsage } from "../../open-sse/utils/usageTracking.js";
|
||||
|
||||
const originalDataDir = process.env.DATA_DIR;
|
||||
let tempDir;
|
||||
let db;
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "9router-cached-e2e-"));
|
||||
process.env.DATA_DIR = tempDir;
|
||||
vi.resetModules();
|
||||
db = await import("@/lib/db/index.js");
|
||||
await db.initDb();
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
if (tempDir) fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
if (originalDataDir === undefined) delete process.env.DATA_DIR;
|
||||
else process.env.DATA_DIR = originalDataDir;
|
||||
});
|
||||
|
||||
describe("cached-token end-to-end (persist + aggregate + cost)", () => {
|
||||
it("Claude cache usage: canonical prompt is inclusive, cached persisted, cost correct", async () => {
|
||||
// Raw Claude usage (cache-EXCLUSIVE prompt): input 100, cache_read 200, cache_creation 30, output 50
|
||||
const canonical = canonicalizeUsage({
|
||||
prompt_tokens: 100,
|
||||
completion_tokens: 50,
|
||||
cache_read_input_tokens: 200,
|
||||
cache_creation_input_tokens: 30,
|
||||
});
|
||||
expect(canonical.prompt_tokens).toBe(330); // inclusive
|
||||
|
||||
await db.saveRequestUsage({
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-4-6",
|
||||
connectionId: "c-cache",
|
||||
tokens: canonical,
|
||||
endpoint: "/v1/messages",
|
||||
status: "ok",
|
||||
});
|
||||
|
||||
const stats = await db.getUsageStats("24h");
|
||||
expect(stats.totalCachedTokens).toBe(200);
|
||||
expect(stats.totalPromptTokens).toBe(330);
|
||||
expect(stats.byProvider.anthropic.cachedTokens).toBe(200);
|
||||
|
||||
// Cost: nonCached=330-200-30=100 @3 + cached 200 @0.30 + creation 30 @3.75 + output 50 @15
|
||||
const expected = (100 * 3 + 200 * 0.3 + 30 * 3.75 + 50 * 15) / 1_000_000;
|
||||
const hist = await db.getUsageHistory({ provider: "anthropic" });
|
||||
expect(hist.length).toBe(1);
|
||||
expect(hist[0].cost).toBeCloseTo(expected, 12);
|
||||
expect(hist[0].tokens.cached_tokens).toBe(200);
|
||||
expect(hist[0].tokens.cache_creation_input_tokens).toBe(30);
|
||||
});
|
||||
|
||||
it("OpenAI cache usage: inclusive prompt passes through, cached counted once", async () => {
|
||||
const canonical = canonicalizeUsage({
|
||||
prompt_tokens: 1000, // already includes cached
|
||||
completion_tokens: 200,
|
||||
cached_tokens: 600,
|
||||
});
|
||||
expect(canonical.prompt_tokens).toBe(1000);
|
||||
expect(canonical.cached_tokens).toBe(600);
|
||||
|
||||
await db.saveRequestUsage({
|
||||
provider: "openai",
|
||||
model: "gpt-4o",
|
||||
connectionId: "c-oai",
|
||||
tokens: canonical,
|
||||
endpoint: "/v1/chat/completions",
|
||||
status: "ok",
|
||||
});
|
||||
|
||||
const hist = await db.getUsageHistory({ provider: "openai" });
|
||||
expect(hist[0].tokens.prompt_tokens).toBe(1000);
|
||||
expect(hist[0].tokens.cached_tokens).toBe(600);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,188 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { canonicalizeUsage, extractUsage, mergeUsage } from "../../open-sse/utils/usageTracking.js";
|
||||
import { calculateCostFromTokens } from "../../open-sse/providers/pricing.js";
|
||||
import { toOpenAIUsage } from "../../open-sse/translator/concerns/usage.js";
|
||||
|
||||
// Canonical convention (single source of truth for storage + cost):
|
||||
// prompt_tokens = total input INCLUDING cache read + cache creation
|
||||
// cached_tokens = cache-read portion (subset of prompt_tokens)
|
||||
// cache_creation_input_tokens = cache-write portion (subset of prompt_tokens)
|
||||
// completion_tokens = output
|
||||
// Discriminator: Claude reports cache separately (prompt EXCLUDES cache);
|
||||
// OpenAI/Gemini report prompt INCLUDING cached_tokens.
|
||||
describe("canonicalizeUsage", () => {
|
||||
it("folds Claude exclusive cache into an inclusive prompt count", () => {
|
||||
// Claude: input_tokens excludes cache; cache_read + cache_creation are separate
|
||||
const out = canonicalizeUsage({
|
||||
prompt_tokens: 100,
|
||||
completion_tokens: 50,
|
||||
cache_read_input_tokens: 200,
|
||||
cache_creation_input_tokens: 30,
|
||||
});
|
||||
expect(out.prompt_tokens).toBe(330); // 100 + 200 + 30
|
||||
expect(out.completion_tokens).toBe(50);
|
||||
expect(out.cached_tokens).toBe(200);
|
||||
expect(out.cache_creation_input_tokens).toBe(30);
|
||||
});
|
||||
|
||||
it("passes through OpenAI inclusive prompt unchanged", () => {
|
||||
// OpenAI: prompt_tokens already includes cached_tokens (a subset)
|
||||
const out = canonicalizeUsage({
|
||||
prompt_tokens: 330,
|
||||
completion_tokens: 50,
|
||||
cached_tokens: 200,
|
||||
});
|
||||
expect(out.prompt_tokens).toBe(330);
|
||||
expect(out.cached_tokens).toBe(200);
|
||||
expect(out.cache_creation_input_tokens).toBe(0);
|
||||
});
|
||||
|
||||
it("passes through Gemini inclusive prompt (cachedContent already counted)", () => {
|
||||
const out = canonicalizeUsage({
|
||||
prompt_tokens: 500,
|
||||
completion_tokens: 80,
|
||||
cached_tokens: 120,
|
||||
reasoning_tokens: 40,
|
||||
});
|
||||
expect(out.prompt_tokens).toBe(500);
|
||||
expect(out.cached_tokens).toBe(120);
|
||||
expect(out.reasoning_tokens).toBe(40);
|
||||
});
|
||||
|
||||
it("handles no-cache usage", () => {
|
||||
const out = canonicalizeUsage({ prompt_tokens: 100, completion_tokens: 50 });
|
||||
expect(out.prompt_tokens).toBe(100);
|
||||
expect(out.cached_tokens).toBe(0);
|
||||
expect(out.cache_creation_input_tokens).toBe(0);
|
||||
});
|
||||
|
||||
it("is idempotent (running twice yields the same canonical shape)", () => {
|
||||
const once = canonicalizeUsage({
|
||||
prompt_tokens: 100,
|
||||
completion_tokens: 50,
|
||||
cache_read_input_tokens: 200,
|
||||
cache_creation_input_tokens: 30,
|
||||
});
|
||||
const twice = canonicalizeUsage(once);
|
||||
expect(twice.prompt_tokens).toBe(330);
|
||||
expect(twice.cached_tokens).toBe(200);
|
||||
expect(twice.cache_creation_input_tokens).toBe(30);
|
||||
expect(twice.completion_tokens).toBe(50);
|
||||
});
|
||||
|
||||
it("returns null for invalid input", () => {
|
||||
expect(canonicalizeUsage(null)).toBeNull();
|
||||
expect(canonicalizeUsage(undefined)).toBeNull();
|
||||
});
|
||||
|
||||
it("folds a Claude cache-miss first write (cache_creation only, no cache_read yet)", () => {
|
||||
// Cache-miss on first write: upstream emits cache_creation_input_tokens but
|
||||
// no cache_read_input_tokens at all (not even 0). Must still fold into prompt
|
||||
// instead of falling through to the OpenAI passthrough branch.
|
||||
const out = canonicalizeUsage({
|
||||
prompt_tokens: 100,
|
||||
completion_tokens: 20,
|
||||
cache_creation_input_tokens: 500,
|
||||
});
|
||||
expect(out.prompt_tokens).toBe(600); // 100 + 0 (no read) + 500
|
||||
expect(out.cached_tokens).toBe(0);
|
||||
expect(out.cache_creation_input_tokens).toBe(500);
|
||||
});
|
||||
});
|
||||
|
||||
describe("calculateCostFromTokens (canonical inclusive convention)", () => {
|
||||
const pricing = { input: 3, output: 15, cached: 0.3, cache_creation: 3.75 };
|
||||
|
||||
it("prices cached + cache_creation as subsets of an inclusive prompt without double-counting", () => {
|
||||
// prompt=330 includes 200 cached + 30 cache_creation → 100 full-price input
|
||||
const cost = calculateCostFromTokens(
|
||||
{ prompt_tokens: 330, completion_tokens: 50, cached_tokens: 200, cache_creation_input_tokens: 30 },
|
||||
pricing
|
||||
);
|
||||
const expected =
|
||||
(100 * 3 + 200 * 0.3 + 30 * 3.75 + 50 * 15) / 1_000_000;
|
||||
expect(cost).toBeCloseTo(expected, 12);
|
||||
});
|
||||
|
||||
it("does not let cache_creation drive nonCached negative", () => {
|
||||
// pathological: cached + creation exceeds prompt → nonCached clamps at 0
|
||||
const cost = calculateCostFromTokens(
|
||||
{ prompt_tokens: 100, completion_tokens: 0, cached_tokens: 80, cache_creation_input_tokens: 40 },
|
||||
pricing
|
||||
);
|
||||
const expected = (0 * 3 + 80 * 0.3 + 40 * 3.75) / 1_000_000;
|
||||
expect(cost).toBeCloseTo(expected, 12);
|
||||
});
|
||||
|
||||
it("matches plain input pricing when no cache present", () => {
|
||||
const cost = calculateCostFromTokens({ prompt_tokens: 100, completion_tokens: 50 }, pricing);
|
||||
expect(cost).toBeCloseTo((100 * 3 + 50 * 15) / 1_000_000, 12);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Anthropic streaming usage (message_start carries cache, message_delta output-only)", () => {
|
||||
it("extractUsage reads input + cache from message_start", () => {
|
||||
const u = extractUsage({
|
||||
type: "message_start",
|
||||
message: { usage: { input_tokens: 100, output_tokens: 1, cache_read_input_tokens: 200, cache_creation_input_tokens: 30 } },
|
||||
});
|
||||
expect(u.prompt_tokens).toBe(100);
|
||||
expect(u.cache_read_input_tokens).toBe(200);
|
||||
expect(u.cache_creation_input_tokens).toBe(30);
|
||||
});
|
||||
|
||||
it("merges message_start cache with message_delta output without clobbering", () => {
|
||||
// Real Anthropic SSE: cache only in message_start, real output only in message_delta.
|
||||
const start = extractUsage({
|
||||
type: "message_start",
|
||||
message: { usage: { input_tokens: 100, output_tokens: 1, cache_read_input_tokens: 200, cache_creation_input_tokens: 30 } },
|
||||
});
|
||||
const delta = extractUsage({ type: "message_delta", usage: { output_tokens: 50 } });
|
||||
const merged = mergeUsage(start, delta);
|
||||
expect(merged.prompt_tokens).toBe(100);
|
||||
expect(merged.cache_read_input_tokens).toBe(200);
|
||||
expect(merged.cache_creation_input_tokens).toBe(30);
|
||||
expect(merged.completion_tokens).toBe(50);
|
||||
|
||||
// And it canonicalizes to a cache-inclusive prompt for storage/cost.
|
||||
const canon = canonicalizeUsage(merged);
|
||||
expect(canon.prompt_tokens).toBe(330); // 100 + 200 + 30
|
||||
expect(canon.cached_tokens).toBe(200);
|
||||
expect(canon.cache_creation_input_tokens).toBe(30);
|
||||
expect(canon.completion_tokens).toBe(50);
|
||||
});
|
||||
|
||||
it("does not let a NaN field poison the running max-merge", () => {
|
||||
// typeof NaN === "number", so a naive Math.max(prev, NaN) is NaN — one
|
||||
// malformed chunk must not wipe out an already-accumulated good value.
|
||||
const prev = { prompt_tokens: 100, cache_read_input_tokens: 200 };
|
||||
const bad = { prompt_tokens: NaN, completion_tokens: 50 };
|
||||
const merged = mergeUsage(prev, bad);
|
||||
expect(merged.prompt_tokens).toBe(100);
|
||||
expect(merged.cache_read_input_tokens).toBe(200);
|
||||
expect(merged.completion_tokens).toBe(50);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Kiro usage pass-through", () => {
|
||||
it("passes through plain input/output when no cache fields are present", () => {
|
||||
const out = toOpenAIUsage({ inputTokens: 100, outputTokens: 50 }, "kiro");
|
||||
expect(out.prompt_tokens).toBe(100);
|
||||
expect(out.completion_tokens).toBe(50);
|
||||
expect(out.total_tokens).toBe(150);
|
||||
expect(out.prompt_tokens_details).toBeUndefined();
|
||||
});
|
||||
|
||||
it("forward-compat: surfaces cache fields if Kiro event shape grows them", () => {
|
||||
// ponytail: Amazon Q upstream doesn't expose cache today, but if it starts
|
||||
// sending cache_read_input_tokens / cache_creation_input_tokens / cachedTokens,
|
||||
// cost tracking should pick them up automatically without another change.
|
||||
const out = toOpenAIUsage(
|
||||
{ inputTokens: 500, outputTokens: 100, cache_read_input_tokens: 200, cache_creation_input_tokens: 50 },
|
||||
"kiro"
|
||||
);
|
||||
expect(out.prompt_tokens_details).toBeDefined();
|
||||
expect(out.prompt_tokens_details.cached_tokens).toBe(200);
|
||||
expect(out.prompt_tokens_details.cache_creation_tokens).toBe(50);
|
||||
});
|
||||
});
|
||||
@@ -38,14 +38,14 @@ async function setupTestContext(nodeData) {
|
||||
};
|
||||
}
|
||||
|
||||
function makeRequest(provider) {
|
||||
function makeRequest(provider, name = "Test Connection") {
|
||||
return new Request("https://9router.local/api/providers", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
provider,
|
||||
apiKey: "test-key",
|
||||
name: "Test Connection",
|
||||
name,
|
||||
defaultModel: "test-model",
|
||||
}),
|
||||
});
|
||||
@@ -156,8 +156,8 @@ describe("compatible provider connections API", () => {
|
||||
});
|
||||
cleanup = ctx.cleanup;
|
||||
|
||||
const firstResponse = await ctx.POST(makeRequest(ctx.node.id));
|
||||
const secondResponse = await ctx.POST(makeRequest(ctx.node.id));
|
||||
const firstResponse = await ctx.POST(makeRequest(ctx.node.id, "Key A"));
|
||||
const secondResponse = await ctx.POST(makeRequest(ctx.node.id, "Key B"));
|
||||
const storedConnections = await ctx.getProviderConnections({ provider: ctx.node.id });
|
||||
|
||||
expect(firstResponse.status).toBe(201);
|
||||
|
||||
Reference in New Issue
Block a user