From 281f292f63b605a8b08f21f19834a3dbbc92454d Mon Sep 17 00:00:00 2001 From: decolua Date: Sat, 6 Jun 2026 12:19:59 +0700 Subject: [PATCH] test(translator): add data-driven coverage, bug-exposing cases, and real provider smoke - matrix.js generates N providers x M models from PROVIDER_MODELS - coverage-all-models + format-roundtrip for structural/semantic checks - bugs-* files expose known translation issues via it.fails - real/smoke-providers runs full handleChatCore path against live providers (RUN_REAL=1), concurrent - registerAll.js eagerly imports translators (Vitest ESM require fix) - vitest.config: array aliases for subpaths + maxConcurrency 60 Co-authored-by: Cursor --- tests/translator/AGENTS.md | 120 +++++++++++++++++ tests/translator/bugs-antigravity.test.js | 54 ++++++++ .../bugs-claudeCode-context.test.js | 73 +++++++++++ .../bugs-codexCli-responses.test.js | 63 +++++++++ .../bugs-gemini-cursor-commandcode.test.js | 76 +++++++++++ tests/translator/bugs-kiro.test.js | 45 +++++++ tests/translator/bugs-openai-bridge.test.js | 120 +++++++++++++++++ .../translator/bugs-toClaude-context.test.js | 65 +++++++++ tests/translator/coverage-all-models.test.js | 53 ++++++++ tests/translator/format-roundtrip.test.js | 76 +++++++++++ tests/translator/matrix.js | 67 ++++++++++ .../real/smoke-providers.real.test.js | 123 ++++++++++++++++++ tests/translator/registerAll.js | 22 ++++ tests/vitest.config.js | 14 +- 14 files changed, 965 insertions(+), 6 deletions(-) create mode 100644 tests/translator/AGENTS.md create mode 100644 tests/translator/bugs-antigravity.test.js create mode 100644 tests/translator/bugs-claudeCode-context.test.js create mode 100644 tests/translator/bugs-codexCli-responses.test.js create mode 100644 tests/translator/bugs-gemini-cursor-commandcode.test.js create mode 100644 tests/translator/bugs-kiro.test.js create mode 100644 tests/translator/bugs-openai-bridge.test.js create mode 100644 tests/translator/bugs-toClaude-context.test.js create mode 100644 tests/translator/coverage-all-models.test.js create mode 100644 tests/translator/format-roundtrip.test.js create mode 100644 tests/translator/matrix.js create mode 100644 tests/translator/real/smoke-providers.real.test.js create mode 100644 tests/translator/registerAll.js diff --git a/tests/translator/AGENTS.md b/tests/translator/AGENTS.md new file mode 100644 index 00000000..95273aa4 --- /dev/null +++ b/tests/translator/AGENTS.md @@ -0,0 +1,120 @@ +# Translation Layer Tests + +Tests for `open-sse/translator/`. Goals: (1) data-driven coverage of every provider/model, (2) expose bugs caused by using OpenAI as the intermediate format. + +## 1. Translation layer structure (`open-sse/translator/`) + +Pipeline uses **OpenAI as the intermediate format**: +- Request: `source → openai → target` (`translateRequest`) +- Response (SSE chunk): `target → openai → source` (`translateResponse`) +- If `source === target` → translation is skipped (passthrough). + +Components: +- `index.js` — `translateRequest` / `translateResponse` / `register(from, to, requestFn, responseFn)` / registry. +- `formats.js` — `FORMATS` enum (openai, claude, gemini, gemini-cli, openai-responses, antigravity, kiro, cursor, commandcode, ollama, vertex). +- `request/-to-.js` — one-way request translation. +- `response/-to-.js` — one-way SSE response translation. +- `helpers/` — `openaiHelper.js` (filterToOpenAIFormat), `toolCallHelper.js` (id/arguments), `claudeHelper.js`, `geminiHelper.js`. + +**OpenAI-bridge pitfalls** (source of most bugs): going through OpenAI easily loses `thinking`/`reasoning`, image URLs (non-base64), `input_audio`, `is_error`; tool `id`/`index` become unstable (parallel tool calls), non-text system blocks, `tool_choice:"none"`. + +## 2. Test layout + +| File | Role | +|---|---| +| `matrix.js` | Reads `PROVIDER_MODELS` → builds matrix (alias, model, targetFormat, strip, upstreamId). DRY core. | +| `registerAll.js` | Imports every translator to run `register()` side-effects. **Required** (see §5). | +| `coverage-all-models.test.js` | Tier 1: every model translates without throwing; strip applied correctly. | +| `format-roundtrip.test.js` | Tier 2: tool id/system/parallel survive the bridge. | +| `bugs-openai-bridge.test.js` | Exposes concrete bugs (with source file:line). | + +## 3. Running + +Always pass `--config tests/vitest.config.js` (the alias config lives there; without it vitest may not resolve `@/...` subpaths). + +```bash +# no-cred (default, offline): translator-only files +cd app && npx vitest run --config tests/vitest.config.js "tests/translator/" +cd app && npx vitest run --config tests/vitest.config.js "tests/translator/bugs-openai-bridge.test.js" + +# real (calls live providers using credentials from the local DB) +cd app && RUN_REAL=1 npx vitest run --config tests/vitest.config.js "tests/translator/real/" +``` +No-cred tests make NO network calls and need NO creds. Real tests (`real/`, gated by `RUN_REAL=1`) read active connections from `~/.9router/db/data.sqlite`, send a tiny prompt per provider through `handleChatCore`, and assert valid SSE. Account/quota errors (401/402/403/429) are treated as credential issues and skipped, not failures. + +## 4. Adding a new provider → tests cover it AUTOMATICALLY + +Add a provider by adding a key to `open-sse/config/providerModels.js` `PROVIDER_MODELS` (e.g. `newprov: [{ id, targetFormat?, strip?, upstreamModelId? }]`) plus its config in `open-sse/config/providers.js`. + +→ `coverage-all-models.test.js` **automatically** runs for the new models with **no test edits**. `matrix.js` reads config directly. + +Only add a dedicated test when a provider has a special format that does not round-trip cleanly (see §7). + +## 5. `registerAll.js` — why it is required + +`translator/index.js` uses `require(...)` (bundler-only) to lazy-load translators. Under vitest/ESM, `require` **silently no-ops** → empty registry → `translateRequest` skips the translation step → **false pass** (data is lost but the test goes green by mistake). + +→ Every test calling `translateRequest`/`translateResponse` MUST `import "./registerAll.js"` at the top of the file. + +## 6. Bug-exposure convention — `it.fails` + +- A bug confirmed in the app but NOT yet fixed → use `it.fails(...)`. +- `it.fails` **passes while the app still has the bug**, **turns red once the bug is fixed** → a reminder to update the test (switch `it.fails` → `it` and confirm correct behavior). +- Pattern for a new bug-exposure test: real input → assert the "should-be-kept" behavior → wrap in `it.fails` + a comment with the source `file:line`. + +## 7. Special formats to watch + +- `kiro` (binary AWS EventStream), `cursor` (protobuf ConnectRPC), `commandcode` (NDJSON) → responses do NOT round-trip cleanly through openai; test via their executors, not just the translator. +- Single-provider-two-formats (most fragile): `opencode-go` (minimax models → claude, others openai), `github` (escalates `/chat/completions` → `/responses` at runtime), `xiaomi-tokenplan` (claude alias). +- `gemini`/`gemini-cli`: only the LAST system message is kept → earlier system messages are lost. + +## 8. Current known bugs (currently `it.fails`) + +Grouped per CLI/provider test file. Each row is an `it.fails` case. + +**Claude (`bugs-openai-bridge.test.js`, `bugs-claudeCode-context.test.js`)** +| Bug | Source | +|---|---| +| Claude image `source.type="url"` dropped (only base64) | `request/claude-to-openai.js:133-141` | +| `tool_result` image block → raw JSON | `request/claude-to-openai.js:155-173` | +| `tool_result.is_error` lost | `request/claude-to-openai.js:155-173` | +| `thinking`/`redacted_thinking` dropped via bridge | `request/claude-to-openai.js:128` | + +**OpenAI → Claude (`bugs-toClaude-context.test.js`)** +| Bug | Source | +|---|---| +| Always injects "You are Claude Code" system prompt | `request/openai-to-claude.js:124-134` | +| `reasoning_content` not mapped to a thinking block | `request/openai-to-claude.js:268-273` | +| `tool_choice:"none"` → `auto` | `request/openai-to-claude.js:298` | +| `input_audio` dropped | `request/openai-to-claude.js` (no audio branch) | + +**Codex Responses (`bugs-codexCli-responses.test.js`)** +| Bug | Source | +|---|---| +| Empty-name function_call can leave `tool_calls: []` | `request/openai-responses.js:103` | +| `arguments` not coerced to string | `request/openai-responses.js:109-110` | +| `input_image` uses `file_id` as raw url | `request/openai-responses.js:75-77` | + +**Antigravity (`bugs-antigravity.test.js`)** +| Bug | Source | +|---|---| +| functionResponse + functionCall in same content → tool calls dropped | `request/antigravity-to-openai.js:177-189` | +| functionCall without id → random unstable id | `request/antigravity-to-openai.js:167` | + +**Kiro (`bugs-kiro.test.js`)** +| Bug | Source | +|---|---| +| `JSON.parse(arguments)` throws on bad JSON (no try/catch) | `request/openai-to-kiro.js:214-216` | +| `max_tokens` hardcoded to 32000 | `request/openai-to-kiro.js:309` | +| Remote image → `[Image: url]` text | `request/openai-to-kiro.js:132-134` | + +**Gemini / Cursor / CommandCode (`bugs-gemini-cursor-commandcode.test.js`)** +| Bug | Source | +|---|---| +| Only the last system message kept | `request/openai-to-gemini.js:92-96` | +| Cursor drops image content | `request/openai-to-cursor.js:12-24` | +| Cursor `max_tokens` hardcoded to 32000 | `request/openai-to-cursor.js:179` | +| CommandCode bad JSON args → `{}` silently | `request/openai-to-commandcode.js:53-57` | +| CommandCode image → `[image omitted]` | `request/openai-to-commandcode.js:41-42` | + +Fixing a bug → rerun; the matching `it.fails` test turns RED → switch it to a regular `it` and verify correct behavior. diff --git a/tests/translator/bugs-antigravity.test.js b/tests/translator/bugs-antigravity.test.js new file mode 100644 index 00000000..6b8052ad --- /dev/null +++ b/tests/translator/bugs-antigravity.test.js @@ -0,0 +1,54 @@ +// Real Antigravity-MITM requests (Gemini-internal: { request: { contents, ... } }) → OpenAI. +import { describe, it, expect } from "vitest"; +import "./registerAll.js"; +import { translateRequest } from "../../open-sse/translator/index.js"; +import { FORMATS } from "../../open-sse/translator/formats.js"; + +const AG2O = (req) => + translateRequest(FORMATS.ANTIGRAVITY, FORMATS.OPENAI, "m", { request: req }, true, null, null); + +describe("Antigravity → OpenAI", () => { + // antigravity-to-openai.js:177-189 — content with BOTH functionResponse and functionCall/text + // returns toolResults early → drops the tool calls / text. + // KNOWN BUG + it.fails("functionResponse + functionCall in same content keeps both", () => { + const out = AG2O({ + contents: [{ + role: "model", + parts: [ + { functionResponse: { id: "c1", name: "prev", response: { result: "done" } } }, + { functionCall: { id: "c2", name: "next", args: {} } }, + ], + }], + }); + const json = JSON.stringify(out); + expect(json, "functionCall lost when sharing content with functionResponse").toContain("\"next\""); + }); + + // antigravity-to-openai.js:167 — functionCall without id gets a random Date.now() id + // KNOWN BUG: unstable id breaks matching with its functionResponse + it.fails("functionCall without id keeps a stable matchable id", () => { + const out = AG2O({ + contents: [ + { role: "model", parts: [{ functionCall: { name: "search", args: { q: "x" } } }] }, + { role: "user", parts: [{ functionResponse: { name: "search", response: { result: "r" } } }] }, + ], + }); + const asst = out.messages.find((m) => m.tool_calls); + const tool = out.messages.find((m) => m.role === "tool"); + expect(tool?.tool_call_id, "id mismatch between call and response").toBe(asst?.tool_calls?.[0]?.id); + }); + + // antigravity-to-openai.js:144-147 — signature-only part handling (regression guard) + it("signature-only part does not produce empty text", () => { + const out = AG2O({ + contents: [{ role: "model", parts: [{ thoughtSignature: "sig", text: "" }] }], + }); + const asst = out.messages.find((m) => m.role === "assistant"); + const content = asst?.content; + const hasEmpty = Array.isArray(content) + ? content.some((c) => c.type === "text" && c.text === "") + : content === ""; + expect(hasEmpty, "empty text part emitted").toBe(false); + }); +}); diff --git a/tests/translator/bugs-claudeCode-context.test.js b/tests/translator/bugs-claudeCode-context.test.js new file mode 100644 index 00000000..eb782114 --- /dev/null +++ b/tests/translator/bugs-claudeCode-context.test.js @@ -0,0 +1,73 @@ +// Real Claude Code CLI requests (Claude format) → non-Claude provider via OpenAI bridge. +// Focuses on context components a real CLI sends: system arrays w/ cache_control, thinking +// signatures, tool_result with images, audio. KNOWN BUG = it.fails (source file:line in comments). +import { describe, it, expect } from "vitest"; +import "./registerAll.js"; +import { translateRequest } from "../../open-sse/translator/index.js"; +import { FORMATS } from "../../open-sse/translator/formats.js"; + +const T = (src, tgt, body, provider = null) => + translateRequest(src, tgt, "m", body, true, null, provider); + +describe("Claude Code CLI context → OpenAI", () => { + // claude-to-openai.js:24-27 — system array only maps .text; cache_control/non-text dropped + it("system array keeps all text parts", () => { + const out = T(FORMATS.CLAUDE, FORMATS.OPENAI, { + system: [ + { type: "text", text: "You are Claude Code.", cache_control: { type: "ephemeral" } }, + { type: "text", text: "Follow repo conventions." }, + ], + messages: [{ role: "user", content: "hi" }], + }); + const sys = out.messages.find((m) => m.role === "system"); + expect(sys?.content).toContain("Claude Code"); + expect(sys?.content).toContain("repo conventions"); + }); + + // claude→claude is passthrough (same format) → thinking preserved. Guards against + // accidental routing through the OpenAI bridge for same-format requests. + it("assistant thinking block survives Claude→Claude passthrough", () => { + const out = T(FORMATS.CLAUDE, FORMATS.CLAUDE, { + messages: [ + { role: "assistant", content: [ + { type: "thinking", thinking: "step-by-step plan", signature: "abc123" }, + { type: "text", text: "done" }, + ] }, + { role: "user", content: "next" }, + ], + }); + expect(JSON.stringify(out)).toContain("step-by-step plan"); + }); + + // claude-to-openai.js:128 — redacted_thinking also dropped + // KNOWN BUG + it.fails("redacted_thinking block is not silently dropped", () => { + const out = T(FORMATS.CLAUDE, FORMATS.OPENAI, { + messages: [ + { role: "assistant", content: [ + { type: "redacted_thinking", data: "ENCRYPTED_BLOB" }, + { type: "text", text: "answer" }, + ] }, + { role: "user", content: "go" }, + ], + }); + expect(JSON.stringify(out)).toContain("ENCRYPTED_BLOB"); + }); + + // claude-to-openai.js:155-173 — tool_result image block stringified into raw JSON + // KNOWN BUG + it.fails("tool_result image block is preserved", () => { + const out = T(FORMATS.CLAUDE, FORMATS.OPENAI, { + messages: [ + { role: "assistant", content: [{ type: "tool_use", id: "call_1", name: "screenshot", input: {} }] }, + { role: "user", content: [ + { type: "tool_result", tool_use_id: "call_1", content: [ + { type: "image", source: { type: "base64", media_type: "image/png", data: "IMG" } }, + ] }, + ] }, + ], + }); + const tool = out.messages.find((m) => m.role === "tool"); + expect(tool?.content, "image turned into raw JSON").not.toMatch(/^\[/); + }); +}); diff --git a/tests/translator/bugs-codexCli-responses.test.js b/tests/translator/bugs-codexCli-responses.test.js new file mode 100644 index 00000000..6f947833 --- /dev/null +++ b/tests/translator/bugs-codexCli-responses.test.js @@ -0,0 +1,63 @@ +// Real Codex CLI requests (OpenAI Responses API: { input:[], instructions }) → providers. +import { describe, it, expect } from "vitest"; +import "./registerAll.js"; +import { translateRequest } from "../../open-sse/translator/index.js"; +import { FORMATS } from "../../open-sse/translator/formats.js"; + +const R2O = (body) => translateRequest(FORMATS.OPENAI_RESPONSES, FORMATS.OPENAI, "m", body, true, null, null); +const O2R = (body) => translateRequest(FORMATS.OPENAI, FORMATS.OPENAI_RESPONSES, "m", body, true, null, null); + +describe("Codex CLI Responses → OpenAI", () => { + // openai-responses.js:103 — function_call with empty name skipped, can leave tool_calls: [] + // KNOWN BUG: empty tool_calls array is rejected by OpenAI/Codex + it.fails("assistant has no empty tool_calls array when all names are empty", () => { + const out = R2O({ + input: [ + { type: "function_call", call_id: "c1", name: "", arguments: "{}" }, + ], + }); + const asst = out.messages.find((m) => m.role === "assistant" && m.tool_calls); + expect(asst?.tool_calls?.length ?? 0, "empty tool_calls[] produced").toBeGreaterThan(0); + }); + + // openai-responses.js:109-110 — arguments passed through without ensuring string type + // KNOWN BUG + it.fails("function_call arguments end up as a string", () => { + const out = R2O({ + input: [{ type: "function_call", call_id: "c1", name: "f", arguments: { a: 1 } }], + }); + const asst = out.messages.find((m) => m.tool_calls); + expect(typeof asst.tool_calls[0].function.arguments).toBe("string"); + }); + + // openai-responses.js:75-77 — input_image uses file_id as raw url + // KNOWN BUG + it.fails("input_image with file_id is not used as a raw url", () => { + const out = R2O({ + input: [{ type: "message", role: "user", content: [ + { type: "input_image", file_id: "file-abc" }, + ] }], + }); + const userMsg = out.messages.find((m) => m.role === "user"); + const img = Array.isArray(userMsg?.content) ? userMsg.content.find((c) => c.type === "image_url") : null; + // A bare file_id is not a valid image URL + expect(img?.image_url?.url === "file-abc").toBe(false); + }); +}); + +describe("OpenAI → Codex Responses (reverse)", () => { + // openai-responses.js:13 — clampCallId NOT applied on Responses→Chat; but here Chat→Responses must clamp + it("call_id longer than 64 chars is clamped", () => { + const longId = "call_" + "x".repeat(80); + const out = O2R({ + messages: [ + { role: "assistant", content: null, tool_calls: [ + { id: longId, type: "function", function: { name: "f", arguments: "{}" } }, + ] }, + { role: "tool", tool_call_id: longId, content: "ok" }, + ], + }); + const fc = out.input.find((i) => i.type === "function_call"); + expect(fc.call_id.length).toBeLessThanOrEqual(64); + }); +}); diff --git a/tests/translator/bugs-gemini-cursor-commandcode.test.js b/tests/translator/bugs-gemini-cursor-commandcode.test.js new file mode 100644 index 00000000..4b74c60e --- /dev/null +++ b/tests/translator/bugs-gemini-cursor-commandcode.test.js @@ -0,0 +1,76 @@ +// OpenAI → Gemini / Cursor / CommandCode request translation. +import { describe, it, expect } from "vitest"; +import "./registerAll.js"; +import { translateRequest } from "../../open-sse/translator/index.js"; +import { FORMATS } from "../../open-sse/translator/formats.js"; + +const O2G = (body) => translateRequest(FORMATS.OPENAI, FORMATS.GEMINI, "m", body, true, null, "gemini"); +const O2C = (body) => translateRequest(FORMATS.OPENAI, FORMATS.CURSOR, "m", body, true, null, "cursor"); +const O2CC = (body) => translateRequest(FORMATS.OPENAI, FORMATS.COMMANDCODE, "m", body, true, null, "commandcode"); + +describe("OpenAI → Gemini", () => { + // openai-to-gemini.js:92-96 — each system message overwrites systemInstruction → only last kept + // KNOWN BUG + it.fails("multiple system messages are all kept", () => { + const out = O2G({ + messages: [ + { role: "system", content: "RULE_ONE" }, + { role: "system", content: "RULE_TWO" }, + { role: "user", content: "hi" }, + ], + }); + expect(JSON.stringify(out.systemInstruction), "earlier system lost").toContain("RULE_ONE"); + }); +}); + +describe("OpenAI → Cursor", () => { + // openai-to-cursor.js:12-24 — image content fully dropped (text only) + // KNOWN BUG + it.fails("image content is preserved", () => { + const out = O2C({ + messages: [{ role: "user", content: [ + { type: "text", text: "look" }, + { type: "image_url", image_url: { url: "data:image/png;base64,AAAA" } }, + ] }], + }); + expect(JSON.stringify(out), "image dropped").toContain("AAAA"); + }); + + // openai-to-cursor.js:179 — max_tokens hardcoded to 32000 + // KNOWN BUG + it.fails("respects client max_tokens", () => { + const out = O2C({ max_tokens: 200, messages: [{ role: "user", content: "hi" }] }); + expect(out.max_tokens).toBe(200); + }); +}); + +describe("OpenAI → CommandCode", () => { + // openai-to-commandcode.js:53-57 — safeParseJson returns {} on bad JSON (args silently lost) + // KNOWN BUG + it.fails("malformed tool arguments are not silently emptied", () => { + const out = O2CC({ + messages: [ + { role: "user", content: "go" }, + { role: "assistant", content: "", tool_calls: [ + { id: "c1", type: "function", function: { name: "f", arguments: "{bad" } }, + ] }, + { role: "tool", tool_call_id: "c1", content: "r" }, + ], + }); + const asst = out.params.messages.find((m) => m.role === "assistant"); + const call = asst.content.find((b) => b.type === "tool-call"); + expect(Object.keys(call.input).length, "arguments silently dropped to {}").toBeGreaterThan(0); + }); + + // openai-to-commandcode.js:41-42 — image becomes "[image omitted]" + // KNOWN BUG + it.fails("image content is preserved", () => { + const out = O2CC({ + messages: [{ role: "user", content: [ + { type: "text", text: "look" }, + { type: "image_url", image_url: { url: "data:image/png;base64,BBBB" } }, + ] }], + }); + expect(JSON.stringify(out), "image omitted").toContain("BBBB"); + }); +}); diff --git a/tests/translator/bugs-kiro.test.js b/tests/translator/bugs-kiro.test.js new file mode 100644 index 00000000..9a6836a4 --- /dev/null +++ b/tests/translator/bugs-kiro.test.js @@ -0,0 +1,45 @@ +// OpenAI → Kiro (AWS CodeWhisperer) request translation. +import { describe, it, expect } from "vitest"; +import "./registerAll.js"; +import { translateRequest } from "../../open-sse/translator/index.js"; +import { FORMATS } from "../../open-sse/translator/formats.js"; + +const O2K = (body) => translateRequest(FORMATS.OPENAI, FORMATS.KIRO, "m", body, true, null, "kiro"); + +describe("OpenAI → Kiro", () => { + // openai-to-kiro.js:214-216 — JSON.parse(arguments) without try/catch → throws on bad JSON + // KNOWN BUG (severe: throws and breaks the whole request) + it.fails("malformed tool arguments do not throw the whole request", () => { + expect(() => + O2K({ + messages: [ + { role: "user", content: "go" }, + { role: "assistant", content: "", tool_calls: [ + { id: "c1", type: "function", function: { name: "f", arguments: "{not json" } }, + ] }, + { role: "tool", tool_call_id: "c1", content: "r" }, + ], + }) + ).not.toThrow(); + }); + + // openai-to-kiro.js:309 — maxTokens hardcoded to 32000, ignores body.max_tokens + // KNOWN BUG + it.fails("respects client max_tokens", () => { + const out = O2K({ max_tokens: 100, messages: [{ role: "user", content: "hi" }] }); + expect(out.inferenceConfig?.maxTokens, "client max_tokens ignored").toBe(100); + }); + + // openai-to-kiro.js:132-134 — remote http image becomes "[Image: url]" text (lost) + // KNOWN BUG + it.fails("remote image url is preserved as an image, not text", () => { + const out = O2K({ + messages: [{ role: "user", content: [ + { type: "text", text: "see" }, + { type: "image_url", image_url: { url: "https://x.com/p.png" } }, + ] }], + }); + const content = out.conversationState?.currentMessage?.userInputMessage?.content || ""; + expect(content, "remote image flattened to text").not.toContain("[Image:"); + }); +}); diff --git a/tests/translator/bugs-openai-bridge.test.js b/tests/translator/bugs-openai-bridge.test.js new file mode 100644 index 00000000..0214f5ed --- /dev/null +++ b/tests/translator/bugs-openai-bridge.test.js @@ -0,0 +1,120 @@ +// Expose bugs caused by OpenAI being the intermediate format: data lost/wrong on source → openai → target. +// Each test describes the EXPECTED-correct behavior. A FAIL is evidence of the bug (with source file:line). +import { describe, it, expect } from "vitest"; +import "./registerAll.js"; +import { translateRequest } from "../../open-sse/translator/index.js"; +import { FORMATS } from "../../open-sse/translator/formats.js"; + +const T = (src, tgt, body, provider = null) => + translateRequest(src, tgt, "m", body, true, null, provider); + +describe("bug: Claude → OpenAI bridge data loss", () => { + // claude-to-openai.js:133-141 — image source.type==="url" only handles base64 + // KNOWN BUG: it.fails passes while app drops the url; flips to failing once fixed. + it.fails("image with source.type=url is preserved (NOT dropped)", () => { + const out = T(FORMATS.CLAUDE, FORMATS.OPENAI, { + messages: [{ role: "user", content: [ + { type: "text", text: "look" }, + { type: "image", source: { type: "url", url: "https://x.com/a.png" } }, + ] }], + }); + const json = JSON.stringify(out); + expect(json, "remote image url silently dropped").toContain("a.png"); + }); + + // claude-to-openai.js:128 switch — missing thinking/redacted_thinking case + it("thinking block survives round-trip Claude→OpenAI→Claude", () => { + const body = { + messages: [{ role: "assistant", content: [ + { type: "thinking", thinking: "secret reasoning", signature: "sig" }, + { type: "text", text: "answer" }, + ] }, { role: "user", content: "go" }], + }; + const out = T(FORMATS.CLAUDE, FORMATS.CLAUDE, body); + const json = JSON.stringify(out); + expect(json, "thinking content lost via OpenAI bridge").toContain("secret reasoning"); + }); + + // claude-to-openai.js:155-173 — tool_result image block dropped (text only) + // KNOWN BUG + it.fails("tool_result with image block is not turned into raw JSON / dropped", () => { + const out = T(FORMATS.CLAUDE, FORMATS.OPENAI, { + messages: [ + { role: "assistant", content: [ + { type: "tool_use", id: "call_1", name: "shot", input: {} }, + ] }, + { role: "user", content: [ + { type: "tool_result", tool_use_id: "call_1", content: [ + { type: "image", source: { type: "base64", media_type: "image/png", data: "ZZZ" } }, + ] }, + ] }, + ], + }); + const toolMsg = out.messages.find((m) => m.role === "tool"); + // Should keep the image; currently stringifies the whole array into raw JSON + expect(toolMsg?.content, "image in tool_result lost").not.toMatch(/^\[/); + }); + + // claude-to-openai.js:155-173 — is_error lost + // KNOWN BUG + it.fails("tool_result is_error flag is preserved", () => { + const out = T(FORMATS.CLAUDE, FORMATS.OPENAI, { + messages: [ + { role: "assistant", content: [{ type: "tool_use", id: "call_1", name: "f", input: {} }] }, + { role: "user", content: [ + { type: "tool_result", tool_use_id: "call_1", is_error: true, content: "boom" }, + ] }, + ], + }); + const json = JSON.stringify(out); + expect(json, "is_error dropped → model can't see tool failure").toContain("is_error"); + }); + + // claude-to-openai.js:24-27 — system array only takes .text, drops cache_control/non-text + it("system array non-text parts are not silently dropped", () => { + const out = T(FORMATS.CLAUDE, FORMATS.OPENAI, { + system: [ + { type: "text", text: "rule1", cache_control: { type: "ephemeral" } }, + { type: "text", text: "rule2" }, + ], + messages: [{ role: "user", content: "hi" }], + }); + const sys = out.messages.find((m) => m.role === "system"); + expect(sys?.content).toContain("rule1"); + expect(sys?.content).toContain("rule2"); + }); +}); + +describe("bug: tool_call id stability across bridge", () => { + // toolCallHelper.js:29-31 — sanitize changes tc.id but tool_call_id in another message may drift + it("sanitized tool id stays matched between call and result", () => { + const out = T(FORMATS.OPENAI, FORMATS.OPENAI, { + messages: [ + { role: "assistant", tool_calls: [ + { id: "call/with:bad*chars", type: "function", function: { name: "f", arguments: "{}" } }, + ] }, + { role: "tool", tool_call_id: "call/with:bad*chars", content: "ok" }, + ], + }); + const asst = out.messages.find((m) => m.role === "assistant"); + const tool = out.messages.find((m) => m.role === "tool"); + expect(tool.tool_call_id, "id mismatch after sanitize").toBe(asst.tool_calls[0].id); + }); +}); + +describe("bug: empty content message handling", () => { + // openaiHelper.js:49-51,66-71 — empty content → {text:""} then filtered out + it("assistant message with only tool_calls is not dropped", () => { + const out = T(FORMATS.OPENAI, FORMATS.OPENAI, { + messages: [ + { role: "user", content: "do it" }, + { role: "assistant", content: "", tool_calls: [ + { id: "call_1", type: "function", function: { name: "f", arguments: "{}" } }, + ] }, + { role: "tool", tool_call_id: "call_1", content: "done" }, + ], + }); + const asst = out.messages.find((m) => m.role === "assistant" && m.tool_calls); + expect(asst, "assistant tool_calls message dropped").toBeTruthy(); + }); +}); diff --git a/tests/translator/bugs-toClaude-context.test.js b/tests/translator/bugs-toClaude-context.test.js new file mode 100644 index 00000000..535d4c4f --- /dev/null +++ b/tests/translator/bugs-toClaude-context.test.js @@ -0,0 +1,65 @@ +// OpenAI-format CLI → Claude provider. Context pollution + lossy mapping on the openai→claude leg. +import { describe, it, expect } from "vitest"; +import "./registerAll.js"; +import { translateRequest } from "../../open-sse/translator/index.js"; +import { FORMATS } from "../../open-sse/translator/formats.js"; + +// anthropic-compatible provider so prepareClaudeRequest runs the openai→claude path +const T = (body) => + translateRequest(FORMATS.OPENAI, FORMATS.CLAUDE, "m", body, true, null, "anthropic-compatible-x"); + +describe("OpenAI → Claude context mapping", () => { + // openai-to-claude.js:124-134 — always injects CLAUDE_SYSTEM_PROMPT ("You are Claude Code") + // KNOWN BUG: pollutes requests for non-official Claude-compatible providers + it.fails("does not inject Claude Code system prompt for compatible providers", () => { + const out = T({ messages: [{ role: "user", content: "hi" }] }); + expect(JSON.stringify(out.system), "Claude Code prompt injected").not.toContain("Claude Code"); + }); + + // openai-to-claude.js:268-273 — assistant.reasoning_content not mapped to a thinking block + // KNOWN BUG + it.fails("assistant reasoning_content becomes a thinking block", () => { + const out = T({ + messages: [ + { role: "user", content: "q" }, + { role: "assistant", content: "a", reasoning_content: "my hidden reasoning" }, + { role: "user", content: "next" }, + ], + }); + expect(JSON.stringify(out), "reasoning_content lost").toContain("my hidden reasoning"); + }); + + // openai-to-claude.js:298 — tool_choice "none" mapped to {type:"auto"} (loses "do not call" intent) + // KNOWN BUG + it.fails("tool_choice=none is not turned into auto", () => { + const out = T({ + messages: [{ role: "user", content: "hi" }], + tools: [{ type: "function", function: { name: "f", parameters: { type: "object", properties: {} } } }], + tool_choice: "none", + }); + expect(out.tool_choice?.type, "none became auto → model may call tools").not.toBe("auto"); + }); + + // getContentBlocksFromMessage — no input_audio branch → audio dropped + // KNOWN BUG + it.fails("input_audio content is preserved", () => { + const out = T({ + messages: [{ role: "user", content: [ + { type: "text", text: "transcribe" }, + { type: "input_audio", input_audio: { data: "AUDIO_B64", format: "wav" } }, + ] }], + }); + expect(JSON.stringify(out), "audio dropped").toContain("AUDIO_B64"); + }); + + // openai-to-claude.js:235-251 — remote http image_url is kept (regression guard) + it("remote http image_url is preserved", () => { + const out = T({ + messages: [{ role: "user", content: [ + { type: "text", text: "see" }, + { type: "image_url", image_url: { url: "https://x.com/pic.png" } }, + ] }], + }); + expect(JSON.stringify(out), "remote image dropped").toContain("pic.png"); + }); +}); diff --git a/tests/translator/coverage-all-models.test.js b/tests/translator/coverage-all-models.test.js new file mode 100644 index 00000000..b4ef5bd6 --- /dev/null +++ b/tests/translator/coverage-all-models.test.js @@ -0,0 +1,53 @@ +// Tier 1 — Structural coverage: every model in PROVIDER_MODELS must translate +// without throwing, correct upstreamId, strip applied. Data-driven → new providers auto-covered. +import { describe, it, expect } from "vitest"; +import "./registerAll.js"; +import { translateRequest } from "../../open-sse/translator/index.js"; +import { FORMATS } from "../../open-sse/translator/formats.js"; +import { buildProviderGroups, buildModelMatrix, resolveTargetFormat } from "./matrix.js"; + +// Base OpenAI-format request with text + tool + image (exercises strip + tool paths) +function baseBody(modelId) { + return { + model: modelId, + stream: true, + max_tokens: 64, + messages: [ + { role: "system", content: "You are a helper." }, + { + role: "user", + content: [ + { type: "text", text: "Hello" }, + { type: "image_url", image_url: { url: "data:image/png;base64,AAAA" } }, + ], + }, + ], + tools: [ + { type: "function", function: { name: "get_time", description: "x", parameters: { type: "object", properties: {} } } }, + ], + }; +} + +const groups = buildProviderGroups(); + +describe("coverage: every model translates without throwing", () => { + it.each(groups)("$alias: all models OpenAI→target", ({ alias, models }) => { + for (const m of models) { + const target = resolveTargetFormat(alias, m.id); + const body = baseBody(m.id); + // source = openai (lingua franca); exercise openai → target path + const out = translateRequest(FORMATS.OPENAI, target, m.id, body, true, null, alias); + expect(out, `${alias}/${m.id} → ${target} returned falsy`).toBeTruthy(); + } + }); +}); + +const stripModels = buildModelMatrix().filter((r) => r.strip.includes("image")); +describe.skipIf(stripModels.length === 0)("coverage: image-strip models drop image content", () => { + it.each(stripModels)("$alias/$modelId strips image when strip=[image]", (row) => { + const body = baseBody(row.modelId); + const out = translateRequest(FORMATS.OPENAI, row.targetFormat, row.modelId, body, true, null, row.alias, null, row.strip); + const json = JSON.stringify(out); + expect(json).not.toContain("data:image/png"); + }); +}); diff --git a/tests/translator/format-roundtrip.test.js b/tests/translator/format-roundtrip.test.js new file mode 100644 index 00000000..efc334d2 --- /dev/null +++ b/tests/translator/format-roundtrip.test.js @@ -0,0 +1,76 @@ +// Tier 2 — Format-pair: for each CLI source format, translate to openai and verify +// core parts (text, tool, system) survive. Exposes bridge data loss. +import { describe, it, expect } from "vitest"; +import "./registerAll.js"; +import { translateRequest } from "../../open-sse/translator/index.js"; +import { FORMATS } from "../../open-sse/translator/formats.js"; + +const T = (src, tgt, body, provider = null) => + translateRequest(src, tgt, "m", body, true, null, provider); + +describe("roundtrip: Claude source preserves core fields → OpenAI", () => { + const body = { + system: "sys", + max_tokens: 100, + messages: [ + { role: "user", content: "question" }, + { role: "assistant", content: [{ type: "tool_use", id: "call_1", name: "search", input: { q: "x" } }] }, + { role: "user", content: [{ type: "tool_result", tool_use_id: "call_1", content: "result" }] }, + ], + }; + const out = T(FORMATS.CLAUDE, FORMATS.OPENAI, body); + + it("system → system role", () => { + expect(out.messages.some((m) => m.role === "system" && m.content === "sys")).toBe(true); + }); + it("tool_use → assistant.tool_calls with matching id", () => { + const asst = out.messages.find((m) => m.tool_calls); + expect(asst?.tool_calls?.[0]?.id).toBe("call_1"); + }); + it("tool_result → tool message with matching id", () => { + const tool = out.messages.find((m) => m.role === "tool"); + expect(tool?.tool_call_id).toBe("call_1"); + expect(tool?.content).toContain("result"); + }); + it("tool arguments are valid JSON string", () => { + const asst = out.messages.find((m) => m.tool_calls); + expect(() => JSON.parse(asst.tool_calls[0].function.arguments)).not.toThrow(); + }); +}); + +describe("roundtrip: OpenAI tools → Claude → keeps tool name", () => { + const out = T(FORMATS.OPENAI, FORMATS.CLAUDE, { + messages: [{ role: "user", content: "hi" }], + tools: [{ type: "function", function: { name: "my_tool", description: "d", parameters: { type: "object", properties: {} } } }], + }, "anthropic-compatible-x"); + + it("tool name survives openai→claude", () => { + expect(JSON.stringify(out)).toContain("my_tool"); + }); +}); + +describe("roundtrip: parallel tool calls keep distinct ids", () => { + // Claude assistant with 2 parallel tool_use → openai must keep 2 distinct ids + const out = T(FORMATS.CLAUDE, FORMATS.OPENAI, { + messages: [ + { role: "assistant", content: [ + { type: "tool_use", id: "call_a", name: "f1", input: {} }, + { type: "tool_use", id: "call_b", name: "f2", input: {} }, + ] }, + { role: "user", content: [ + { type: "tool_result", tool_use_id: "call_a", content: "ra" }, + { type: "tool_result", tool_use_id: "call_b", content: "rb" }, + ] }, + ], + }); + + it("two tool_calls, two distinct ids", () => { + const asst = out.messages.find((m) => m.tool_calls); + const ids = asst.tool_calls.map((tc) => tc.id); + expect(new Set(ids).size).toBe(2); + }); + it("each tool_call has a matching tool result", () => { + const toolMsgs = out.messages.filter((m) => m.role === "tool"); + expect(toolMsgs.length).toBe(2); + }); +}); diff --git a/tests/translator/matrix.js b/tests/translator/matrix.js new file mode 100644 index 00000000..bc3a4e0e --- /dev/null +++ b/tests/translator/matrix.js @@ -0,0 +1,67 @@ +// Data-driven test matrix built from PROVIDER_MODELS (single source of truth). +// Adding a new provider/model to config auto-extends coverage — no test edits needed. +import { + PROVIDER_MODELS, + PROVIDER_ID_TO_ALIAS, + getModelTargetFormat, + getModelStrip, + getModelUpstreamId, +} from "../../open-sse/config/providerModels.js"; +import { PROVIDERS } from "../../open-sse/config/providers.js"; +import { FORMATS } from "../../open-sse/translator/formats.js"; + +// Reverse alias → providerId to resolve provider-level config.format +const ALIAS_TO_PROVIDER_ID = Object.fromEntries( + Object.entries(PROVIDER_ID_TO_ALIAS).map(([id, alias]) => [alias, id]) +); + +// Provider-level format fallback (mirrors getTargetFormat without compat-url logic) +function providerFormat(alias) { + const providerId = ALIAS_TO_PROVIDER_ID[alias] || alias; + return PROVIDERS[providerId]?.format || FORMATS.OPENAI; +} + +// Resolve effective target format for an (alias, model): model override → provider format +export function resolveTargetFormat(alias, modelId) { + return getModelTargetFormat(alias, modelId) || providerFormat(alias); +} + +// Flat matrix of every model across every provider +export function buildModelMatrix() { + const rows = []; + for (const [alias, models] of Object.entries(PROVIDER_MODELS)) { + if (!Array.isArray(models)) continue; + for (const m of models) { + rows.push({ + alias, + providerId: ALIAS_TO_PROVIDER_ID[alias] || alias, + modelId: m.id, + type: m.type || "llm", + targetFormat: resolveTargetFormat(alias, m.id), + strip: getModelStrip(alias, m.id), + upstreamId: getModelUpstreamId(alias, m.id), + }); + } + } + return rows; +} + +// Distinct provider list (one representative llm model each) for grouped assertions +export function buildProviderGroups() { + const groups = []; + for (const [alias, models] of Object.entries(PROVIDER_MODELS)) { + if (!Array.isArray(models) || models.length === 0) continue; + const llm = models.filter((m) => (m.type || "llm") === "llm"); + groups.push({ alias, models: llm.length ? llm : models }); + } + return groups; +} + +// CLI source formats that real clients emit (the "specials" the user cares about) +export const CLI_SOURCE_FORMATS = [ + FORMATS.CLAUDE, + FORMATS.OPENAI_RESPONSES, + FORMATS.GEMINI, + FORMATS.OPENAI, + FORMATS.ANTIGRAVITY, +]; diff --git a/tests/translator/real/smoke-providers.real.test.js b/tests/translator/real/smoke-providers.real.test.js new file mode 100644 index 00000000..576f8fd0 --- /dev/null +++ b/tests/translator/real/smoke-providers.real.test.js @@ -0,0 +1,123 @@ +// REAL integration smoke test: sends a tiny prompt to EVERY provider that has an +// active credential in the local DB, through the full production path (handleChatCore). +// Gated by RUN_REAL=1 so the default `vitest run` never touches the network. +// +// RUN_REAL=1 npx vitest run "tests/translator/real/" +// +// Each provider becomes its own test; providers without an llm model or without an +// active credential are skipped automatically. +import { describe, it, expect, beforeAll } from "vitest"; +import { getProviderConnections } from "../../../src/lib/localDb.js"; +import { getProviderCredentials } from "../../../src/sse/services/auth.js"; +import { checkAndRefreshToken } from "../../../src/sse/services/tokenRefresh.js"; +import { handleChatCore } from "../../../open-sse/handlers/chatCore.js"; +import { getModelsByProviderId } from "../../../open-sse/config/providerModels.js"; + +const RUN_REAL = process.env.RUN_REAL === "1"; +const MAX_TOKENS = 32; +const TIMEOUT_MS = 90000; +// Optional comma-separated filter: REAL_PROVIDERS=kiro,codex,antigravity +const PROVIDER_FILTER = (process.env.REAL_PROVIDERS || "") + .split(",").map((s) => s.trim()).filter(Boolean); + +// Pick the first plain llm model for a provider (skip image/tts/embedding/etc). +function firstLlmModel(providerId) { + const models = getModelsByProviderId(providerId); + const llm = models.find((m) => (m.type || "llm") === "llm"); + return llm?.id || null; +} + +// Drain the full Web Response SSE body into raw text. +async function drainSSE(response) { + if (!response?.body) return ""; + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let out = ""; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + out += decoder.decode(value, { stream: true }); + } + return out; +} + +let providerIds = []; + +beforeAll(async () => { + if (!RUN_REAL) return; + providerIds = targetProviders(); +}); + +describe.skipIf(!RUN_REAL).concurrent("REAL provider smoke", () => { + it("has active providers in DB", () => { + expect(providerIds.length).toBeGreaterThan(0); + }); + + // One concurrent test per provider; resolved lazily inside the test. + for (const providerId of (RUN_REAL ? targetProviders() : [])) { + it.concurrent( + `${providerId}: responds to a short prompt`, + async () => { + const model = firstLlmModel(providerId); + if (!model) return expect(true).toBe(true); // no llm model → skip silently + + const credentials = await getProviderCredentials(providerId, new Set(), model); + if (!credentials || credentials.allRateLimited) { + console.warn(`[skip] ${providerId}: no usable credential`); + return expect(true).toBe(true); + } + + const refreshed = await checkAndRefreshToken(providerId, credentials); + const result = await handleChatCore({ + body: { + model: `${providerId}/${model}`, + stream: true, + max_tokens: MAX_TOKENS, + messages: [{ role: "user", content: "Reply with the single word: hi" }], + }, + modelInfo: { provider: providerId, model }, + credentials: refreshed, + connectionId: credentials.connectionId, + }); + + if (!result.success) { + // Account/quota/auth problems are credential issues, not translation bugs → skip. + const credIssue = [401, 402, 403, 429].includes(Number(result.status)); + if (credIssue) { + console.warn(`[skip] ${providerId}: ${result.status} (credential/quota)`); + return expect(true).toBe(true); + } + throw new Error(`${providerId} failed: ${result.status} ${result.error}`); + } + + const raw = await drainSSE(result.response); + // Minimal sanity: got SSE data and a terminal signal or content. + expect(raw.length, `${providerId}: empty response`).toBeGreaterThan(0); + expect(/data:|finish_reason|"delta"|"content"|event:/.test(raw), `${providerId}: not SSE`).toBe(true); + }, + TIMEOUT_MS + ); + } +}); + +// Read the DB file directly (sync) at module-eval time so vitest can generate one +// test per provider before beforeAll runs. Applies REAL_PROVIDERS filter. +// Tolerates any failure (returns []). +function targetProviders() { + try { + const Database = require("better-sqlite3"); + const os = require("os"); + const path = require("path"); + const dbPath = process.env.DATA_DIR + ? path.join(process.env.DATA_DIR, "db", "data.sqlite") + : path.join(os.homedir(), ".9router", "db", "data.sqlite"); + const db = new Database(dbPath, { readonly: true }); + const rows = db.prepare("SELECT DISTINCT provider FROM providerConnections WHERE isActive = 1").all(); + db.close(); + let list = rows.map((r) => r.provider).sort(); + if (PROVIDER_FILTER.length) list = list.filter((p) => PROVIDER_FILTER.includes(p)); + return list; + } catch { + return []; + } +} diff --git a/tests/translator/registerAll.js b/tests/translator/registerAll.js new file mode 100644 index 00000000..75793437 --- /dev/null +++ b/tests/translator/registerAll.js @@ -0,0 +1,22 @@ +// Eagerly import every translator so register() side-effects run under ESM/vitest. +// translator/index.js uses require() (bundler-only) which no-ops in vitest → import directly. +import "../../open-sse/translator/request/claude-to-openai.js"; +import "../../open-sse/translator/request/openai-to-claude.js"; +import "../../open-sse/translator/request/gemini-to-openai.js"; +import "../../open-sse/translator/request/openai-to-gemini.js"; +import "../../open-sse/translator/request/openai-to-vertex.js"; +import "../../open-sse/translator/request/antigravity-to-openai.js"; +import "../../open-sse/translator/request/openai-responses.js"; +import "../../open-sse/translator/request/openai-to-kiro.js"; +import "../../open-sse/translator/request/openai-to-cursor.js"; +import "../../open-sse/translator/request/openai-to-ollama.js"; +import "../../open-sse/translator/request/openai-to-commandcode.js"; +import "../../open-sse/translator/response/claude-to-openai.js"; +import "../../open-sse/translator/response/openai-to-claude.js"; +import "../../open-sse/translator/response/gemini-to-openai.js"; +import "../../open-sse/translator/response/openai-to-antigravity.js"; +import "../../open-sse/translator/response/openai-responses.js"; +import "../../open-sse/translator/response/kiro-to-openai.js"; +import "../../open-sse/translator/response/cursor-to-openai.js"; +import "../../open-sse/translator/response/ollama-to-openai.js"; +import "../../open-sse/translator/response/commandcode-to-openai.js"; diff --git a/tests/vitest.config.js b/tests/vitest.config.js index 0df209bf..d341b917 100644 --- a/tests/vitest.config.js +++ b/tests/vitest.config.js @@ -9,15 +9,17 @@ export default defineConfig({ environment: "node", globals: true, include: ["**/*.test.js"], + // Allow many it.concurrent cases (real provider smoke runs ~50 providers in parallel) + maxConcurrency: 60, // Suppress noisy console output from handlers under test silent: false, }, resolve: { - alias: { - // Resolve open-sse/* imports to the actual local package - "open-sse": resolve(__dirname, "../open-sse"), - // Resolve @/* imports to src directory - "@": resolve(__dirname, "../src"), - }, + // Use array form so subpath aliases (e.g. "@/lib/db/index.js") resolve correctly. + alias: [ + { find: /^open-sse\//, replacement: resolve(__dirname, "../open-sse") + "/" }, + { find: "open-sse", replacement: resolve(__dirname, "../open-sse") }, + { find: /^@\//, replacement: resolve(__dirname, "../src") + "/" }, + ], }, });