Merge PR #1628: fix(model-test) route image and STT probes to their real endpoints

Route image model tests to /api/v1/images/generations and STT to /api/v1/audio/transcriptions instead of forcing all non-embedding models through chat completions. Adds kind-aware pingModelByKind, hf->huggingface alias, and silent WAV sample for STT reachability. Scoped to dashboard/internal model testing only; runtime inference routing is unchanged. Author: yicone <yicone@gmail.com> Closes #1628
2026-09-22 13:38:31 +00:00 · 2026-06-06 10:03:32 +07:00
parent 41f94ce8c8 c980e1f7ad
commit 8671468360
7 changed files with 491 additions and 163 deletions
@@ -0,0 +1,12 @@
+import { describe, it, expect } from "vitest";
+import { parseModel } from "../../open-sse/services/model.js";
+
+describe("HuggingFace model alias parsing", () => {
+  it("resolves hf alias to huggingface provider", () => {
+    expect(parseModel("hf/black-forest-labs/FLUX.1-schnell")).toMatchObject({
+      provider: "huggingface",
+      model: "black-forest-labs/FLUX.1-schnell",
+      providerAlias: "hf",
+    });
+  });
+});
@@ -0,0 +1,194 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+  getApiKeys: vi.fn(),
+  getConsistentMachineId: vi.fn(),
+}));
+
+vi.mock("@/lib/localDb", () => ({
+  getApiKeys: mocks.getApiKeys,
+}));
+
+vi.mock("@/shared/utils/machineId", () => ({
+  getConsistentMachineId: mocks.getConsistentMachineId,
+}));
+
+vi.mock("next/server", () => ({
+  NextResponse: {
+    json(body, init = {}) {
+      return new Response(JSON.stringify(body), {
+        status: init.status || 200,
+        headers: { "Content-Type": "application/json" },
+      });
+    },
+  },
+}));
+
+const originalFetch = global.fetch;
+
+describe("model test route kind routing", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    mocks.getApiKeys.mockResolvedValue([{ key: "sk-internal", isActive: true }]);
+    mocks.getConsistentMachineId.mockResolvedValue("cli-token");
+    global.fetch = vi.fn().mockResolvedValue(new Response(JSON.stringify({
+      created: 1,
+      data: [{ b64_json: "abc" }],
+    }), {
+      status: 200,
+      headers: { "Content-Type": "application/json" },
+    }));
+  });
+
+  afterEach(() => {
+    global.fetch = originalFetch;
+  });
+
+  it("routes image model tests to /api/v1/images/generations", async () => {
+    const { POST } = await import("../../src/app/api/models/test/route.js");
+
+    const req = new Request("http://localhost/api/models/test", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        model: "hf/black-forest-labs/FLUX.1-schnell",
+        kind: "image",
+      }),
+    });
+
+    const res = await POST(req);
+    const body = await res.json();
+
+    expect(body.ok).toBe(true);
+    expect(global.fetch).toHaveBeenCalledWith(
+      expect.stringContaining("/api/v1/images/generations"),
+      expect.objectContaining({
+        method: "POST",
+        body: JSON.stringify({
+          model: "hf/black-forest-labs/FLUX.1-schnell",
+          prompt: "test",
+        }),
+      })
+    );
+  });
+
+  it("routes embedding model tests to /api/v1/embeddings", async () => {
+    global.fetch = vi.fn().mockResolvedValue(new Response(JSON.stringify({
+      data: [{ embedding: [0.1, 0.2] }],
+    }), {
+      status: 200,
+      headers: { "Content-Type": "application/json" },
+    }));
+
+    const { POST } = await import("../../src/app/api/models/test/route.js");
+
+    const req = new Request("http://localhost/api/models/test", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        model: "voyage/voyage-3-large",
+        kind: "embedding",
+      }),
+    });
+
+    const res = await POST(req);
+    const body = await res.json();
+
+    expect(body.ok).toBe(true);
+    expect(global.fetch).toHaveBeenCalledWith(
+      expect.stringContaining("/api/v1/embeddings"),
+      expect.objectContaining({
+        method: "POST",
+        body: JSON.stringify({
+          model: "voyage/voyage-3-large",
+          input: "test",
+        }),
+      })
+    );
+  });
+
+  it("fails embedding model tests when provider returns no embedding data", async () => {
+    global.fetch = vi.fn().mockResolvedValue(new Response(JSON.stringify({
+      data: [{ embedding: null }],
+    }), {
+      status: 200,
+      headers: { "Content-Type": "application/json" },
+    }));
+
+    const { POST } = await import("../../src/app/api/models/test/route.js");
+
+    const req = new Request("http://localhost/api/models/test", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        model: "voyage/voyage-3-large",
+        kind: "embedding",
+      }),
+    });
+
+    const res = await POST(req);
+    const body = await res.json();
+
+    expect(body.ok).toBe(false);
+    expect(body.error).toBe("Provider returned no embedding data");
+  });
+
+  it("routes stt model tests to /api/v1/audio/transcriptions", async () => {
+    global.fetch = vi.fn().mockResolvedValue(new Response(JSON.stringify({
+      text: "test",
+    }), {
+      status: 200,
+      headers: { "Content-Type": "application/json" },
+    }));
+
+    const { POST } = await import("../../src/app/api/models/test/route.js");
+
+    const req = new Request("http://localhost/api/models/test", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        model: "hf/openai/whisper-small",
+        kind: "stt",
+      }),
+    });
+
+    const res = await POST(req);
+    const body = await res.json();
+
+    expect(body.ok).toBe(true);
+    expect(global.fetch).toHaveBeenCalledWith(
+      expect.stringContaining("/api/v1/audio/transcriptions"),
+      expect.objectContaining({
+        method: "POST",
+        body: expect.any(FormData),
+      })
+    );
+  });
+
+  it("returns formatted HTTP errors for non-2xx embedding responses", async () => {
+    global.fetch = vi.fn().mockResolvedValue(new Response(JSON.stringify({
+      error: { message: "bad upstream" },
+    }), {
+      status: 502,
+      headers: { "Content-Type": "application/json" },
+    }));
+
+    const { POST } = await import("../../src/app/api/models/test/route.js");
+
+    const req = new Request("http://localhost/api/models/test", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        model: "voyage/voyage-3-large",
+        kind: "embedding",
+      }),
+    });
+
+    const res = await POST(req);
+    const body = await res.json();
+
+    expect(body.ok).toBe(false);
+    expect(body.status).toBe(502);
+    expect(body.error).toBe("HTTP 502: bad upstream");
+  });
+});
@@ -0,0 +1,83 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+  getProviderConnectionById: vi.fn(),
+  getApiKeys: vi.fn(),
+  getConsistentMachineId: vi.fn(),
+}));
+
+vi.mock("@/lib/localDb", () => ({
+  getProviderConnectionById: mocks.getProviderConnectionById,
+  getApiKeys: mocks.getApiKeys,
+}));
+
+vi.mock("@/shared/utils/machineId", () => ({
+  getConsistentMachineId: mocks.getConsistentMachineId,
+}));
+
+vi.mock("next/server", () => ({
+  NextResponse: {
+    json(body, init = {}) {
+      return new Response(JSON.stringify(body), {
+        status: init.status || 200,
+        headers: { "Content-Type": "application/json" },
+      });
+    },
+  },
+}));
+
+const originalFetch = global.fetch;
+
+describe("provider test-models route kind routing", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    mocks.getProviderConnectionById.mockResolvedValue({
+      id: "conn-hf",
+      provider: "huggingface",
+    });
+    mocks.getApiKeys.mockResolvedValue([{ key: "sk-internal", isActive: true }]);
+    mocks.getConsistentMachineId.mockResolvedValue("cli-token");
+    global.fetch = vi.fn((url) => {
+      if (String(url).includes("/api/v1/images/generations")) {
+        return Promise.resolve(new Response(JSON.stringify({
+          created: 1,
+          data: [{ b64_json: "abc" }],
+        }), {
+          status: 200,
+          headers: { "Content-Type": "application/json" },
+        }));
+      }
+      return Promise.resolve(new Response(JSON.stringify({
+        choices: [{ message: { role: "assistant", content: "ok" } }],
+      }), {
+        status: 200,
+        headers: { "Content-Type": "application/json" },
+      }));
+    });
+  });
+
+  afterEach(() => {
+    global.fetch = originalFetch;
+  });
+
+  it("routes huggingface image models to /api/v1/images/generations", async () => {
+    const { POST } = await import("../../src/app/api/providers/[id]/test-models/route.js");
+
+    const req = new Request("http://localhost/api/providers/conn-hf/test-models", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+    });
+
+    const res = await POST(req, { params: Promise.resolve({ id: "conn-hf" }) });
+    const body = await res.json();
+
+    expect(body.provider).toBe("huggingface");
+    expect(body.results.some((r) => r.modelId === "black-forest-labs/FLUX.1-schnell" && r.ok)).toBe(true);
+    expect(global.fetch).toHaveBeenCalledWith(
+      expect.stringContaining("/api/v1/images/generations"),
+      expect.objectContaining({
+        method: "POST",
+      })
+    );
+  });
+});