mirror of
https://github.com/Nezumi-2711/9router.git
synced 2026-09-22 13:38:31 +00:00
fix(antigravity): retry transient upstream failures
Retry short-lived 5xx/capacity errors (500/502/503/504 + message patterns) with bounded backoff capped at 15s; honor Retry-After/reset hints and skip when wait is too long. Keep 400 non-retryable. Enable the retry hook for 500 alongside existing 429/503. Deduplicate sanitized Antigravity tool names before emitting the single functionDeclarations group to avoid upstream "Tool names must be unique" rejections. Add Headroom size diagnostics and phantom-savings warning when reported token delta does not shrink the outbound payload. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
committed by
decolua
co-authored by
Cursor
parent
2deacf69b1
commit
639f1204d0
@@ -16,8 +16,26 @@ function sanitizeFunctionName(name) {
|
||||
}
|
||||
|
||||
const MAX_RETRY_AFTER_MS = 10000;
|
||||
const ANTIGRAVITY_TRANSIENT_RETRY_MAX_MS = 15000;
|
||||
const MAX_ANTIGRAVITY_OUTPUT_TOKENS = 16384;
|
||||
|
||||
const ANTIGRAVITY_TRANSIENT_ERROR_PATTERNS = [
|
||||
/high\s+traffic/i,
|
||||
/agent\s+(execution\s+)?terminated\s+due\s+to\s+error/i,
|
||||
/capacity/i,
|
||||
/temporarily\s+unavailable/i,
|
||||
/timeout/i,
|
||||
/stream\s+(ended|closed|terminated|interrupted)/i,
|
||||
/empty\s+response/i,
|
||||
];
|
||||
|
||||
const ANTIGRAVITY_TRANSIENT_STATUSES = new Set([
|
||||
HTTP_STATUS.SERVER_ERROR,
|
||||
HTTP_STATUS.BAD_GATEWAY,
|
||||
HTTP_STATUS.SERVICE_UNAVAILABLE,
|
||||
HTTP_STATUS.GATEWAY_TIMEOUT,
|
||||
]);
|
||||
|
||||
// Fields Google generateContent rejects (Claude/OpenAI/Qwen thinking fields set at body root by thinkingUnified.js)
|
||||
const ANTIGRAVITY_REQUEST_BLACKLIST = [
|
||||
"output_config",
|
||||
@@ -170,15 +188,22 @@ export class AntigravityExecutor extends BaseExecutor {
|
||||
|
||||
if (tools && tools.length > 0) {
|
||||
// Merge all groups into a single functionDeclarations group (Gemini expects 1 group)
|
||||
const allDeclarations = tools.flatMap(group =>
|
||||
(group.functionDeclarations || []).map(fn => ({
|
||||
const seenToolNames = new Set();
|
||||
const allDeclarations = [];
|
||||
for (const group of tools) {
|
||||
for (const fn of group.functionDeclarations || []) {
|
||||
const name = sanitizeFunctionName(fn.name);
|
||||
if (seenToolNames.has(name)) continue;
|
||||
seenToolNames.add(name);
|
||||
allDeclarations.push({
|
||||
...fn,
|
||||
name: sanitizeFunctionName(fn.name),
|
||||
name,
|
||||
parameters: fn.parameters
|
||||
? cleanJSONSchemaForAntigravity(structuredClone(fn.parameters))
|
||||
: { type: "object", properties: { reason: { type: "string", description: "Brief explanation" } }, required: ["reason"] }
|
||||
}))
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
tools = allDeclarations.length > 0 ? [{ functionDeclarations: allDeclarations }] : [];
|
||||
}
|
||||
|
||||
@@ -305,23 +330,49 @@ export class AntigravityExecutor extends BaseExecutor {
|
||||
return totalMs > 0 ? totalMs : null;
|
||||
}
|
||||
|
||||
// Hook called by BaseExecutor.tryRetry: derive delay from Retry-After (header → body),
|
||||
// cap at MAX_RETRY_AFTER_MS, else exponential backoff for 429. Return false to veto (fallback URL).
|
||||
async computeRetryDelay(response, attempt) {
|
||||
let retryMs = this.parseRetryHeaders(response.headers);
|
||||
if (!retryMs) {
|
||||
try {
|
||||
const errorJson = JSON.parse(await response.clone().text());
|
||||
retryMs = this.parseRetryFromErrorMessage(errorJson?.error?.message || errorJson?.message || "");
|
||||
} catch {
|
||||
// ignore parse errors → fall through to backoff
|
||||
extractErrorMessage(errorJson, bodyText = "") {
|
||||
return [
|
||||
errorJson?.error?.message,
|
||||
errorJson?.message,
|
||||
errorJson?.error,
|
||||
bodyText,
|
||||
].filter(Boolean).map(v => typeof v === "string" ? v : JSON.stringify(v)).join("\n");
|
||||
}
|
||||
|
||||
isTransientAntigravityError(status, message) {
|
||||
if (status === HTTP_STATUS.RATE_LIMITED) return true;
|
||||
if (ANTIGRAVITY_TRANSIENT_STATUSES.has(status)) return true;
|
||||
return ANTIGRAVITY_TRANSIENT_ERROR_PATTERNS.some(pattern => pattern.test(message || ""));
|
||||
}
|
||||
|
||||
// Hook called by BaseExecutor.tryRetry: derive delay from Retry-After (header → body),
|
||||
// cap at MAX_RETRY_AFTER_MS, else retry transient Antigravity failures with backoff.
|
||||
// Return false to veto (fallback URL / final error).
|
||||
async computeRetryDelay(response, attempt) {
|
||||
let bodyText = "";
|
||||
let errorJson = null;
|
||||
let retryMs = this.parseRetryHeaders(response.headers);
|
||||
|
||||
try {
|
||||
bodyText = await response.clone().text();
|
||||
errorJson = bodyText ? JSON.parse(bodyText) : null;
|
||||
} catch {
|
||||
// ignore parse errors → fall through to status/message based retry
|
||||
}
|
||||
|
||||
const errorMessage = this.extractErrorMessage(errorJson, bodyText);
|
||||
|
||||
if (!retryMs) {
|
||||
retryMs = this.parseRetryFromErrorMessage(errorMessage);
|
||||
}
|
||||
if (retryMs) return retryMs <= MAX_RETRY_AFTER_MS ? retryMs : false;
|
||||
if (response.status === HTTP_STATUS.RATE_LIMITED) {
|
||||
return Math.min(1000 * (2 ** attempt), MAX_RETRY_AFTER_MS); // exponential backoff
|
||||
}
|
||||
return false;
|
||||
|
||||
if (!this.isTransientAntigravityError(response.status, errorMessage)) return false;
|
||||
|
||||
const cap = response.status === HTTP_STATUS.RATE_LIMITED
|
||||
? MAX_RETRY_AFTER_MS
|
||||
: ANTIGRAVITY_TRANSIENT_RETRY_MAX_MS;
|
||||
return Math.min(1000 * (2 ** attempt), cap); // exponential backoff
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -32,6 +32,9 @@ export default {
|
||||
"429": {
|
||||
attempts: 3,
|
||||
},
|
||||
"500": {
|
||||
attempts: 3,
|
||||
},
|
||||
"503": {
|
||||
attempts: 3,
|
||||
},
|
||||
|
||||
@@ -32,8 +32,38 @@ describe("antigravity computeRetryDelay hook (D3)", () => {
|
||||
expect(await ag.computeRetryDelay(res(429), 3)).toBe(Math.min(1000 * 2 ** 3, MAX));
|
||||
});
|
||||
|
||||
it("503 without retry info → veto (no auto backoff)", async () => {
|
||||
expect(await ag.computeRetryDelay(res(503), 1)).toBe(false);
|
||||
it("503 without retry info → transient backoff", async () => {
|
||||
expect(await ag.computeRetryDelay(res(503), 1)).toBe(2000);
|
||||
});
|
||||
|
||||
it("retries Antigravity agent terminated body even when status is not 429", async () => {
|
||||
const r = res(500, {}, { error: { message: "Agent execution terminated due to error" } });
|
||||
expect(await ag.computeRetryDelay(r, 1)).toBe(2000);
|
||||
});
|
||||
|
||||
it("retries high traffic body", async () => {
|
||||
const r = res(500, {}, { error: { message: "Our servers are experiencing high traffic" } });
|
||||
expect(await ag.computeRetryDelay(r, 2)).toBe(4000);
|
||||
});
|
||||
|
||||
it("does not retry non-transient 400 errors", async () => {
|
||||
const r = res(400, {}, { error: { message: "Invalid request" } });
|
||||
expect(await ag.computeRetryDelay(r, 1)).toBe(false);
|
||||
});
|
||||
|
||||
it("deduplicates sanitized tool names", () => {
|
||||
const out = ag.transformRequest("claude-opus-4-6-thinking", {
|
||||
request: {
|
||||
contents: [{ role: "user", parts: [{ text: "hi" }] }],
|
||||
tools: [{ functionDeclarations: [
|
||||
{ name: "read/file", parameters: { type: "object", properties: {} } },
|
||||
{ name: "read file", parameters: { type: "object", properties: {} } },
|
||||
{ name: "read/file", parameters: { type: "object", properties: {} } },
|
||||
] }],
|
||||
},
|
||||
}, true, { projectId: "project-1", connectionId: "conn-1" });
|
||||
|
||||
expect(out.request.tools[0].functionDeclarations.map(fn => fn.name)).toEqual(["read_file"]);
|
||||
});
|
||||
|
||||
it("buildHeaders includes cached session id after transformRequest", () => {
|
||||
|
||||
@@ -85,6 +85,16 @@ describe("BaseExecutor.execute — network error retry/fallback", () => {
|
||||
});
|
||||
|
||||
describe("BaseExecutor.execute — computeRetryDelay hook veto", () => {
|
||||
it("only invokes computeRetryDelay when status has retry config", async () => {
|
||||
const ex = makeExec({ baseUrl: "https://x/api", retry: { 503: { attempts: 1, delayMs: 0 } } });
|
||||
ex.computeRetryDelay = vi.fn().mockResolvedValue(0);
|
||||
fetchMock.mockResolvedValueOnce(res(500));
|
||||
const out = await ex.execute({ model: "m", body: {}, stream: false, credentials: creds });
|
||||
expect(out.response.status).toBe(500);
|
||||
expect(ex.computeRetryDelay).not.toHaveBeenCalled();
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("hook returning false skips retry (uses fallback path)", async () => {
|
||||
const ex = makeExec({ baseUrl: "https://x/api", retry: { 429: { attempts: 5, delayMs: 0 } } });
|
||||
ex.computeRetryDelay = vi.fn().mockResolvedValue(false);
|
||||
|
||||
Reference in New Issue
Block a user