mirror of
https://github.com/decolua/9router.git
synced 2026-05-08 12:01:28 +00:00
Enhance token refresh functionality across multiple executors
- Updated refreshCredentials methods in various executors (Antigravity, Base, Default, Github, Kiro) to accept optional proxyOptions for improved proxy handling. - Modified token refresh logic to utilize proxy-aware fetch for better network management. - Enhanced usage retrieval functions to support proxy options, ensuring seamless integration with proxy configurations. - Updated ModelSelectModal and ProviderInfoCard components to incorporate kind filtering for improved user experience in model selection. - Added validation for API keys in the provider validation route, including support for webSearch/webFetch providers.
This commit is contained in:
decolua
@@ -114,11 +114,11 @@ export class AntigravityExecutor extends BaseExecutor {
|
||||
};
|
||||
}
|
||||
|
||||
async refreshCredentials(credentials, log) {
|
||||
async refreshCredentials(credentials, log, proxyOptions = null) {
|
||||
if (!credentials.refreshToken) return null;
|
||||
|
||||
try {
|
||||
const response = await fetch(OAUTH_ENDPOINTS.google.token, {
|
||||
const response = await proxyAwareFetch(OAUTH_ENDPOINTS.google.token, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/x-www-form-urlencoded", "Accept": "application/json" },
|
||||
body: new URLSearchParams({
|
||||
@@ -127,7 +127,7 @@ export class AntigravityExecutor extends BaseExecutor {
|
||||
client_id: this.config.clientId,
|
||||
client_secret: this.config.clientSecret
|
||||
})
|
||||
});
|
||||
}, proxyOptions);
|
||||
|
||||
if (!response.ok) return null;
|
||||
|
||||
|
||||
@@ -85,7 +85,7 @@ export class BaseExecutor {
|
||||
}
|
||||
|
||||
// Override in subclass for provider-specific refresh
|
||||
async refreshCredentials(credentials, log) {
|
||||
async refreshCredentials(credentials, log, proxyOptions = null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ import { PROVIDERS } from "../config/providers.js";
|
||||
import { OAUTH_ENDPOINTS, buildKimiHeaders } from "../config/appConstants.js";
|
||||
import { buildClineHeaders } from "../../src/shared/utils/clineAuth.js";
|
||||
import { getCachedClaudeHeaders } from "../utils/claudeHeaderCache.js";
|
||||
import { proxyAwareFetch } from "../utils/proxyFetch.js";
|
||||
|
||||
export class DefaultExecutor extends BaseExecutor {
|
||||
constructor(provider) {
|
||||
@@ -154,19 +155,19 @@ export class DefaultExecutor extends BaseExecutor {
|
||||
return headers;
|
||||
}
|
||||
|
||||
async refreshCredentials(credentials, log) {
|
||||
async refreshCredentials(credentials, log, proxyOptions = null) {
|
||||
if (!credentials.refreshToken) return null;
|
||||
|
||||
const refreshers = {
|
||||
claude: () => this.refreshWithJSON(OAUTH_ENDPOINTS.anthropic.token, { grant_type: "refresh_token", refresh_token: credentials.refreshToken, client_id: PROVIDERS.claude.clientId }),
|
||||
codex: () => this.refreshWithForm(OAUTH_ENDPOINTS.openai.token, { grant_type: "refresh_token", refresh_token: credentials.refreshToken, client_id: PROVIDERS.codex.clientId, scope: "openid profile email offline_access" }),
|
||||
qwen: () => this.refreshWithForm(OAUTH_ENDPOINTS.qwen.token, { grant_type: "refresh_token", refresh_token: credentials.refreshToken, client_id: PROVIDERS.qwen.clientId }),
|
||||
iflow: () => this.refreshIflow(credentials.refreshToken),
|
||||
gemini: () => this.refreshGoogle(credentials.refreshToken),
|
||||
kiro: () => this.refreshKiro(credentials.refreshToken),
|
||||
cline: () => this.refreshCline(credentials.refreshToken),
|
||||
"kimi-coding": () => this.refreshKimiCoding(credentials.refreshToken),
|
||||
kilocode: () => this.refreshKilocode(credentials.refreshToken)
|
||||
claude: () => this.refreshWithJSON(OAUTH_ENDPOINTS.anthropic.token, { grant_type: "refresh_token", refresh_token: credentials.refreshToken, client_id: PROVIDERS.claude.clientId }, proxyOptions),
|
||||
codex: () => this.refreshWithForm(OAUTH_ENDPOINTS.openai.token, { grant_type: "refresh_token", refresh_token: credentials.refreshToken, client_id: PROVIDERS.codex.clientId, scope: "openid profile email offline_access" }, proxyOptions),
|
||||
qwen: () => this.refreshWithForm(OAUTH_ENDPOINTS.qwen.token, { grant_type: "refresh_token", refresh_token: credentials.refreshToken, client_id: PROVIDERS.qwen.clientId }, proxyOptions),
|
||||
iflow: () => this.refreshIflow(credentials.refreshToken, proxyOptions),
|
||||
gemini: () => this.refreshGoogle(credentials.refreshToken, proxyOptions),
|
||||
kiro: () => this.refreshKiro(credentials.refreshToken, proxyOptions),
|
||||
cline: () => this.refreshCline(credentials.refreshToken, proxyOptions),
|
||||
"kimi-coding": () => this.refreshKimiCoding(credentials.refreshToken, proxyOptions),
|
||||
kilocode: () => this.refreshKilocode(credentials.refreshToken, proxyOptions)
|
||||
};
|
||||
|
||||
const refresher = refreshers[this.provider];
|
||||
@@ -182,69 +183,69 @@ export class DefaultExecutor extends BaseExecutor {
|
||||
}
|
||||
}
|
||||
|
||||
async refreshWithJSON(url, body) {
|
||||
const response = await fetch(url, {
|
||||
async refreshWithJSON(url, body, proxyOptions = null) {
|
||||
const response = await proxyAwareFetch(url, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", "Accept": "application/json" },
|
||||
body: JSON.stringify(body)
|
||||
});
|
||||
}, proxyOptions);
|
||||
if (!response.ok) return null;
|
||||
const tokens = await response.json();
|
||||
return { accessToken: tokens.access_token, refreshToken: tokens.refresh_token || body.refresh_token, expiresIn: tokens.expires_in };
|
||||
}
|
||||
|
||||
async refreshWithForm(url, params) {
|
||||
const response = await fetch(url, {
|
||||
async refreshWithForm(url, params, proxyOptions = null) {
|
||||
const response = await proxyAwareFetch(url, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/x-www-form-urlencoded", "Accept": "application/json" },
|
||||
body: new URLSearchParams(params)
|
||||
});
|
||||
}, proxyOptions);
|
||||
if (!response.ok) return null;
|
||||
const tokens = await response.json();
|
||||
return { accessToken: tokens.access_token, refreshToken: tokens.refresh_token || params.refresh_token, expiresIn: tokens.expires_in };
|
||||
}
|
||||
|
||||
async refreshIflow(refreshToken) {
|
||||
async refreshIflow(refreshToken, proxyOptions = null) {
|
||||
const basicAuth = btoa(`${PROVIDERS.iflow.clientId}:${PROVIDERS.iflow.clientSecret}`);
|
||||
const response = await fetch(OAUTH_ENDPOINTS.iflow.token, {
|
||||
const response = await proxyAwareFetch(OAUTH_ENDPOINTS.iflow.token, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/x-www-form-urlencoded", "Accept": "application/json", "Authorization": `Basic ${basicAuth}` },
|
||||
body: new URLSearchParams({ grant_type: "refresh_token", refresh_token: refreshToken, client_id: PROVIDERS.iflow.clientId, client_secret: PROVIDERS.iflow.clientSecret })
|
||||
});
|
||||
}, proxyOptions);
|
||||
if (!response.ok) return null;
|
||||
const tokens = await response.json();
|
||||
return { accessToken: tokens.access_token, refreshToken: tokens.refresh_token || refreshToken, expiresIn: tokens.expires_in };
|
||||
}
|
||||
|
||||
async refreshGoogle(refreshToken) {
|
||||
const response = await fetch(OAUTH_ENDPOINTS.google.token, {
|
||||
async refreshGoogle(refreshToken, proxyOptions = null) {
|
||||
const response = await proxyAwareFetch(OAUTH_ENDPOINTS.google.token, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/x-www-form-urlencoded", "Accept": "application/json" },
|
||||
body: new URLSearchParams({ grant_type: "refresh_token", refresh_token: refreshToken, client_id: this.config.clientId, client_secret: this.config.clientSecret })
|
||||
});
|
||||
}, proxyOptions);
|
||||
if (!response.ok) return null;
|
||||
const tokens = await response.json();
|
||||
return { accessToken: tokens.access_token, refreshToken: tokens.refresh_token || refreshToken, expiresIn: tokens.expires_in };
|
||||
}
|
||||
|
||||
async refreshKiro(refreshToken) {
|
||||
const response = await fetch(PROVIDERS.kiro.tokenUrl, {
|
||||
async refreshKiro(refreshToken, proxyOptions = null) {
|
||||
const response = await proxyAwareFetch(PROVIDERS.kiro.tokenUrl, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", "Accept": "application/json", "User-Agent": "kiro-cli/1.0.0" },
|
||||
body: JSON.stringify({ refreshToken })
|
||||
});
|
||||
}, proxyOptions);
|
||||
if (!response.ok) return null;
|
||||
const tokens = await response.json();
|
||||
return { accessToken: tokens.accessToken, refreshToken: tokens.refreshToken || refreshToken, expiresIn: tokens.expiresIn };
|
||||
}
|
||||
|
||||
async refreshCline(refreshToken) {
|
||||
async refreshCline(refreshToken, proxyOptions = null) {
|
||||
console.log('[DEBUG] Refreshing Cline token, refreshToken length:', refreshToken?.length);
|
||||
const response = await fetch("https://api.cline.bot/api/v1/auth/refresh", {
|
||||
const response = await proxyAwareFetch("https://api.cline.bot/api/v1/auth/refresh", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", "Accept": "application/json" },
|
||||
body: JSON.stringify({ refreshToken, grantType: "refresh_token", clientType: "extension" })
|
||||
});
|
||||
}, proxyOptions);
|
||||
console.log('[DEBUG] Cline refresh response status:', response.status);
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
@@ -260,9 +261,9 @@ export class DefaultExecutor extends BaseExecutor {
|
||||
return { accessToken: data?.accessToken, refreshToken: data?.refreshToken || refreshToken, expiresIn };
|
||||
}
|
||||
|
||||
async refreshKimiCoding(refreshToken) {
|
||||
async refreshKimiCoding(refreshToken, proxyOptions = null) {
|
||||
const kimiHeaders = buildKimiHeaders();
|
||||
const response = await fetch("https://auth.kimi.com/api/oauth/token", {
|
||||
const response = await proxyAwareFetch("https://auth.kimi.com/api/oauth/token", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
@@ -270,13 +271,13 @@ export class DefaultExecutor extends BaseExecutor {
|
||||
...kimiHeaders
|
||||
},
|
||||
body: new URLSearchParams({ grant_type: "refresh_token", refresh_token: refreshToken, client_id: "17e5f671-d194-4dfb-9706-5516cb48c098" })
|
||||
});
|
||||
}, proxyOptions);
|
||||
if (!response.ok) return null;
|
||||
const tokens = await response.json();
|
||||
return { accessToken: tokens.access_token, refreshToken: tokens.refresh_token || refreshToken, expiresIn: tokens.expires_in };
|
||||
}
|
||||
|
||||
async refreshKilocode(refreshToken) {
|
||||
async refreshKilocode(refreshToken, proxyOptions = null) {
|
||||
// Kilocode uses device code flow, no refresh token support
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -271,9 +271,9 @@ export class GithubExecutor extends BaseExecutor {
|
||||
};
|
||||
}
|
||||
|
||||
async refreshCopilotToken(githubAccessToken, log) {
|
||||
async refreshCopilotToken(githubAccessToken, log, proxyOptions = null) {
|
||||
try {
|
||||
const response = await fetch("https://api.github.com/copilot_internal/v2/token", {
|
||||
const response = await proxyAwareFetch("https://api.github.com/copilot_internal/v2/token", {
|
||||
headers: {
|
||||
"Authorization": `token ${githubAccessToken}`,
|
||||
"User-Agent": GITHUB_COPILOT.USER_AGENT,
|
||||
@@ -282,7 +282,7 @@ export class GithubExecutor extends BaseExecutor {
|
||||
"Accept": "application/json",
|
||||
"x-github-api-version": GITHUB_COPILOT.API_VERSION
|
||||
}
|
||||
});
|
||||
}, proxyOptions);
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
log?.error?.("TOKEN", `Copilot token refresh failed: ${response.status} ${errorText}`);
|
||||
@@ -297,7 +297,7 @@ export class GithubExecutor extends BaseExecutor {
|
||||
}
|
||||
}
|
||||
|
||||
async refreshGitHubToken(refreshToken, log) {
|
||||
async refreshGitHubToken(refreshToken, log, proxyOptions = null) {
|
||||
try {
|
||||
const params = {
|
||||
grant_type: "refresh_token",
|
||||
@@ -308,11 +308,11 @@ export class GithubExecutor extends BaseExecutor {
|
||||
params.client_secret = this.config.clientSecret;
|
||||
}
|
||||
|
||||
const response = await fetch(OAUTH_ENDPOINTS.github.token, {
|
||||
const response = await proxyAwareFetch(OAUTH_ENDPOINTS.github.token, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/x-www-form-urlencoded", "Accept": "application/json" },
|
||||
body: new URLSearchParams(params)
|
||||
});
|
||||
}, proxyOptions);
|
||||
if (!response.ok) return null;
|
||||
const tokens = await response.json();
|
||||
log?.info?.("TOKEN", "GitHub token refreshed");
|
||||
@@ -323,13 +323,13 @@ export class GithubExecutor extends BaseExecutor {
|
||||
}
|
||||
}
|
||||
|
||||
async refreshCredentials(credentials, log) {
|
||||
let copilotResult = await this.refreshCopilotToken(credentials.accessToken, log);
|
||||
async refreshCredentials(credentials, log, proxyOptions = null) {
|
||||
let copilotResult = await this.refreshCopilotToken(credentials.accessToken, log, proxyOptions);
|
||||
|
||||
if (!copilotResult && credentials.refreshToken) {
|
||||
const githubTokens = await this.refreshGitHubToken(credentials.refreshToken, log);
|
||||
const githubTokens = await this.refreshGitHubToken(credentials.refreshToken, log, proxyOptions);
|
||||
if (githubTokens?.accessToken) {
|
||||
copilotResult = await this.refreshCopilotToken(githubTokens.accessToken, log);
|
||||
copilotResult = await this.refreshCopilotToken(githubTokens.accessToken, log, proxyOptions);
|
||||
if (copilotResult) {
|
||||
return { ...githubTokens, copilotToken: copilotResult.token, copilotTokenExpiresAt: copilotResult.expiresAt };
|
||||
}
|
||||
|
||||
@@ -378,7 +378,7 @@ export class KiroExecutor extends BaseExecutor {
|
||||
});
|
||||
}
|
||||
|
||||
async refreshCredentials(credentials, log) {
|
||||
async refreshCredentials(credentials, log, proxyOptions = null) {
|
||||
if (!credentials.refreshToken) return null;
|
||||
|
||||
try {
|
||||
@@ -386,7 +386,8 @@ export class KiroExecutor extends BaseExecutor {
|
||||
const result = await refreshKiroToken(
|
||||
credentials.refreshToken,
|
||||
credentials.providerSpecificData,
|
||||
log
|
||||
log,
|
||||
proxyOptions
|
||||
);
|
||||
|
||||
return result;
|
||||
|
||||
237
open-sse/handlers/fetch/index.js
Normal file
237
open-sse/handlers/fetch/index.js
Normal file
@@ -0,0 +1,237 @@
|
||||
// Web Fetch handler — dispatches to firecrawl, jina-reader, tavily, exa
|
||||
// Returns normalized shape across all providers
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 15000;
|
||||
const DEFAULT_FORMAT = "markdown";
|
||||
|
||||
/**
|
||||
* @typedef {Object} FetchResult
|
||||
* @property {boolean} success
|
||||
* @property {number} [status]
|
||||
* @property {string} [error]
|
||||
* @property {Object} [data]
|
||||
*/
|
||||
|
||||
/**
|
||||
* Fetch with timeout abort.
|
||||
* @param {string} url
|
||||
* @param {RequestInit} init
|
||||
* @param {number} timeoutMs
|
||||
*/
|
||||
// Strip non-ASCII chars from header values (HTTP headers must be ByteString).
|
||||
function sanitizeHeaders(headers) {
|
||||
if (!headers) return headers;
|
||||
const out = {};
|
||||
for (const [k, v] of Object.entries(headers)) {
|
||||
out[k] = typeof v === "string" ? v.replace(/[^\x00-\xFF]/g, "").trim() : v;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function tryFetch(url, init, timeoutMs) {
|
||||
const ctrl = new AbortController();
|
||||
const timer = setTimeout(() => ctrl.abort(), timeoutMs);
|
||||
try {
|
||||
const res = await fetch(url, { ...init, headers: sanitizeHeaders(init.headers), signal: ctrl.signal });
|
||||
return { ok: true, res };
|
||||
} catch (err) {
|
||||
const isAbort = err?.name === "AbortError";
|
||||
return { ok: false, timeout: isAbort, error: err?.message || String(err) };
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
function truncate(text, max) {
|
||||
if (!text || typeof text !== "string") return text || "";
|
||||
if (!max || max <= 0) return text;
|
||||
return text.length > max ? text.slice(0, max) : text;
|
||||
}
|
||||
|
||||
function parseJinaTitle(text) {
|
||||
const m = String(text || "").match(/^\s*#\s+(.+)$/m);
|
||||
return m ? m[1].trim() : null;
|
||||
}
|
||||
|
||||
function buildData({ provider, url, title, format, text, costUsd, responseMs, upstreamMs }) {
|
||||
return {
|
||||
provider,
|
||||
url,
|
||||
title: title || null,
|
||||
content: { format, text: text || "", length: (text || "").length },
|
||||
metadata: { author: null, published_at: null, language: null },
|
||||
usage: { fetch_cost_usd: costUsd ?? null },
|
||||
metrics: { response_time_ms: responseMs, upstream_latency_ms: upstreamMs }
|
||||
};
|
||||
}
|
||||
|
||||
async function readJsonOrText(res) {
|
||||
const ct = res.headers.get("content-type") || "";
|
||||
if (ct.includes("application/json")) {
|
||||
try { return { json: await res.json() }; } catch { return { text: "" }; }
|
||||
}
|
||||
return { text: await res.text() };
|
||||
}
|
||||
|
||||
/**
|
||||
* Main handler.
|
||||
* @param {Object} params
|
||||
* @param {string} params.url
|
||||
* @param {string} [params.format]
|
||||
* @param {number} [params.maxCharacters]
|
||||
* @param {string} params.provider
|
||||
* @param {Object} [params.providerConfig]
|
||||
* @param {Object} [params.credentials]
|
||||
* @param {Function} [params.log]
|
||||
* @returns {Promise<FetchResult>}
|
||||
*/
|
||||
export async function handleFetchCore({ url, format, maxCharacters, provider, providerConfig, credentials, log }) {
|
||||
if (!url || typeof url !== "string") {
|
||||
return { success: false, status: 400, error: "url is required" };
|
||||
}
|
||||
if (!provider) {
|
||||
return { success: false, status: 400, error: "provider is required" };
|
||||
}
|
||||
|
||||
const fmt = format || DEFAULT_FORMAT;
|
||||
const timeoutMs = providerConfig?.timeoutMs || DEFAULT_TIMEOUT_MS;
|
||||
const apiKey = credentials?.apiKey || credentials?.key || credentials?.token || "";
|
||||
const costPerQuery = providerConfig?.costPerQuery ?? null;
|
||||
const startedAt = Date.now();
|
||||
|
||||
try {
|
||||
if (provider === "firecrawl") {
|
||||
return await runFirecrawl({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery, startedAt });
|
||||
}
|
||||
if (provider === "jina-reader") {
|
||||
return await runJina({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery, startedAt });
|
||||
}
|
||||
if (provider === "tavily") {
|
||||
return await runTavily({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery, startedAt });
|
||||
}
|
||||
if (provider === "exa") {
|
||||
return await runExa({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery, startedAt });
|
||||
}
|
||||
return { success: false, status: 400, error: `Unsupported provider: ${provider}` };
|
||||
} catch (err) {
|
||||
log?.("fetch handler error:", err?.message || err);
|
||||
return { success: false, status: 502, error: err?.message || "Internal fetch error" };
|
||||
}
|
||||
}
|
||||
|
||||
async function runFirecrawl({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery, startedAt }) {
|
||||
const upstreamStart = Date.now();
|
||||
const r = await tryFetch("https://api.firecrawl.dev/v1/scrape", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
...(apiKey ? { authorization: `Bearer ${apiKey}` } : {})
|
||||
},
|
||||
body: JSON.stringify({ url, formats: [fmt] })
|
||||
}, timeoutMs);
|
||||
|
||||
if (!r.ok) {
|
||||
return { success: false, status: r.timeout ? 504 : 502, error: r.error };
|
||||
}
|
||||
const upstreamMs = Date.now() - upstreamStart;
|
||||
const { json } = await readJsonOrText(r.res);
|
||||
if (!r.res.ok) {
|
||||
return { success: false, status: r.res.status, error: json?.error || `Firecrawl error: ${r.res.status}` };
|
||||
}
|
||||
const d = json?.data || {};
|
||||
const text = truncate(d.markdown || d.html || d.text || "", maxCharacters);
|
||||
const title = d.metadata?.title || null;
|
||||
return {
|
||||
success: true,
|
||||
data: buildData({
|
||||
provider: "firecrawl", url, title, format: fmt, text,
|
||||
costUsd: costPerQuery, responseMs: Date.now() - startedAt, upstreamMs
|
||||
})
|
||||
};
|
||||
}
|
||||
|
||||
async function runJina({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery, startedAt }) {
|
||||
const target = `https://r.jina.ai/${encodeURIComponent(url)}`;
|
||||
const upstreamStart = Date.now();
|
||||
const r = await tryFetch(target, {
|
||||
method: "GET",
|
||||
headers: apiKey ? { authorization: `Bearer ${apiKey}` } : {}
|
||||
}, timeoutMs);
|
||||
|
||||
if (!r.ok) {
|
||||
return { success: false, status: r.timeout ? 504 : 502, error: r.error };
|
||||
}
|
||||
const upstreamMs = Date.now() - upstreamStart;
|
||||
const body = await r.res.text();
|
||||
if (!r.res.ok) {
|
||||
return { success: false, status: r.res.status, error: body?.slice(0, 500) || `Jina error: ${r.res.status}` };
|
||||
}
|
||||
const text = truncate(body, maxCharacters);
|
||||
return {
|
||||
success: true,
|
||||
data: buildData({
|
||||
provider: "jina-reader", url, title: parseJinaTitle(body), format: fmt, text,
|
||||
costUsd: costPerQuery, responseMs: Date.now() - startedAt, upstreamMs
|
||||
})
|
||||
};
|
||||
}
|
||||
|
||||
async function runTavily({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery, startedAt }) {
|
||||
const upstreamStart = Date.now();
|
||||
const r = await tryFetch("https://api.tavily.com/extract", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
...(apiKey ? { authorization: `Bearer ${apiKey}` } : {})
|
||||
},
|
||||
body: JSON.stringify({ urls: [url], extract_depth: "basic" })
|
||||
}, timeoutMs);
|
||||
|
||||
if (!r.ok) {
|
||||
return { success: false, status: r.timeout ? 504 : 502, error: r.error };
|
||||
}
|
||||
const upstreamMs = Date.now() - upstreamStart;
|
||||
const { json } = await readJsonOrText(r.res);
|
||||
if (!r.res.ok) {
|
||||
return { success: false, status: r.res.status, error: json?.error || `Tavily error: ${r.res.status}` };
|
||||
}
|
||||
const first = json?.results?.[0] || {};
|
||||
const text = truncate(first.raw_content || "", maxCharacters);
|
||||
return {
|
||||
success: true,
|
||||
data: buildData({
|
||||
provider: "tavily", url, title: null, format: fmt, text,
|
||||
costUsd: costPerQuery, responseMs: Date.now() - startedAt, upstreamMs
|
||||
})
|
||||
};
|
||||
}
|
||||
|
||||
async function runExa({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery, startedAt }) {
|
||||
const upstreamStart = Date.now();
|
||||
const r = await tryFetch("https://api.exa.ai/contents", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
...(apiKey ? { "x-api-key": apiKey } : {})
|
||||
},
|
||||
body: JSON.stringify({ ids: [url], text: true })
|
||||
}, timeoutMs);
|
||||
|
||||
if (!r.ok) {
|
||||
return { success: false, status: r.timeout ? 504 : 502, error: r.error };
|
||||
}
|
||||
const upstreamMs = Date.now() - upstreamStart;
|
||||
const { json } = await readJsonOrText(r.res);
|
||||
if (!r.res.ok) {
|
||||
return { success: false, status: r.res.status, error: json?.error || `Exa error: ${r.res.status}` };
|
||||
}
|
||||
const first = json?.results?.[0] || {};
|
||||
const text = truncate(first.text || "", maxCharacters);
|
||||
return {
|
||||
success: true,
|
||||
data: buildData({
|
||||
provider: "exa", url, title: first.title || null, format: fmt, text,
|
||||
costUsd: costPerQuery, responseMs: Date.now() - startedAt, upstreamMs
|
||||
})
|
||||
};
|
||||
}
|
||||
371
open-sse/handlers/search/callers.js
Normal file
371
open-sse/handlers/search/callers.js
Normal file
@@ -0,0 +1,371 @@
|
||||
/**
|
||||
* Search Provider Request Builders
|
||||
*
|
||||
* Ported from OmniRoute open-sse/handlers/search.ts (lines 223-610).
|
||||
* Builds HTTP request `{ url, init }` for 10 search providers.
|
||||
*
|
||||
* @typedef {Object} SearchProviderConfig
|
||||
* @property {string} id
|
||||
* @property {string} baseUrl
|
||||
* @property {string} [method]
|
||||
*
|
||||
* @typedef {Object} ContentOptions
|
||||
* @property {boolean} [snippet]
|
||||
* @property {boolean} [full_page]
|
||||
* @property {string} [format]
|
||||
* @property {number} [max_characters]
|
||||
*
|
||||
* @typedef {Object} SearchRequestParams
|
||||
* @property {string} query
|
||||
* @property {string} searchType
|
||||
* @property {number} maxResults
|
||||
* @property {string} [token]
|
||||
* @property {string} [country]
|
||||
* @property {string} [language]
|
||||
* @property {string} [timeRange]
|
||||
* @property {number} [offset]
|
||||
* @property {string[]} [domainFilter]
|
||||
* @property {ContentOptions} [contentOptions]
|
||||
* @property {Record<string,unknown>} [providerOptions]
|
||||
* @property {Record<string,unknown>} [providerSpecificData]
|
||||
*/
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Split domain filter into includes / excludes (excludes prefixed with "-").
|
||||
* @param {string[]} [domainFilter]
|
||||
* @returns {{includes: string[], excludes: string[]}}
|
||||
*/
|
||||
export function parseDomainFilter(domainFilter) {
|
||||
if (!domainFilter?.length) return { includes: [], excludes: [] };
|
||||
const includes = domainFilter.filter((d) => !d.startsWith("-"));
|
||||
const excludes = domainFilter.filter((d) => d.startsWith("-")).map((d) => d.slice(1));
|
||||
return { includes, excludes };
|
||||
}
|
||||
|
||||
/**
|
||||
* Read string setting from providerOptions first, then providerSpecificData.
|
||||
* @param {SearchRequestParams} params
|
||||
* @param {string} key
|
||||
* @returns {string|undefined}
|
||||
*/
|
||||
export function getProviderSetting(params, key) {
|
||||
const fromOptions = params.providerOptions?.[key];
|
||||
if (typeof fromOptions === "string" && fromOptions.trim().length > 0) {
|
||||
return fromOptions.trim();
|
||||
}
|
||||
const fromProviderData = params.providerSpecificData?.[key];
|
||||
if (typeof fromProviderData === "string" && fromProviderData.trim().length > 0) {
|
||||
return fromProviderData.trim();
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve base URL with optional override from providerOptions.baseUrl.
|
||||
* @param {SearchProviderConfig} config
|
||||
* @param {SearchRequestParams} params
|
||||
* @returns {string}
|
||||
*/
|
||||
export function resolveBaseUrl(config, params) {
|
||||
const override = getProviderSetting(params, "baseUrl");
|
||||
return (override || config.baseUrl).replace(/\/+$/, "");
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert offset+maxResults to 1-indexed page number.
|
||||
* @param {number|undefined} offset
|
||||
* @param {number} maxResults
|
||||
* @returns {number|undefined}
|
||||
*/
|
||||
export function toPageNumber(offset, maxResults) {
|
||||
if (typeof offset !== "number" || offset <= 0 || maxResults <= 0) return undefined;
|
||||
return Math.floor(offset / maxResults) + 1;
|
||||
}
|
||||
|
||||
// ── Provider Request Builders ───────────────────────────────────────────
|
||||
|
||||
function buildSerperRequest(config, params) {
|
||||
const endpoint = params.searchType === "news" ? "/news" : "/search";
|
||||
const body = { q: params.query, num: params.maxResults };
|
||||
if (params.country) body.gl = params.country.toLowerCase();
|
||||
if (params.language) body.hl = params.language;
|
||||
return {
|
||||
url: `${resolveBaseUrl(config, params)}${endpoint}`,
|
||||
init: {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", "X-API-Key": params.token },
|
||||
body: JSON.stringify(body),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function buildBraveRequest(config, params) {
|
||||
const endpoint = params.searchType === "news" ? "/news/search" : "/web/search";
|
||||
const qp = new URLSearchParams({ q: params.query, count: String(params.maxResults) });
|
||||
if (params.country) qp.set("country", params.country);
|
||||
if (params.language) qp.set("search_lang", params.language);
|
||||
return {
|
||||
url: `${resolveBaseUrl(config, params)}${endpoint}?${qp}`,
|
||||
init: {
|
||||
method: "GET",
|
||||
headers: { Accept: "application/json", "X-Subscription-Token": params.token },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function buildPerplexityRequest(config, params) {
|
||||
const body = { query: params.query, max_results: params.maxResults };
|
||||
if (params.country) body.country = params.country;
|
||||
if (params.language) body.search_language_filter = [params.language];
|
||||
if (params.domainFilter?.length) body.search_domain_filter = params.domainFilter;
|
||||
return {
|
||||
url: resolveBaseUrl(config, params),
|
||||
init: {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", Authorization: `Bearer ${params.token}` },
|
||||
body: JSON.stringify(body),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function buildExaRequest(config, params) {
|
||||
const { includes, excludes } = parseDomainFilter(params.domainFilter);
|
||||
const body = {
|
||||
query: params.query,
|
||||
numResults: params.maxResults,
|
||||
type: "auto",
|
||||
text: true,
|
||||
highlights: true,
|
||||
};
|
||||
if (includes.length) body.includeDomains = includes;
|
||||
if (excludes.length) body.excludeDomains = excludes;
|
||||
if (params.searchType === "news") body.category = "news";
|
||||
return {
|
||||
url: resolveBaseUrl(config, params),
|
||||
init: {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", "x-api-key": params.token },
|
||||
body: JSON.stringify(body),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function buildTavilyRequest(config, params) {
|
||||
const { includes, excludes } = parseDomainFilter(params.domainFilter);
|
||||
const body = {
|
||||
query: params.query,
|
||||
max_results: params.maxResults,
|
||||
topic: params.searchType === "news" ? "news" : "general",
|
||||
};
|
||||
if (includes.length) body.include_domains = includes;
|
||||
if (excludes.length) body.exclude_domains = excludes;
|
||||
if (params.country) body.country = params.country;
|
||||
return {
|
||||
url: resolveBaseUrl(config, params),
|
||||
init: {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", Authorization: `Bearer ${params.token}` },
|
||||
body: JSON.stringify(body),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function buildGooglePseRequest(config, params) {
|
||||
const apiKey = params.token;
|
||||
const cx = getProviderSetting(params, "cx");
|
||||
if (!apiKey || !cx) {
|
||||
throw new Error("Google Programmable Search requires both apiKey and cx");
|
||||
}
|
||||
const qp = new URLSearchParams({
|
||||
key: apiKey,
|
||||
cx,
|
||||
q: params.query,
|
||||
num: String(Math.min(params.maxResults, 10)),
|
||||
});
|
||||
if (params.country) qp.set("gl", params.country.toLowerCase());
|
||||
if (params.language) qp.set("hl", params.language);
|
||||
if (params.timeRange && params.timeRange !== "any") {
|
||||
const dateRestrictMap = { day: "d1", week: "w1", month: "m1", year: "y1" };
|
||||
const dateRestrict = dateRestrictMap[params.timeRange];
|
||||
if (dateRestrict) qp.set("dateRestrict", dateRestrict);
|
||||
}
|
||||
if (typeof params.offset === "number" && params.offset > 0) {
|
||||
qp.set("start", String(Math.min(params.offset + 1, 91)));
|
||||
}
|
||||
return {
|
||||
url: `${resolveBaseUrl(config, params)}?${qp}`,
|
||||
init: {
|
||||
method: "GET",
|
||||
headers: { Accept: "application/json" },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function buildLinkupRequest(config, params) {
|
||||
const apiKey = params.token;
|
||||
if (!apiKey) throw new Error("Linkup Search requires an API key");
|
||||
|
||||
const { includes, excludes } = parseDomainFilter(params.domainFilter);
|
||||
const requestedDepth = getProviderSetting(params, "depth");
|
||||
const depth =
|
||||
requestedDepth && ["fast", "standard", "deep"].includes(requestedDepth)
|
||||
? requestedDepth
|
||||
: "standard";
|
||||
|
||||
const body = {
|
||||
q: params.query,
|
||||
depth,
|
||||
outputType: "searchResults",
|
||||
maxResults: params.maxResults,
|
||||
};
|
||||
if (includes.length) body.includeDomains = includes;
|
||||
if (excludes.length) body.excludeDomains = excludes;
|
||||
if (params.timeRange && params.timeRange !== "any") {
|
||||
const today = new Date();
|
||||
const toDate = today.toISOString().slice(0, 10);
|
||||
const from = new Date(today);
|
||||
if (params.timeRange === "day") from.setUTCDate(from.getUTCDate() - 1);
|
||||
if (params.timeRange === "week") from.setUTCDate(from.getUTCDate() - 7);
|
||||
if (params.timeRange === "month") from.setUTCMonth(from.getUTCMonth() - 1);
|
||||
if (params.timeRange === "year") from.setUTCFullYear(from.getUTCFullYear() - 1);
|
||||
body.fromDate = from.toISOString().slice(0, 10);
|
||||
body.toDate = toDate;
|
||||
}
|
||||
|
||||
return {
|
||||
url: resolveBaseUrl(config, params),
|
||||
init: {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}` },
|
||||
body: JSON.stringify(body),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function buildSearchApiRequest(config, params) {
|
||||
const apiKey = params.token;
|
||||
if (!apiKey) throw new Error("SearchAPI requires an API key");
|
||||
|
||||
const qp = new URLSearchParams({
|
||||
engine: params.searchType === "news" ? "google_news" : "google",
|
||||
q: params.query,
|
||||
api_key: apiKey,
|
||||
});
|
||||
if (params.country) qp.set("gl", params.country.toLowerCase());
|
||||
if (params.language) qp.set("hl", params.language);
|
||||
|
||||
const page = toPageNumber(params.offset, params.maxResults);
|
||||
if (page) qp.set("page", String(page));
|
||||
|
||||
return {
|
||||
url: `${resolveBaseUrl(config, params)}?${qp}`,
|
||||
init: {
|
||||
method: "GET",
|
||||
headers: { Accept: "application/json" },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function buildYouComRequest(config, params) {
|
||||
const apiKey = params.token;
|
||||
if (!apiKey) throw new Error("You.com Search requires an API key");
|
||||
|
||||
const { includes, excludes } = parseDomainFilter(params.domainFilter);
|
||||
const qp = new URLSearchParams({
|
||||
query: params.query,
|
||||
count: String(Math.min(params.maxResults, 100)),
|
||||
});
|
||||
|
||||
if (params.timeRange && params.timeRange !== "any") qp.set("freshness", params.timeRange);
|
||||
if (typeof params.offset === "number" && params.offset > 0 && params.maxResults > 0) {
|
||||
qp.set("offset", String(Math.min(Math.floor(params.offset / params.maxResults), 9)));
|
||||
}
|
||||
if (params.country) qp.set("country", params.country);
|
||||
if (params.language) qp.set("language", params.language);
|
||||
if (includes.length) qp.set("include_domains", includes.join(","));
|
||||
if (excludes.length) qp.set("exclude_domains", excludes.join(","));
|
||||
|
||||
if (params.contentOptions?.full_page) {
|
||||
qp.set("livecrawl", params.searchType === "news" ? "news" : "web");
|
||||
qp.append(
|
||||
"livecrawl_formats",
|
||||
params.contentOptions.format === "markdown" ? "markdown" : "html"
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
url: `${resolveBaseUrl(config, params)}?${qp}`,
|
||||
init: {
|
||||
method: "GET",
|
||||
headers: { Accept: "application/json", "X-API-Key": apiKey },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function buildSearxngRequest(config, params) {
|
||||
const baseUrl = resolveBaseUrl(config, params);
|
||||
const url = baseUrl.endsWith("/search") ? baseUrl : `${baseUrl}/search`;
|
||||
const qp = new URLSearchParams({
|
||||
q: params.query,
|
||||
format: "json",
|
||||
categories: params.searchType === "news" ? "news" : "general",
|
||||
});
|
||||
if (params.language) qp.set("language", params.language);
|
||||
if (params.timeRange && params.timeRange !== "any") qp.set("time_range", params.timeRange);
|
||||
|
||||
const page = toPageNumber(params.offset, params.maxResults);
|
||||
if (page) qp.set("pageno", String(page));
|
||||
|
||||
return {
|
||||
url: `${url}?${qp}`,
|
||||
init: {
|
||||
method: "GET",
|
||||
headers: { Accept: "application/json" },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── Dispatcher ──────────────────────────────────────────────────────────
|
||||
|
||||
const BUILDERS = {
|
||||
"serper": buildSerperRequest,
|
||||
"brave-search": buildBraveRequest,
|
||||
"perplexity": buildPerplexityRequest,
|
||||
"exa": buildExaRequest,
|
||||
"tavily": buildTavilyRequest,
|
||||
"google-pse": buildGooglePseRequest,
|
||||
"linkup": buildLinkupRequest,
|
||||
"searchapi": buildSearchApiRequest,
|
||||
"youcom": buildYouComRequest,
|
||||
"searxng": buildSearxngRequest,
|
||||
};
|
||||
|
||||
/**
|
||||
* Dispatch to the correct provider builder by `provider.id`.
|
||||
* Falls back to generic POST + bearer auth for unknown providers.
|
||||
* @param {SearchProviderConfig} provider
|
||||
* @param {SearchRequestParams} params
|
||||
* @returns {{url: string, init: RequestInit}}
|
||||
*/
|
||||
export function buildSearchRequest(provider, params) {
|
||||
const builder = BUILDERS[provider.id];
|
||||
if (builder) return builder(provider, params);
|
||||
|
||||
return {
|
||||
url: resolveBaseUrl(provider, params),
|
||||
init: {
|
||||
method: provider.method || "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
...(params.token ? { Authorization: `Bearer ${params.token}` } : {}),
|
||||
},
|
||||
body: JSON.stringify({
|
||||
query: params.query,
|
||||
max_results: params.maxResults,
|
||||
search_type: params.searchType,
|
||||
}),
|
||||
},
|
||||
};
|
||||
}
|
||||
409
open-sse/handlers/search/chatSearch.js
Normal file
409
open-sse/handlers/search/chatSearch.js
Normal file
@@ -0,0 +1,409 @@
|
||||
/**
|
||||
* Wrap chat-completions endpoints (with built-in web search) into the unified
|
||||
* /v1/search response format. Supports gemini, openai, xai, kimi, minimax, perplexity.
|
||||
*/
|
||||
|
||||
const REQUEST_TIMEOUT_MS = 15000;
|
||||
const DEFAULT_MAX_RESULTS = 10;
|
||||
|
||||
/**
|
||||
* Normalize a citation entry into the unified result shape.
|
||||
* @param {{url:string, title?:string, snippet?:string}} c
|
||||
* @param {number} index
|
||||
* @param {string} provider
|
||||
* @param {string} retrievedAt
|
||||
*/
|
||||
function toResult(c, index, provider, retrievedAt) {
|
||||
return {
|
||||
title: c.title || "",
|
||||
url: c.url,
|
||||
snippet: c.snippet || "",
|
||||
position: index + 1,
|
||||
score: null,
|
||||
published_at: null,
|
||||
favicon_url: null,
|
||||
content: null,
|
||||
metadata: {},
|
||||
citation: { provider, retrieved_at: retrievedAt, rank: index + 1 },
|
||||
provider_raw: null
|
||||
};
|
||||
}
|
||||
|
||||
/** Coerce a citation that might be a raw URL string or an object. */
|
||||
function normalizeCitation(c) {
|
||||
if (!c) return null;
|
||||
if (typeof c === "string") return { url: c };
|
||||
if (typeof c === "object" && c.url) return c;
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Provider-specific configuration map. All providers must implement:
|
||||
* { endpoint, defaultModel, buildBody, buildHeaders, extractAnswer }
|
||||
*/
|
||||
const CHAT_SEARCH_CONFIG = {
|
||||
gemini: {
|
||||
endpoint: (model) =>
|
||||
`https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent`,
|
||||
defaultModel: "gemini-2.5-flash",
|
||||
buildBody: (query) => ({
|
||||
contents: [{ role: "user", parts: [{ text: query }] }],
|
||||
tools: [{ google_search: {} }]
|
||||
}),
|
||||
buildHeaders: (token) => ({
|
||||
"Content-Type": "application/json",
|
||||
"x-goog-api-key": token
|
||||
}),
|
||||
extractAnswer: (data) => {
|
||||
const candidate = data?.candidates?.[0];
|
||||
const parts = candidate?.content?.parts || [];
|
||||
const text = parts.map((p) => p?.text || "").filter(Boolean).join("");
|
||||
const chunks = candidate?.groundingMetadata?.groundingChunks || [];
|
||||
const citations = chunks
|
||||
.map((ch) => ch?.web)
|
||||
.filter(Boolean)
|
||||
.map((w) => ({ url: w.uri || w.url, title: w.title || "" }))
|
||||
.filter((c) => c.url);
|
||||
const tokens = data?.usageMetadata?.totalTokenCount || 0;
|
||||
return { text, citations, tokens };
|
||||
}
|
||||
},
|
||||
|
||||
openai: {
|
||||
endpoint: () => "https://api.openai.com/v1/chat/completions",
|
||||
defaultModel: "gpt-4o-mini",
|
||||
buildBody: (query, model) => {
|
||||
const body = {
|
||||
model,
|
||||
messages: [{ role: "user", content: query }]
|
||||
};
|
||||
// Non-search-preview models need explicit web_search tool
|
||||
if (!/search/i.test(model)) {
|
||||
body.tools = [{ type: "web_search" }];
|
||||
}
|
||||
return body;
|
||||
},
|
||||
buildHeaders: (token) => ({
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${token}`
|
||||
}),
|
||||
extractAnswer: (data) => {
|
||||
const msg = data?.choices?.[0]?.message || {};
|
||||
const text = msg.content || "";
|
||||
const annotations = Array.isArray(msg.annotations) ? msg.annotations : [];
|
||||
const fromAnn = annotations
|
||||
.map((a) => a?.url_citation)
|
||||
.filter(Boolean)
|
||||
.map((u) => ({ url: u.url, title: u.title || "" }));
|
||||
const fromTop = Array.isArray(data?.citations)
|
||||
? data.citations.map(normalizeCitation).filter(Boolean)
|
||||
: [];
|
||||
const citations = fromAnn.length ? fromAnn : fromTop;
|
||||
const tokens = data?.usage?.total_tokens || 0;
|
||||
return { text, citations, tokens };
|
||||
}
|
||||
},
|
||||
|
||||
xai: {
|
||||
endpoint: () => "https://api.x.ai/v1/responses",
|
||||
defaultModel: "grok-4.20-reasoning",
|
||||
buildBody: (query, model) => ({
|
||||
model,
|
||||
input: [{ role: "user", content: query }],
|
||||
tools: [{ type: "web_search" }]
|
||||
}),
|
||||
buildHeaders: (token) => ({
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${token}`
|
||||
}),
|
||||
extractAnswer: (data) => {
|
||||
// /v1/responses returns output[] array of message/tool blocks
|
||||
const output = Array.isArray(data?.output) ? data.output : [];
|
||||
let text = "";
|
||||
const citations = [];
|
||||
for (const item of output) {
|
||||
const parts = Array.isArray(item?.content) ? item.content : [];
|
||||
for (const p of parts) {
|
||||
if (typeof p?.text === "string") text += p.text;
|
||||
const anns = Array.isArray(p?.annotations) ? p.annotations : [];
|
||||
for (const a of anns) {
|
||||
const c = normalizeCitation(a?.url ? a : a?.url_citation);
|
||||
if (c) citations.push(c);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Fallback: top-level citations array (some response variants)
|
||||
if (!citations.length && Array.isArray(data?.citations)) {
|
||||
for (const c of data.citations) {
|
||||
const n = normalizeCitation(c);
|
||||
if (n) citations.push(n);
|
||||
}
|
||||
}
|
||||
const tokens = data?.usage?.total_tokens || 0;
|
||||
return { text, citations, tokens };
|
||||
}
|
||||
},
|
||||
|
||||
kimi: {
|
||||
endpoint: () => "https://api.moonshot.cn/v1/chat/completions",
|
||||
defaultModel: "kimi-k2.5",
|
||||
buildBody: (query, model) => ({
|
||||
model,
|
||||
messages: [{ role: "user", content: query }],
|
||||
tools: [
|
||||
{ type: "builtin_function", function: { name: "$web_search" } }
|
||||
]
|
||||
}),
|
||||
buildHeaders: (token) => ({
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${token}`
|
||||
}),
|
||||
extractAnswer: (data) => {
|
||||
const msg = data?.choices?.[0]?.message || {};
|
||||
const text = msg.content || "";
|
||||
const calls = Array.isArray(msg.tool_calls) ? msg.tool_calls : [];
|
||||
const citations = [];
|
||||
for (const call of calls) {
|
||||
const argStr = call?.function?.arguments;
|
||||
if (!argStr) continue;
|
||||
let parsed;
|
||||
try {
|
||||
parsed = typeof argStr === "string" ? JSON.parse(argStr) : argStr;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const items =
|
||||
parsed?.search_results ||
|
||||
parsed?.results ||
|
||||
parsed?.references ||
|
||||
[];
|
||||
if (Array.isArray(items)) {
|
||||
for (const it of items) {
|
||||
const url = it?.url || it?.link;
|
||||
if (!url) continue;
|
||||
citations.push({
|
||||
url,
|
||||
title: it.title || "",
|
||||
snippet: it.snippet || it.summary || ""
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
const tokens = data?.usage?.total_tokens || 0;
|
||||
return { text, citations, tokens };
|
||||
}
|
||||
},
|
||||
|
||||
minimax: {
|
||||
endpoint: () => "https://api.minimaxi.com/v1/text/chatcompletion_v2",
|
||||
defaultModel: "MiniMax-M2.7",
|
||||
buildBody: (query, model) => ({
|
||||
model,
|
||||
messages: [{ role: "user", content: query }],
|
||||
tools: [{ type: "web_search" }]
|
||||
}),
|
||||
buildHeaders: (token) => ({
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${token}`
|
||||
}),
|
||||
extractAnswer: (data) => {
|
||||
const msg = data?.choices?.[0]?.message || {};
|
||||
const text = msg.content || "";
|
||||
const citations = [];
|
||||
const direct = Array.isArray(data?.web_search_results)
|
||||
? data.web_search_results
|
||||
: [];
|
||||
for (const it of direct) {
|
||||
const url = it?.url || it?.link;
|
||||
if (url) {
|
||||
citations.push({
|
||||
url,
|
||||
title: it.title || "",
|
||||
snippet: it.snippet || it.summary || ""
|
||||
});
|
||||
}
|
||||
}
|
||||
if (!citations.length) {
|
||||
const calls = Array.isArray(msg.tool_calls) ? msg.tool_calls : [];
|
||||
for (const call of calls) {
|
||||
const argStr = call?.function?.arguments;
|
||||
if (!argStr) continue;
|
||||
let parsed;
|
||||
try {
|
||||
parsed = typeof argStr === "string" ? JSON.parse(argStr) : argStr;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const items = parsed?.results || parsed?.search_results || [];
|
||||
if (Array.isArray(items)) {
|
||||
for (const it of items) {
|
||||
const url = it?.url || it?.link;
|
||||
if (!url) continue;
|
||||
citations.push({
|
||||
url,
|
||||
title: it.title || "",
|
||||
snippet: it.snippet || ""
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
const tokens = data?.usage?.total_tokens || 0;
|
||||
return { text, citations, tokens };
|
||||
}
|
||||
},
|
||||
|
||||
perplexity: {
|
||||
endpoint: () => "https://api.perplexity.ai/chat/completions",
|
||||
defaultModel: "sonar",
|
||||
buildBody: (query, model) => ({
|
||||
model,
|
||||
messages: [{ role: "user", content: query }]
|
||||
}),
|
||||
buildHeaders: (token) => ({
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${token}`
|
||||
}),
|
||||
extractAnswer: (data) => {
|
||||
const msg = data?.choices?.[0]?.message || {};
|
||||
const text = msg.content || "";
|
||||
const raw = data?.citations || [];
|
||||
const citations = Array.isArray(raw)
|
||||
? raw.map(normalizeCitation).filter(Boolean)
|
||||
: [];
|
||||
const tokens = data?.usage?.total_tokens || 0;
|
||||
return { text, citations, tokens };
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Execute a chat-search request against the chosen provider.
|
||||
* @param {object} params
|
||||
* @param {string} params.provider
|
||||
* @param {string} params.query
|
||||
* @param {number} [params.maxResults]
|
||||
* @param {string} [params.model]
|
||||
* @param {{apiKey?:string, accessToken?:string}} params.credentials
|
||||
* @param {{info?:Function, warn?:Function, error?:Function}} [params.log]
|
||||
* @returns {Promise<{success:boolean, status?:number, error?:string, data?:object}>}
|
||||
*/
|
||||
export async function handleChatSearch({
|
||||
provider,
|
||||
query,
|
||||
maxResults,
|
||||
model,
|
||||
credentials,
|
||||
log
|
||||
}) {
|
||||
const startTime = Date.now();
|
||||
const cfg = CHAT_SEARCH_CONFIG[provider];
|
||||
|
||||
if (!cfg) {
|
||||
return {
|
||||
success: false,
|
||||
status: 400,
|
||||
error: `Unsupported chat-search provider: ${provider}`
|
||||
};
|
||||
}
|
||||
|
||||
if (!query || typeof query !== "string") {
|
||||
return { success: false, status: 400, error: "Missing query" };
|
||||
}
|
||||
|
||||
const token = credentials?.apiKey || credentials?.accessToken;
|
||||
if (!token) {
|
||||
return {
|
||||
success: false,
|
||||
status: 401,
|
||||
error: "Missing credentials (apiKey or accessToken)"
|
||||
};
|
||||
}
|
||||
|
||||
const limit =
|
||||
Number.isFinite(maxResults) && maxResults > 0
|
||||
? Math.floor(maxResults)
|
||||
: DEFAULT_MAX_RESULTS;
|
||||
const useModel = model || cfg.defaultModel;
|
||||
const url = cfg.endpoint(useModel);
|
||||
const body = cfg.buildBody(query, useModel);
|
||||
const headers = cfg.buildHeaders(token);
|
||||
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
|
||||
|
||||
let upstreamStart = Date.now();
|
||||
let resp;
|
||||
try {
|
||||
resp = await fetch(url, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify(body),
|
||||
signal: controller.signal
|
||||
});
|
||||
} catch (err) {
|
||||
clearTimeout(timer);
|
||||
if (err?.name === "AbortError") {
|
||||
log?.warn?.(`[chatSearch] timeout provider=${provider}`);
|
||||
return { success: false, status: 504, error: "Upstream timeout" };
|
||||
}
|
||||
log?.error?.(`[chatSearch] network error provider=${provider}: ${err?.message}`);
|
||||
return {
|
||||
success: false,
|
||||
status: 502,
|
||||
error: `Network error: ${err?.message || "unknown"}`
|
||||
};
|
||||
}
|
||||
clearTimeout(timer);
|
||||
const upstreamLatency = Date.now() - upstreamStart;
|
||||
|
||||
let data;
|
||||
try {
|
||||
data = await resp.json();
|
||||
} catch {
|
||||
return {
|
||||
success: false,
|
||||
status: 502,
|
||||
error: `Invalid upstream response (status ${resp.status})`
|
||||
};
|
||||
}
|
||||
|
||||
if (!resp.ok) {
|
||||
const errMsg =
|
||||
data?.error?.message ||
|
||||
data?.error ||
|
||||
data?.message ||
|
||||
`Upstream HTTP ${resp.status}`;
|
||||
log?.warn?.(`[chatSearch] upstream error provider=${provider} status=${resp.status}`);
|
||||
return {
|
||||
success: false,
|
||||
status: resp.status,
|
||||
error: typeof errMsg === "string" ? errMsg : JSON.stringify(errMsg)
|
||||
};
|
||||
}
|
||||
|
||||
const { text, citations, tokens } = cfg.extractAnswer(data);
|
||||
const retrievedAt = new Date().toISOString();
|
||||
const limited = (citations || []).slice(0, limit);
|
||||
const results = limited.map((c, i) => toResult(c, i, provider, retrievedAt));
|
||||
|
||||
return {
|
||||
success: true,
|
||||
status: 200,
|
||||
data: {
|
||||
provider,
|
||||
query,
|
||||
results,
|
||||
answer: { source: provider, text: text || "", model: useModel },
|
||||
usage: { queries_used: 1, search_cost_usd: 0, llm_tokens: tokens || 0 },
|
||||
metrics: {
|
||||
response_time_ms: Date.now() - startTime,
|
||||
upstream_latency_ms: upstreamLatency,
|
||||
total_results_available: null
|
||||
},
|
||||
errors: []
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export { CHAT_SEARCH_CONFIG };
|
||||
201
open-sse/handlers/search/index.js
Normal file
201
open-sse/handlers/search/index.js
Normal file
@@ -0,0 +1,201 @@
|
||||
/**
|
||||
* Search Dispatcher — routes /v1/search requests to dedicated search APIs
|
||||
* or chat-based LLM search wrappers, with retry-friendly error envelope.
|
||||
*
|
||||
* Dependency map:
|
||||
* provider.searchConfig → dedicated search API (callers + normalizers)
|
||||
* provider.searchViaChat → wrap chat-completions (chatSearch.js)
|
||||
*/
|
||||
|
||||
import { buildSearchRequest } from "./callers.js";
|
||||
import { normalizeSearchResponse } from "./normalizers.js";
|
||||
import { handleChatSearch } from "./chatSearch.js";
|
||||
|
||||
const GLOBAL_TIMEOUT_MS = 15000;
|
||||
const NON_RETRIABLE = new Set([400, 401, 403, 404]);
|
||||
|
||||
const CONTROL_CHAR_RE = /[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/;
|
||||
|
||||
/** Normalize and validate query string. */
|
||||
function sanitizeQuery(query) {
|
||||
if (CONTROL_CHAR_RE.test(query)) return { error: "Query contains invalid control characters" };
|
||||
const clean = query.normalize("NFKC").trim().replace(/\s+/g, " ");
|
||||
if (!clean) return { error: "Query is empty after normalization" };
|
||||
return { clean };
|
||||
}
|
||||
|
||||
// Strip non-ASCII chars from header values (HTTP headers must be ByteString).
|
||||
function sanitizeHeaders(headers) {
|
||||
if (!headers) return headers;
|
||||
const out = {};
|
||||
for (const [k, v] of Object.entries(headers)) {
|
||||
out[k] = typeof v === "string" ? v.replace(/[^\x00-\xFF]/g, "").trim() : v;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Build a JSON Response wrapper used by the auth layer. */
|
||||
function jsonResponse(payload, status = 200) {
|
||||
return new Response(JSON.stringify(payload), {
|
||||
status,
|
||||
headers: { "Content-Type": "application/json", "Access-Control-Allow-Origin": "*" }
|
||||
});
|
||||
}
|
||||
|
||||
/** Wrap an error result with a Response object so the auth wrapper can return it directly. */
|
||||
function errorResult(status, error) {
|
||||
return {
|
||||
success: false,
|
||||
status,
|
||||
error,
|
||||
response: jsonResponse({ error: { message: error, code: status } }, status)
|
||||
};
|
||||
}
|
||||
|
||||
/** Wrap a success payload. */
|
||||
function successResult(data) {
|
||||
return { success: true, data, response: jsonResponse(data, 200) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a single dedicated search provider attempt.
|
||||
* @returns {Promise<{success:boolean, status?:number, error?:string, data?:object}>}
|
||||
*/
|
||||
async function tryDedicatedProvider({ provider, providerConfig, body, credentials, log, globalStartTime }) {
|
||||
const startTime = Date.now();
|
||||
const token = credentials?.apiKey || credentials?.accessToken || undefined;
|
||||
|
||||
if (providerConfig.authType !== "none" && !token) {
|
||||
return { success: false, status: 401, error: `No credentials for provider: ${provider.id}` };
|
||||
}
|
||||
|
||||
const params = {
|
||||
query: body.query,
|
||||
searchType: body.search_type || (providerConfig.searchTypes?.[0] || "web"),
|
||||
maxResults: Math.min(body.max_results || providerConfig.defaultMaxResults || 5, providerConfig.maxMaxResults || 100),
|
||||
token,
|
||||
country: body.country,
|
||||
language: body.language,
|
||||
timeRange: body.time_range,
|
||||
offset: body.offset,
|
||||
domainFilter: body.domain_filter,
|
||||
contentOptions: body.content_options,
|
||||
providerOptions: body.provider_options,
|
||||
providerSpecificData: credentials?.providerSpecificData
|
||||
};
|
||||
|
||||
let url, init;
|
||||
try {
|
||||
({ url, init } = buildSearchRequest({ id: provider.id, ...providerConfig }, params));
|
||||
} catch (err) {
|
||||
return { success: false, status: 400, error: err?.message || `Invalid request for ${provider.id}` };
|
||||
}
|
||||
|
||||
// Timeout = min(provider timeout, remaining global)
|
||||
const remaining = GLOBAL_TIMEOUT_MS - (Date.now() - globalStartTime);
|
||||
const timeout = Math.min(providerConfig.timeoutMs || 10000, Math.max(remaining, 1000));
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeout);
|
||||
|
||||
log?.info?.("SEARCH", `${provider.id} | "${params.query.slice(0, 80)}" | type=${params.searchType}`);
|
||||
|
||||
try {
|
||||
const resp = await fetch(url, { ...init, headers: sanitizeHeaders(init.headers), signal: controller.signal });
|
||||
clearTimeout(timer);
|
||||
if (!resp.ok) {
|
||||
const errText = await resp.text().catch(() => "");
|
||||
log?.error?.("SEARCH", `${provider.id} ${resp.status}: ${errText.slice(0, 200)}`);
|
||||
return { success: false, status: resp.status, error: `${provider.id} returned ${resp.status}: ${errText.slice(0, 200)}` };
|
||||
}
|
||||
const data = await resp.json();
|
||||
const normalized = normalizeSearchResponse(provider.id, data, params.query, params.searchType);
|
||||
const results = normalized.results.slice(0, params.maxResults);
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data: {
|
||||
provider: provider.id,
|
||||
query: params.query,
|
||||
results,
|
||||
answer: null,
|
||||
usage: { queries_used: 1, search_cost_usd: providerConfig.costPerQuery || 0 },
|
||||
metrics: { response_time_ms: duration, upstream_latency_ms: duration, total_results_available: normalized.totalResults },
|
||||
errors: []
|
||||
}
|
||||
};
|
||||
} catch (err) {
|
||||
clearTimeout(timer);
|
||||
const isTimeout = err.name === "AbortError";
|
||||
const status = isTimeout ? 504 : 502;
|
||||
log?.error?.("SEARCH", `${provider.id} ${isTimeout ? "timeout" : "error"}: ${err.message}`);
|
||||
return { success: false, status, error: `${provider.id} ${isTimeout ? "timeout" : "error"}: ${err.message}` };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Core search handler. Dispatches to dedicated API or chat-based LLM.
|
||||
* Same calling convention as handleEmbeddingsCore: returns `{success, response, status?, error?}`.
|
||||
*
|
||||
* @param {object} options
|
||||
* @param {object} options.body Sanitized body from auth wrapper
|
||||
* @param {object} options.provider Provider entry from AI_PROVIDERS
|
||||
* @param {object} [options.providerConfig] Provider's searchConfig (if dedicated)
|
||||
* @param {object|null} options.credentials Provider credentials
|
||||
* @param {object} [options.log] Logger
|
||||
*/
|
||||
export async function handleSearchCore({ body, provider, providerConfig, credentials, log }) {
|
||||
const globalStartTime = Date.now();
|
||||
|
||||
// 1. Sanitize query
|
||||
const { clean, error: sanitizeError } = sanitizeQuery(body.query || "");
|
||||
if (sanitizeError) return errorResult(400, sanitizeError);
|
||||
const normalizedBody = { ...body, query: clean };
|
||||
|
||||
// 2. Route: dedicated search API takes priority over chat-based
|
||||
let result;
|
||||
if (providerConfig) {
|
||||
result = await tryDedicatedProvider({
|
||||
provider,
|
||||
providerConfig,
|
||||
body: normalizedBody,
|
||||
credentials,
|
||||
log,
|
||||
globalStartTime
|
||||
});
|
||||
} else if (provider.searchViaChat) {
|
||||
result = await handleChatSearch({
|
||||
provider: provider.id,
|
||||
query: clean,
|
||||
maxResults: normalizedBody.max_results,
|
||||
model: provider.searchViaChat.defaultModel,
|
||||
credentials,
|
||||
log
|
||||
});
|
||||
} else {
|
||||
return errorResult(400, `Provider ${provider.id} does not support web search`);
|
||||
}
|
||||
|
||||
if (result.success) return successResult(result.data);
|
||||
|
||||
// 3. Failover within global timeout for retriable errors
|
||||
if (
|
||||
!NON_RETRIABLE.has(result.status || 0) &&
|
||||
Date.now() - globalStartTime < GLOBAL_TIMEOUT_MS &&
|
||||
provider.searchViaChat &&
|
||||
providerConfig
|
||||
) {
|
||||
log?.warn?.("SEARCH", `${provider.id} dedicated failed (${result.status}), falling back to chat-based search`);
|
||||
const fallback = await handleChatSearch({
|
||||
provider: provider.id,
|
||||
query: clean,
|
||||
maxResults: normalizedBody.max_results,
|
||||
model: provider.searchViaChat.defaultModel,
|
||||
credentials,
|
||||
log
|
||||
});
|
||||
if (fallback.success) return successResult(fallback.data);
|
||||
}
|
||||
|
||||
return errorResult(result.status || 502, result.error || "Search failed");
|
||||
}
|
||||
223
open-sse/handlers/search/normalizers.js
Normal file
223
open-sse/handlers/search/normalizers.js
Normal file
@@ -0,0 +1,223 @@
|
||||
/**
|
||||
* Search Response Normalizers
|
||||
*
|
||||
* Ported from OmniRoute open-sse/handlers/search.ts.
|
||||
* Each normalizer maps a provider-specific response into the unified SearchResult shape.
|
||||
*/
|
||||
|
||||
/** Build a unified SearchResult object. */
|
||||
function makeResult(providerId, item, idx, now) {
|
||||
const url = item.url || "";
|
||||
return {
|
||||
title: item.title || "",
|
||||
url,
|
||||
display_url: url ? url.replace(/^https?:\/\/(www\.)?/, "").split("?")[0] : undefined,
|
||||
snippet: item.snippet || "",
|
||||
position: idx + 1,
|
||||
score: typeof item.score === "number" ? Math.min(1, Math.max(0, item.score)) : null,
|
||||
published_at: item.published_at || null,
|
||||
favicon_url: item.favicon_url || null,
|
||||
content: item.full_text
|
||||
? { format: item.text_format || "text", text: item.full_text, length: item.full_text.length }
|
||||
: null,
|
||||
metadata: {
|
||||
author: item.author || null,
|
||||
language: null,
|
||||
source_type: item.source_type || null,
|
||||
image_url: item.image_url || null,
|
||||
},
|
||||
citation: { provider: providerId, retrieved_at: now, rank: idx + 1 },
|
||||
provider_raw: null,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeSerper(data, _query, searchType) {
|
||||
const now = new Date().toISOString();
|
||||
const items = searchType === "news" ? data.news : data.organic;
|
||||
if (!Array.isArray(items)) return { results: [], totalResults: null };
|
||||
const results = items.map((item, idx) =>
|
||||
makeResult("serper", { title: item.title, url: item.link, snippet: item.snippet || item.description, published_at: item.date }, idx, now)
|
||||
);
|
||||
const total = data.searchParameters?.totalResults;
|
||||
return { results, totalResults: typeof total === "number" ? total : null };
|
||||
}
|
||||
|
||||
function normalizeBrave(data, _query, searchType) {
|
||||
const now = new Date().toISOString();
|
||||
const container = searchType === "news" ? data.news || data : data.web;
|
||||
const items = container?.results;
|
||||
if (!Array.isArray(items)) return { results: [], totalResults: null };
|
||||
const results = items.map((item, idx) =>
|
||||
makeResult("brave-search", {
|
||||
title: item.title,
|
||||
url: item.url,
|
||||
snippet: item.description,
|
||||
published_at: item.page_age || item.age,
|
||||
favicon_url: item.meta_url?.favicon || item.favicon,
|
||||
}, idx, now)
|
||||
);
|
||||
return { results, totalResults: container?.totalCount ?? null };
|
||||
}
|
||||
|
||||
function normalizePerplexity(data, _query, _searchType) {
|
||||
const now = new Date().toISOString();
|
||||
const items = data.results;
|
||||
if (!Array.isArray(items)) return { results: [], totalResults: null };
|
||||
const results = items.map((item, idx) =>
|
||||
makeResult("perplexity", { title: item.title, url: item.url, snippet: item.snippet, published_at: item.date || item.last_updated }, idx, now)
|
||||
);
|
||||
return { results, totalResults: results.length };
|
||||
}
|
||||
|
||||
function normalizeExa(data, _query, _searchType) {
|
||||
const now = new Date().toISOString();
|
||||
const items = data.results;
|
||||
if (!Array.isArray(items)) return { results: [], totalResults: null };
|
||||
const results = items.map((item, idx) =>
|
||||
makeResult("exa", {
|
||||
title: item.title,
|
||||
url: item.url,
|
||||
snippet: item.highlights?.[0] || item.text?.slice(0, 300) || "",
|
||||
score: item.score,
|
||||
published_at: item.publishedDate,
|
||||
favicon_url: item.favicon,
|
||||
author: item.author,
|
||||
image_url: item.image,
|
||||
full_text: item.text,
|
||||
text_format: "text",
|
||||
}, idx, now)
|
||||
);
|
||||
return { results, totalResults: results.length };
|
||||
}
|
||||
|
||||
function normalizeTavily(data, _query, _searchType) {
|
||||
const now = new Date().toISOString();
|
||||
const items = data.results;
|
||||
if (!Array.isArray(items)) return { results: [], totalResults: null };
|
||||
const results = items.map((item, idx) =>
|
||||
makeResult("tavily", {
|
||||
title: item.title,
|
||||
url: item.url,
|
||||
snippet: item.content || "",
|
||||
score: item.score,
|
||||
published_at: item.published_date,
|
||||
full_text: item.raw_content,
|
||||
text_format: "text",
|
||||
}, idx, now)
|
||||
);
|
||||
return { results, totalResults: results.length };
|
||||
}
|
||||
|
||||
function normalizeGooglePse(data, _query, _searchType) {
|
||||
const now = new Date().toISOString();
|
||||
const items = Array.isArray(data.items) ? data.items : [];
|
||||
const results = items.map((item, idx) =>
|
||||
makeResult("google-pse", {
|
||||
title: item.title,
|
||||
url: item.link,
|
||||
snippet: item.snippet,
|
||||
image_url: item.pagemap?.cse_image?.[0]?.src || item.pagemap?.cse_thumbnail?.[0]?.src || item.pagemap?.metatags?.[0]?.["og:image"],
|
||||
}, idx, now)
|
||||
);
|
||||
const raw = data.searchInformation?.totalResults ?? data.queries?.request?.[0]?.totalResults ?? null;
|
||||
const total = typeof raw === "string" ? Number(raw) : raw;
|
||||
return { results, totalResults: Number.isFinite(total) ? total : null };
|
||||
}
|
||||
|
||||
function normalizeLinkup(data, _query, _searchType) {
|
||||
const now = new Date().toISOString();
|
||||
const items = Array.isArray(data.results) ? data.results : [];
|
||||
const results = items.map((item, idx) =>
|
||||
makeResult("linkup", {
|
||||
title: item.name || item.title,
|
||||
url: item.url,
|
||||
snippet: item.content || item.snippet || "",
|
||||
source_type: item.type || "web",
|
||||
image_url: item.image_url || item.imageUrl || null,
|
||||
full_text: item.content,
|
||||
text_format: "text",
|
||||
}, idx, now)
|
||||
);
|
||||
return { results, totalResults: results.length };
|
||||
}
|
||||
|
||||
function normalizeSearchApi(data, _query, _searchType) {
|
||||
const now = new Date().toISOString();
|
||||
const items = Array.isArray(data.organic_results) ? data.organic_results : Array.isArray(data.top_stories) ? data.top_stories : [];
|
||||
const results = items.map((item, idx) =>
|
||||
makeResult("searchapi", {
|
||||
title: item.title,
|
||||
url: item.link,
|
||||
snippet: item.snippet || item.description || "",
|
||||
published_at: item.date || item.published_at,
|
||||
favicon_url: item.favicon,
|
||||
author: item.source || null,
|
||||
image_url: item.thumbnail || null,
|
||||
}, idx, now)
|
||||
);
|
||||
const raw = data.search_information?.total_results;
|
||||
const total = typeof raw === "number" ? raw : typeof raw === "string" ? Number(raw) : null;
|
||||
return { results, totalResults: Number.isFinite(total) ? total : results.length };
|
||||
}
|
||||
|
||||
function normalizeYouCom(data, _query, searchType) {
|
||||
const now = new Date().toISOString();
|
||||
const container = data?.results && typeof data.results === "object" ? data.results : undefined;
|
||||
const section = searchType === "news" ? container?.news || [] : container?.web || [];
|
||||
const items = Array.isArray(section) ? section : [];
|
||||
const results = items.map((item, idx) => {
|
||||
const firstSnippet = Array.isArray(item.snippets) ? item.snippets.find((v) => typeof v === "string") : null;
|
||||
const livecrawlText = typeof item.markdown === "string" ? item.markdown : typeof item.html === "string" ? item.html : undefined;
|
||||
const livecrawlFormat = typeof item.markdown === "string" ? "markdown" : "html";
|
||||
return makeResult("youcom", {
|
||||
title: item.title,
|
||||
url: item.url,
|
||||
snippet: typeof firstSnippet === "string" ? firstSnippet : typeof item.description === "string" ? item.description : "",
|
||||
published_at: item.page_age,
|
||||
favicon_url: item.favicon_url,
|
||||
image_url: item.thumbnail_url,
|
||||
source_type: searchType,
|
||||
full_text: livecrawlText,
|
||||
text_format: livecrawlText ? livecrawlFormat : undefined,
|
||||
}, idx, now);
|
||||
});
|
||||
return { results, totalResults: results.length };
|
||||
}
|
||||
|
||||
function normalizeSearxng(data, _query, _searchType) {
|
||||
const now = new Date().toISOString();
|
||||
const items = Array.isArray(data.results) ? data.results : [];
|
||||
const results = items.map((item, idx) =>
|
||||
makeResult("searxng", {
|
||||
title: item.title,
|
||||
url: item.url,
|
||||
snippet: item.content || item.snippet || "",
|
||||
published_at: item.publishedDate || item.published_date || null,
|
||||
source_type: Array.isArray(item.engines) ? item.engines.join(", ") : item.engine || item.category || null,
|
||||
image_url: item.thumbnail || item.img_src || null,
|
||||
}, idx, now)
|
||||
);
|
||||
return { results, totalResults: results.length };
|
||||
}
|
||||
|
||||
const NORMALIZERS = {
|
||||
"serper": normalizeSerper,
|
||||
"brave-search": normalizeBrave,
|
||||
"perplexity": normalizePerplexity,
|
||||
"exa": normalizeExa,
|
||||
"tavily": normalizeTavily,
|
||||
"google-pse": normalizeGooglePse,
|
||||
"linkup": normalizeLinkup,
|
||||
"searchapi": normalizeSearchApi,
|
||||
"youcom": normalizeYouCom,
|
||||
"searxng": normalizeSearxng,
|
||||
};
|
||||
|
||||
/**
|
||||
* Dispatch to the appropriate normalizer based on providerId.
|
||||
* @returns {{results: Array, totalResults: number|null}}
|
||||
*/
|
||||
export function normalizeSearchResponse(providerId, data, query, searchType) {
|
||||
const fn = NORMALIZERS[providerId];
|
||||
return fn ? fn(data, query, searchType) : { results: [], totalResults: null };
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import { PROVIDERS } from "../config/providers.js";
|
||||
import { OAUTH_ENDPOINTS, GITHUB_COPILOT, REFRESH_LEAD_MS } from "../config/appConstants.js";
|
||||
import { proxyAwareFetch } from "../utils/proxyFetch.js";
|
||||
|
||||
// Default token expiry buffer (refresh if expires within 5 minutes)
|
||||
export const TOKEN_EXPIRY_BUFFER_MS = 5 * 60 * 1000;
|
||||
@@ -242,7 +243,7 @@ export async function refreshCodexToken(refreshToken, log) {
|
||||
* Specialized refresh for Kiro (AWS CodeWhisperer) tokens
|
||||
* Supports both AWS SSO OIDC (Builder ID/IDC) and Social Auth (Google/GitHub)
|
||||
*/
|
||||
export async function refreshKiroToken(refreshToken, providerSpecificData, log) {
|
||||
export async function refreshKiroToken(refreshToken, providerSpecificData, log, proxyOptions = null) {
|
||||
const authMethod = providerSpecificData?.authMethod;
|
||||
const clientId = providerSpecificData?.clientId;
|
||||
const clientSecret = providerSpecificData?.clientSecret;
|
||||
@@ -256,7 +257,7 @@ export async function refreshKiroToken(refreshToken, providerSpecificData, log)
|
||||
? `https://oidc.${region}.amazonaws.com/token`
|
||||
: "https://oidc.us-east-1.amazonaws.com/token";
|
||||
|
||||
const response = await fetch(endpoint, {
|
||||
const response = await proxyAwareFetch(endpoint, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
@@ -268,7 +269,7 @@ export async function refreshKiroToken(refreshToken, providerSpecificData, log)
|
||||
refreshToken: refreshToken,
|
||||
grantType: "refresh_token",
|
||||
}),
|
||||
});
|
||||
}, proxyOptions);
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
@@ -294,7 +295,7 @@ export async function refreshKiroToken(refreshToken, providerSpecificData, log)
|
||||
}
|
||||
|
||||
// Social Auth (Google/GitHub) - use Kiro's refresh endpoint
|
||||
const response = await fetch(PROVIDERS.kiro.tokenUrl, {
|
||||
const response = await proxyAwareFetch(PROVIDERS.kiro.tokenUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
@@ -304,7 +305,7 @@ export async function refreshKiroToken(refreshToken, providerSpecificData, log)
|
||||
body: JSON.stringify({
|
||||
refreshToken: refreshToken,
|
||||
}),
|
||||
});
|
||||
}, proxyOptions);
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
*/
|
||||
|
||||
import { CLIENT_METADATA, getPlatformUserAgent } from "../config/appConstants.js";
|
||||
import { proxyAwareFetch } from "../utils/proxyFetch.js";
|
||||
|
||||
// GitHub API config
|
||||
const GITHUB_CONFIG = {
|
||||
@@ -38,22 +39,22 @@ const CLAUDE_CONFIG = {
|
||||
* @param {Object} connection - Provider connection with accessToken
|
||||
* @returns {Object} Usage data with quotas
|
||||
*/
|
||||
export async function getUsageForProvider(connection) {
|
||||
export async function getUsageForProvider(connection, proxyOptions = null) {
|
||||
const { provider, accessToken, providerSpecificData } = connection;
|
||||
|
||||
switch (provider) {
|
||||
case "github":
|
||||
return await getGitHubUsage(accessToken, providerSpecificData);
|
||||
return await getGitHubUsage(accessToken, providerSpecificData, proxyOptions);
|
||||
case "gemini-cli":
|
||||
return await getGeminiUsage(accessToken);
|
||||
return await getGeminiUsage(accessToken, proxyOptions);
|
||||
case "antigravity":
|
||||
return await getAntigravityUsage(accessToken);
|
||||
return await getAntigravityUsage(accessToken, providerSpecificData, proxyOptions);
|
||||
case "claude":
|
||||
return await getClaudeUsage(accessToken);
|
||||
return await getClaudeUsage(accessToken, proxyOptions);
|
||||
case "codex":
|
||||
return await getCodexUsage(accessToken);
|
||||
return await getCodexUsage(accessToken, proxyOptions);
|
||||
case "kiro":
|
||||
return await getKiroUsage(accessToken, providerSpecificData);
|
||||
return await getKiroUsage(accessToken, providerSpecificData, proxyOptions);
|
||||
case "qwen":
|
||||
return await getQwenUsage(accessToken, providerSpecificData);
|
||||
case "iflow":
|
||||
@@ -103,14 +104,14 @@ function parseResetTime(resetValue) {
|
||||
* GitHub Copilot Usage
|
||||
* Uses GitHub accessToken (not copilotToken) to call copilot_internal/user API
|
||||
*/
|
||||
async function getGitHubUsage(accessToken, providerSpecificData) {
|
||||
async function getGitHubUsage(accessToken, providerSpecificData, proxyOptions = null) {
|
||||
try {
|
||||
if (!accessToken) {
|
||||
throw new Error("No GitHub access token available. Please re-authorize the connection.");
|
||||
}
|
||||
|
||||
// copilot_internal/user API requires GitHub OAuth token, not copilotToken
|
||||
const response = await fetch("https://api.github.com/copilot_internal/user", {
|
||||
const response = await proxyAwareFetch("https://api.github.com/copilot_internal/user", {
|
||||
headers: {
|
||||
"Authorization": `token ${accessToken}`,
|
||||
"Accept": "application/json",
|
||||
@@ -119,7 +120,7 @@ async function getGitHubUsage(accessToken, providerSpecificData) {
|
||||
"Editor-Version": "vscode/1.100.0",
|
||||
"Editor-Plugin-Version": "copilot-chat/0.26.7",
|
||||
},
|
||||
});
|
||||
}, proxyOptions);
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.text();
|
||||
@@ -189,18 +190,19 @@ function formatGitHubQuotaSnapshot(quota) {
|
||||
/**
|
||||
* Gemini CLI Usage (Google Cloud)
|
||||
*/
|
||||
async function getGeminiUsage(accessToken) {
|
||||
async function getGeminiUsage(accessToken, proxyOptions = null) {
|
||||
try {
|
||||
// Gemini CLI uses Google Cloud quotas
|
||||
// Try to get quota info from Cloud Resource Manager
|
||||
const response = await fetch(
|
||||
const response = await proxyAwareFetch(
|
||||
"https://cloudresourcemanager.googleapis.com/v1/projects?filter=lifecycleState:ACTIVE",
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: "application/json",
|
||||
},
|
||||
}
|
||||
},
|
||||
proxyOptions
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
@@ -217,10 +219,10 @@ async function getGeminiUsage(accessToken) {
|
||||
/**
|
||||
* Antigravity Usage - Fetch quota from Google Cloud Code API
|
||||
*/
|
||||
async function getAntigravityUsage(accessToken, providerSpecificData) {
|
||||
async function getAntigravityUsage(accessToken, providerSpecificData, proxyOptions = null) {
|
||||
try {
|
||||
// Fetch subscription info once — reuse for both projectId and plan
|
||||
const subscriptionInfo = await getAntigravitySubscriptionInfo(accessToken);
|
||||
const subscriptionInfo = await getAntigravitySubscriptionInfo(accessToken, proxyOptions);
|
||||
const projectId = subscriptionInfo?.cloudaicompanionProject || null;
|
||||
|
||||
// Fetch quota data with timeout
|
||||
@@ -229,7 +231,7 @@ async function getAntigravityUsage(accessToken, providerSpecificData) {
|
||||
|
||||
let response;
|
||||
try {
|
||||
response = await fetch(ANTIGRAVITY_CONFIG.quotaApiUrl, {
|
||||
response = await proxyAwareFetch(ANTIGRAVITY_CONFIG.quotaApiUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Authorization": `Bearer ${accessToken}`,
|
||||
@@ -243,7 +245,7 @@ async function getAntigravityUsage(accessToken, providerSpecificData) {
|
||||
...(projectId ? { project: projectId } : {})
|
||||
}),
|
||||
signal: controller.signal,
|
||||
});
|
||||
}, proxyOptions);
|
||||
} finally {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
@@ -338,11 +340,11 @@ async function getAntigravityProjectId(accessToken) {
|
||||
/**
|
||||
* Get Antigravity subscription info
|
||||
*/
|
||||
async function getAntigravitySubscriptionInfo(accessToken) {
|
||||
async function getAntigravitySubscriptionInfo(accessToken, proxyOptions = null) {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), 10000); // 10s timeout
|
||||
try {
|
||||
const response = await fetch(ANTIGRAVITY_CONFIG.loadProjectApiUrl, {
|
||||
const response = await proxyAwareFetch(ANTIGRAVITY_CONFIG.loadProjectApiUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Authorization": `Bearer ${accessToken}`,
|
||||
@@ -352,7 +354,7 @@ async function getAntigravitySubscriptionInfo(accessToken) {
|
||||
},
|
||||
body: JSON.stringify({ metadata: CLIENT_METADATA, mode: 1 }),
|
||||
signal: controller.signal,
|
||||
});
|
||||
}, proxyOptions);
|
||||
|
||||
if (!response.ok) return null;
|
||||
return await response.json();
|
||||
@@ -367,17 +369,17 @@ async function getAntigravitySubscriptionInfo(accessToken) {
|
||||
/**
|
||||
* Claude Usage - Primary: OAuth endpoint, Fallback: legacy settings/org endpoint
|
||||
*/
|
||||
async function getClaudeUsage(accessToken) {
|
||||
async function getClaudeUsage(accessToken, proxyOptions = null) {
|
||||
try {
|
||||
// Primary: OAuth usage endpoint (Claude Code consumer OAuth tokens)
|
||||
const oauthResponse = await fetch(CLAUDE_CONFIG.oauthUsageUrl, {
|
||||
const oauthResponse = await proxyAwareFetch(CLAUDE_CONFIG.oauthUsageUrl, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
"Authorization": `Bearer ${accessToken}`,
|
||||
"anthropic-beta": "oauth-2025-04-20",
|
||||
"anthropic-version": CLAUDE_CONFIG.apiVersion,
|
||||
},
|
||||
});
|
||||
}, proxyOptions);
|
||||
|
||||
if (oauthResponse.ok) {
|
||||
const data = await oauthResponse.json();
|
||||
@@ -425,7 +427,7 @@ async function getClaudeUsage(accessToken) {
|
||||
|
||||
// Fallback: legacy settings + org usage endpoint
|
||||
console.warn(`[Claude Usage] OAuth endpoint returned ${oauthResponse.status}, falling back to legacy`);
|
||||
return await getClaudeUsageLegacy(accessToken);
|
||||
return await getClaudeUsageLegacy(accessToken, proxyOptions);
|
||||
} catch (error) {
|
||||
return { message: `Claude connected. Unable to fetch usage: ${error.message}` };
|
||||
}
|
||||
@@ -434,21 +436,21 @@ async function getClaudeUsage(accessToken) {
|
||||
/**
|
||||
* Legacy Claude usage for API key / org admin users
|
||||
*/
|
||||
async function getClaudeUsageLegacy(accessToken) {
|
||||
async function getClaudeUsageLegacy(accessToken, proxyOptions = null) {
|
||||
try {
|
||||
const settingsResponse = await fetch(CLAUDE_CONFIG.settingsUrl, {
|
||||
const settingsResponse = await proxyAwareFetch(CLAUDE_CONFIG.settingsUrl, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
"Authorization": `Bearer ${accessToken}`,
|
||||
"anthropic-version": CLAUDE_CONFIG.apiVersion,
|
||||
},
|
||||
});
|
||||
}, proxyOptions);
|
||||
|
||||
if (settingsResponse.ok) {
|
||||
const settings = await settingsResponse.json();
|
||||
|
||||
if (settings.organization_id) {
|
||||
const usageResponse = await fetch(
|
||||
const usageResponse = await proxyAwareFetch(
|
||||
CLAUDE_CONFIG.usageUrl.replace("{org_id}", settings.organization_id),
|
||||
{
|
||||
method: "GET",
|
||||
@@ -456,7 +458,8 @@ async function getClaudeUsageLegacy(accessToken) {
|
||||
"Authorization": `Bearer ${accessToken}`,
|
||||
"anthropic-version": CLAUDE_CONFIG.apiVersion,
|
||||
},
|
||||
}
|
||||
},
|
||||
proxyOptions
|
||||
);
|
||||
|
||||
if (usageResponse.ok) {
|
||||
@@ -485,15 +488,15 @@ async function getClaudeUsageLegacy(accessToken) {
|
||||
/**
|
||||
* Codex (OpenAI) Usage - Fetch from ChatGPT backend API
|
||||
*/
|
||||
async function getCodexUsage(accessToken) {
|
||||
async function getCodexUsage(accessToken, proxyOptions = null) {
|
||||
try {
|
||||
const response = await fetch(CODEX_CONFIG.usageUrl, {
|
||||
const response = await proxyAwareFetch(CODEX_CONFIG.usageUrl, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
"Authorization": `Bearer ${accessToken}`,
|
||||
"Accept": "application/json",
|
||||
},
|
||||
});
|
||||
}, proxyOptions);
|
||||
|
||||
if (!response.ok) {
|
||||
return { message: `Codex connected. Usage API temporarily unavailable (${response.status}).` };
|
||||
@@ -577,7 +580,7 @@ function parseKiroQuotaData(data) {
|
||||
};
|
||||
}
|
||||
|
||||
async function getKiroUsage(accessToken, providerSpecificData) {
|
||||
async function getKiroUsage(accessToken, providerSpecificData, proxyOptions = null) {
|
||||
// Default profileArn fallback
|
||||
const DEFAULT_PROFILE_ARN = "arn:aws:codewhisperer:us-east-1:638616132270:profile/AAAACCCCXXXX";
|
||||
const profileArn = providerSpecificData?.profileArn || DEFAULT_PROFILE_ARN;
|
||||
@@ -593,7 +596,7 @@ async function getKiroUsage(accessToken, providerSpecificData) {
|
||||
const attempts = [
|
||||
{
|
||||
name: "codewhisperer-get",
|
||||
run: async () => fetch(
|
||||
run: async () => proxyAwareFetch(
|
||||
`https://codewhisperer.us-east-1.amazonaws.com/getUsageLimits?${getUsageParams.toString()}`,
|
||||
{
|
||||
method: "GET",
|
||||
@@ -604,11 +607,12 @@ async function getKiroUsage(accessToken, providerSpecificData) {
|
||||
"user-agent": "aws-sdk-js/1.0.0 KiroIDE",
|
||||
},
|
||||
},
|
||||
proxyOptions
|
||||
),
|
||||
},
|
||||
{
|
||||
name: "codewhisperer-post",
|
||||
run: async () => fetch("https://codewhisperer.us-east-1.amazonaws.com", {
|
||||
run: async () => proxyAwareFetch("https://codewhisperer.us-east-1.amazonaws.com", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Authorization": `Bearer ${accessToken}`,
|
||||
@@ -621,7 +625,7 @@ async function getKiroUsage(accessToken, providerSpecificData) {
|
||||
profileArn,
|
||||
resourceType: "AGENTIC_REQUEST",
|
||||
}),
|
||||
}),
|
||||
}, proxyOptions),
|
||||
},
|
||||
{
|
||||
name: "q-get",
|
||||
@@ -631,13 +635,13 @@ async function getKiroUsage(accessToken, providerSpecificData) {
|
||||
profileArn,
|
||||
resourceType: "AGENTIC_REQUEST",
|
||||
});
|
||||
return fetch(`https://q.us-east-1.amazonaws.com/getUsageLimits?${params}`, {
|
||||
return proxyAwareFetch(`https://q.us-east-1.amazonaws.com/getUsageLimits?${params}`, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
"Authorization": `Bearer ${accessToken}`,
|
||||
"Accept": "application/json",
|
||||
},
|
||||
});
|
||||
}, proxyOptions);
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
Reference in New Issue
Block a user