Add optional modelID input for custom API Key Providers testing (#315)

* feat: add modelId fallback for provider validation

- If /models endpoint unavailable, validate via /chat/completions
- Add optional Model ID input in EditCompatibleNodeModal
- Improves compatibility with providers lacking /models endpoint

* feat: improve provider validation with modelId fallback

- Add Model ID input for chat/completions fallback validation
- Reorder UI: API Key → Model ID → Check button + Badge
- Display detailed BE error messages in FE
- Add status-specific error handling (401/403/400/404/5xx)
- Add unit tests for error message helpers
- Add vitest devDependency
This commit is contained in:
moophat
2026-03-16 09:21:05 +07:00
committed by GitHub
parent 1dd5d60724
commit 65af4328fd
4 changed files with 470 additions and 57 deletions

View File

@@ -1987,6 +1987,7 @@ function EditCompatibleNodeModal({ isOpen, node, onSave, onClose, isAnthropic })
});
const [saving, setSaving] = useState(false);
const [checkKey, setCheckKey] = useState("");
const [checkModelId, setCheckModelId] = useState("");
const [validating, setValidating] = useState(false);
const [validationResult, setValidationResult] = useState(null);
@@ -2030,10 +2031,11 @@ function EditCompatibleNodeModal({ isOpen, node, onSave, onClose, isAnthropic })
const res = await fetch("/api/provider-nodes/validate", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
baseUrl: formData.baseUrl,
apiKey: checkKey,
type: isAnthropic ? "anthropic-compatible" : "openai-compatible"
body: JSON.stringify({
baseUrl: formData.baseUrl,
apiKey: checkKey,
type: isAnthropic ? "anthropic-compatible" : "openai-compatible",
modelId: checkModelId.trim() || undefined
}),
});
const data = await res.json();
@@ -2093,6 +2095,13 @@ function EditCompatibleNodeModal({ isOpen, node, onSave, onClose, isAnthropic })
</Button>
</div>
</div>
<Input
label="Model ID (optional)"
value={checkModelId}
onChange={(e) => setCheckModelId(e.target.value)}
placeholder="e.g. my-model-id"
hint="If provider lacks /models endpoint, enter a model ID to validate via chat/completions instead."
/>
{validationResult && (
<Badge variant={validationResult === "success" ? "success" : "error"}>
{validationResult === "success" ? "Valid" : "Invalid"}

View File

@@ -657,6 +657,7 @@ function AddOpenAICompatibleModal({ isOpen, onClose, onCreated }) {
});
const [submitting, setSubmitting] = useState(false);
const [checkKey, setCheckKey] = useState("");
const [checkModelId, setCheckModelId] = useState("");
const [validating, setValidating] = useState(false);
const [validationResult, setValidationResult] = useState(null);
@@ -705,17 +706,43 @@ function AddOpenAICompatibleModal({ isOpen, onClose, onCreated }) {
const res = await fetch("/api/provider-nodes/validate", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ baseUrl: formData.baseUrl, apiKey: checkKey, type: "openai-compatible" }),
body: JSON.stringify({
baseUrl: formData.baseUrl,
apiKey: checkKey,
type: "openai-compatible",
modelId: checkModelId.trim() || undefined
}),
});
const data = await res.json();
setValidationResult(data.valid ? "success" : "failed");
setValidationResult(data);
} catch {
setValidationResult("failed");
setValidationResult({ valid: false, error: "Network error" });
} finally {
setValidating(false);
}
};
// Helper to render validation result
const renderValidationResult = () => {
if (!validationResult) return null;
const { valid, error, method } = validationResult;
if (valid) {
return (
<>
<Badge variant="success">Valid</Badge>
{method === "chat" && <span className="text-sm text-text-muted">(via inference test)</span>}
</>
);
}
return (
<div className="flex flex-col gap-1">
<Badge variant="error">Invalid</Badge>
{error && <span className="text-sm text-red-500">{error}</span>}
</div>
);
};
return (
<Modal isOpen={isOpen} title="Add OpenAI Compatible" onClose={onClose}>
<div className="flex flex-col gap-4">
@@ -746,25 +773,25 @@ function AddOpenAICompatibleModal({ isOpen, onClose, onCreated }) {
placeholder="https://api.openai.com/v1"
hint="Use the base URL (ending in /v1) for your OpenAI-compatible API."
/>
<div className="flex gap-2">
<Input
label="API Key (for Check)"
type="password"
value={checkKey}
onChange={(e) => setCheckKey(e.target.value)}
className="flex-1"
/>
<div className="pt-6">
<Button onClick={handleValidate} disabled={!checkKey || validating || !formData.baseUrl.trim()} variant="secondary">
{validating ? "Checking..." : "Check"}
</Button>
</div>
<Input
label="API Key (for Check)"
type="password"
value={checkKey}
onChange={(e) => setCheckKey(e.target.value)}
/>
<Input
label="Model ID (optional)"
value={checkModelId}
onChange={(e) => setCheckModelId(e.target.value)}
placeholder="e.g. gpt-4, claude-3-opus"
hint="If provider lacks /models endpoint, enter a model ID to validate via chat/completions instead."
/>
<div className="flex items-center gap-3">
<Button onClick={handleValidate} disabled={!checkKey || validating || !formData.baseUrl.trim()} variant="secondary">
{validating ? "Checking..." : "Check"}
</Button>
{renderValidationResult()}
</div>
{validationResult && (
<Badge variant={validationResult === "success" ? "success" : "error"}>
{validationResult === "success" ? "Valid" : "Invalid"}
</Badge>
)}
<div className="flex gap-2">
<Button onClick={handleSubmit} fullWidth disabled={!formData.name.trim() || !formData.prefix.trim() || !formData.baseUrl.trim() || submitting}>
{submitting ? "Creating..." : "Create"}
@@ -790,13 +817,15 @@ function AddAnthropicCompatibleModal({ isOpen, onClose, onCreated }) {
});
const [submitting, setSubmitting] = useState(false);
const [checkKey, setCheckKey] = useState("");
const [checkModelId, setCheckModelId] = useState("");
const [validating, setValidating] = useState(false);
const [validationResult, setValidationResult] = useState(null);
const [validationResult, setValidationResult] = useState(null); // { valid, error, method }
useEffect(() => {
if (isOpen) {
setValidationResult(null);
setCheckKey("");
setCheckModelId("");
}
}, [isOpen]);
@@ -834,17 +863,43 @@ function AddAnthropicCompatibleModal({ isOpen, onClose, onCreated }) {
const res = await fetch("/api/provider-nodes/validate", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ baseUrl: formData.baseUrl, apiKey: checkKey, type: "anthropic-compatible" }),
body: JSON.stringify({
baseUrl: formData.baseUrl,
apiKey: checkKey,
type: "anthropic-compatible",
modelId: checkModelId.trim() || undefined
}),
});
const data = await res.json();
setValidationResult(data.valid ? "success" : "failed");
setValidationResult(data);
} catch {
setValidationResult("failed");
setValidationResult({ valid: false, error: "Network error" });
} finally {
setValidating(false);
}
};
// Helper to render validation result
const renderValidationResult = () => {
if (!validationResult) return null;
const { valid, error, method } = validationResult;
if (valid) {
return (
<>
<Badge variant="success">Valid</Badge>
{method === "chat" && <span className="text-sm text-text-muted">(via inference test)</span>}
</>
);
}
return (
<div className="flex flex-col gap-1">
<Badge variant="error">Invalid</Badge>
{error && <span className="text-sm text-red-500">{error}</span>}
</div>
);
};
return (
<Modal isOpen={isOpen} title="Add Anthropic Compatible" onClose={onClose}>
<div className="flex flex-col gap-4">
@@ -869,25 +924,25 @@ function AddAnthropicCompatibleModal({ isOpen, onClose, onCreated }) {
placeholder="https://api.anthropic.com/v1"
hint="Use the base URL (ending in /v1) for your Anthropic-compatible API. The system will append /messages."
/>
<div className="flex gap-2">
<Input
label="API Key (for Check)"
type="password"
value={checkKey}
onChange={(e) => setCheckKey(e.target.value)}
className="flex-1"
/>
<div className="pt-6">
<Button onClick={handleValidate} disabled={!checkKey || validating || !formData.baseUrl.trim()} variant="secondary">
{validating ? "Checking..." : "Check"}
</Button>
</div>
<Input
label="API Key (for Check)"
type="password"
value={checkKey}
onChange={(e) => setCheckKey(e.target.value)}
/>
<Input
label="Model ID (optional)"
value={checkModelId}
onChange={(e) => setCheckModelId(e.target.value)}
placeholder="e.g. claude-3-opus"
hint="If provider lacks /models endpoint, enter a model ID to validate via chat/completions instead."
/>
<div className="flex items-center gap-3">
<Button onClick={handleValidate} disabled={!checkKey || validating || !formData.baseUrl.trim()} variant="secondary">
{validating ? "Checking..." : "Check"}
</Button>
{renderValidationResult()}
</div>
{validationResult && (
<Badge variant={validationResult === "success" ? "success" : "error"}>
{validationResult === "success" ? "Valid" : "Invalid"}
</Badge>
)}
<div className="flex gap-2">
<Button onClick={handleSubmit} fullWidth disabled={!formData.name.trim() || !formData.prefix.trim() || !formData.baseUrl.trim() || submitting}>
{submitting ? "Creating..." : "Create"}

View File

@@ -32,11 +32,28 @@ const getErrorMessage = (error) => {
return "Network connection failed - check URL and network connectivity";
};
// Get status-specific error message for /models endpoint
const getModelsErrorMessage = (status) => {
if (status === 401 || status === 403) return "API key unauthorized";
if (status === 404) return "/models endpoint not found - try chat validation with model ID";
if (status >= 500) return "Server error - try again later";
return `Unexpected response (${status})`;
};
// Get status-specific error message for /chat/completions endpoint
const getChatErrorMessage = (status) => {
if (status === 401 || status === 403) return "API key unauthorized";
if (status === 400) return "Invalid model or bad request";
if (status === 404) return "Chat endpoint not found";
if (status >= 500) return "Server error - try again later";
return `Chat request failed (${status})`;
};
// POST /api/provider-nodes/validate - Validate API key against base URL
export async function POST(request) {
try {
const body = await request.json();
const { baseUrl, apiKey, type } = body;
const { baseUrl, apiKey, type, modelId } = body;
if (!baseUrl || !apiKey) {
return NextResponse.json({ error: "Base URL and API key required" }, { status: 400 });
@@ -49,25 +66,55 @@ export async function POST(request) {
// Anthropic Compatible Validation
if (type === "anthropic-compatible") {
// Robustly construct URL: remove trailing slash, and remove trailing /messages if user added it
let normalizedBase = baseUrl.trim().replace(/\/$/, "");
if (normalizedBase.endsWith("/messages")) {
normalizedBase = normalizedBase.slice(0, -9); // remove /messages
normalizedBase = normalizedBase.slice(0, -9);
}
// Use /models endpoint for validation as many compatible providers support it (like OpenAI)
const modelsUrl = `${normalizedBase}/models`;
const res = await fetchWithTimeout(modelsUrl, {
method: "GET",
headers: {
headers: {
"x-api-key": apiKey,
"anthropic-version": "2023-06-01",
"Authorization": `Bearer ${apiKey}` // Add Bearer token for hybrid proxies
"Authorization": `Bearer ${apiKey}`
}
});
return NextResponse.json({ valid: res.ok, error: res.ok ? null : "Invalid API key or unauthorized" });
if (res.ok) return NextResponse.json({ valid: true });
// Auth errors - no point trying chat fallback
if (res.status === 401 || res.status === 403) {
return NextResponse.json({ valid: false, error: "API key unauthorized" });
}
// Fallback: try chat/completions if modelId provided
if (modelId) {
const chatRes = await fetchWithTimeout(`${normalizedBase}/chat/completions`, {
method: "POST",
headers: {
"Authorization": `Bearer ${apiKey}`,
"Content-Type": "application/json",
"x-api-key": apiKey,
"anthropic-version": "2023-06-01"
},
body: JSON.stringify({
model: modelId,
messages: [{ role: "user", content: "ping" }],
max_tokens: 1
})
});
if (chatRes.ok) {
return NextResponse.json({ valid: true, method: "chat" });
}
return NextResponse.json({
valid: false,
error: getChatErrorMessage(chatRes.status),
method: "chat"
});
}
return NextResponse.json({ valid: false, error: getModelsErrorMessage(res.status) });
}
// OpenAI Compatible Validation (Default)
@@ -76,7 +123,38 @@ export async function POST(request) {
headers: { "Authorization": `Bearer ${apiKey}` },
});
return NextResponse.json({ valid: res.ok, error: res.ok ? null : "Invalid API key or unauthorized" });
if (res.ok) return NextResponse.json({ valid: true });
// Auth errors - no point trying chat fallback
if (res.status === 401 || res.status === 403) {
return NextResponse.json({ valid: false, error: "API key unauthorized" });
}
// Fallback: try chat/completions if modelId provided
if (modelId) {
const chatRes = await fetchWithTimeout(`${baseUrl.replace(/\/$/, "")}/chat/completions`, {
method: "POST",
headers: {
"Authorization": `Bearer ${apiKey}`,
"Content-Type": "application/json"
},
body: JSON.stringify({
model: modelId,
messages: [{ role: "user", content: "ping" }],
max_tokens: 1
})
});
if (chatRes.ok) {
return NextResponse.json({ valid: true, method: "chat" });
}
return NextResponse.json({
valid: false,
error: getChatErrorMessage(chatRes.status),
method: "chat"
});
}
return NextResponse.json({ valid: false, error: getModelsErrorMessage(res.status) });
} catch (error) {
const errorMessage = getErrorMessage(error);
console.error("Error validating provider node:", {

View File

@@ -0,0 +1,271 @@
/**
* Unit tests for /api/provider-nodes/validate endpoint
*
* Tests cover:
* - OpenAI-compatible validation via /models
* - Anthropic-compatible validation via /models
* - Fallback to /chat/completions when modelId provided
* - Error message handling
*/
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
// Mock fetch globally
const originalFetch = global.fetch;
describe("Provider Validation API", () => {
beforeEach(() => {
vi.clearAllMocks();
});
afterEach(() => {
global.fetch = originalFetch;
});
describe("OpenAI Compatible", () => {
it("should return valid:true when /models succeeds", async () => {
global.fetch = vi.fn().mockResolvedValue({
ok: true,
json: () => Promise.resolve({ data: [] }),
});
// Simulate the validation logic
const baseUrl = "https://api.openai.com/v1";
const apiKey = "test-key";
const modelsUrl = `${baseUrl}/models`;
const res = await fetch(modelsUrl, {
headers: { Authorization: `Bearer ${apiKey}` },
});
expect(res.ok).toBe(true);
expect(fetch).toHaveBeenCalledWith(modelsUrl, expect.objectContaining({
headers: { Authorization: `Bearer ${apiKey}` },
}));
});
it("should fallback to chat/completions when /models fails and modelId provided", async () => {
const modelsCall = vi.fn().mockResolvedValue({ ok: false, status: 404 });
const chatCall = vi.fn().mockResolvedValue({ ok: true });
global.fetch = vi.fn().mockImplementation((url) => {
if (url.includes("/models")) return modelsCall();
if (url.includes("/chat/completions")) return chatCall();
return Promise.reject(new Error("Unknown URL"));
});
// Simulate validation flow
const baseUrl = "https://custom-provider.com/v1";
const modelsRes = await fetch(`${baseUrl}/models`);
expect(modelsRes.ok).toBe(false);
// Fallback to chat
const chatRes = await fetch(`${baseUrl}/chat/completions`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
model: "gpt-4",
messages: [{ role: "user", content: "ping" }],
max_tokens: 1,
}),
});
expect(chatRes.ok).toBe(true);
});
it("should return error when /models fails and no modelId", async () => {
global.fetch = vi.fn().mockResolvedValue({ ok: false, status: 404 });
const baseUrl = "https://custom-provider.com/v1";
const res = await fetch(`${baseUrl}/models`);
expect(res.ok).toBe(false);
// Expected error: "/models unavailable - provide model ID for chat validation"
});
});
describe("Anthropic Compatible", () => {
it("should normalize URL by removing /messages suffix", () => {
const normalizeUrl = (url) => {
let normalized = url.trim().replace(/\/$/, "");
if (normalized.endsWith("/messages")) {
normalized = normalized.slice(0, -9);
}
return normalized;
};
expect(normalizeUrl("https://api.anthropic.com/v1/messages")).toBe("https://api.anthropic.com/v1");
expect(normalizeUrl("https://api.anthropic.com/v1/messages/")).toBe("https://api.anthropic.com/v1");
expect(normalizeUrl("https://api.anthropic.com/v1")).toBe("https://api.anthropic.com/v1");
});
it("should send correct headers for Anthropic API", async () => {
global.fetch = vi.fn().mockResolvedValue({ ok: true });
const baseUrl = "https://api.anthropic.com/v1";
const apiKey = "test-key";
await fetch(`${baseUrl}/models`, {
method: "GET",
headers: {
"x-api-key": apiKey,
"anthropic-version": "2023-06-01",
Authorization: `Bearer ${apiKey}`,
},
});
expect(fetch).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({
headers: expect.objectContaining({
"x-api-key": apiKey,
"anthropic-version": "2023-06-01",
}),
}));
});
});
describe("Error Messages - Network", () => {
it("should map ECONNREFUSED to user-friendly message", () => {
const getErrorMessage = (error) => {
if (error.cause?.code === "ECONNREFUSED") return "Connection refused - provider node offline or unreachable";
return "Unknown error";
};
const error = { cause: { code: "ECONNREFUSED" } };
expect(getErrorMessage(error)).toBe("Connection refused - provider node offline or unreachable");
});
it("should map ENOTFOUND to user-friendly message", () => {
const getErrorMessage = (error) => {
if (error.cause?.code === "ENOTFOUND") return "DNS lookup failed - invalid domain or network issue";
return "Unknown error";
};
const error = { cause: { code: "ENOTFOUND" } };
expect(getErrorMessage(error)).toBe("DNS lookup failed - invalid domain or network issue");
});
it("should map timeout to user-friendly message", () => {
const getErrorMessage = (error) => {
if (error.message.includes("timeout")) return "Request timeout (>10s) - provider node not responding";
return "Unknown error";
};
const error = { message: "Request timeout" };
expect(getErrorMessage(error)).toBe("Request timeout (>10s) - provider node not responding");
});
it("should map CERT_HAS_EXPIRED to user-friendly message", () => {
const getErrorMessage = (error) => {
if (error.cause?.code === "CERT_HAS_EXPIRED") return "SSL certificate expired";
return "Unknown error";
};
const error = { cause: { code: "CERT_HAS_EXPIRED" } };
expect(getErrorMessage(error)).toBe("SSL certificate expired");
});
});
describe("URL Validation", () => {
it("should validate correct URL format", () => {
const isValidUrl = (url) => {
try {
new URL(url);
return true;
} catch {
return false;
}
};
expect(isValidUrl("https://api.openai.com/v1")).toBe(true);
expect(isValidUrl("http://localhost:8080")).toBe(true);
expect(isValidUrl("not-a-url")).toBe(false);
expect(isValidUrl("")).toBe(false);
});
});
describe("Error Messages - /models Status Codes", () => {
const getModelsErrorMessage = (status) => {
if (status === 401 || status === 403) return "API key unauthorized";
if (status === 404) return "/models endpoint not found - try chat validation with model ID";
if (status >= 500) return "Server error - try again later";
return `Unexpected response (${status})`;
};
it("should return auth error for 401", () => {
expect(getModelsErrorMessage(401)).toBe("API key unauthorized");
});
it("should return auth error for 403", () => {
expect(getModelsErrorMessage(403)).toBe("API key unauthorized");
});
it("should return not found for 404", () => {
expect(getModelsErrorMessage(404)).toBe("/models endpoint not found - try chat validation with model ID");
});
it("should return server error for 500", () => {
expect(getModelsErrorMessage(500)).toBe("Server error - try again later");
});
it("should return server error for 502", () => {
expect(getModelsErrorMessage(502)).toBe("Server error - try again later");
});
it("should return unexpected for other codes", () => {
expect(getModelsErrorMessage(418)).toBe("Unexpected response (418)");
});
});
describe("Error Messages - /chat/completions Status Codes", () => {
const getChatErrorMessage = (status) => {
if (status === 401 || status === 403) return "API key unauthorized";
if (status === 400) return "Invalid model or bad request";
if (status === 404) return "Chat endpoint not found";
if (status >= 500) return "Server error - try again later";
return `Chat request failed (${status})`;
};
it("should return auth error for 401", () => {
expect(getChatErrorMessage(401)).toBe("API key unauthorized");
});
it("should return invalid model for 400", () => {
expect(getChatErrorMessage(400)).toBe("Invalid model or bad request");
});
it("should return not found for 404", () => {
expect(getChatErrorMessage(404)).toBe("Chat endpoint not found");
});
it("should return server error for 503", () => {
expect(getChatErrorMessage(503)).toBe("Server error - try again later");
});
it("should return failed for other codes", () => {
expect(getChatErrorMessage(429)).toBe("Chat request failed (429)");
});
});
describe("Response Format", () => {
it("should return correct format for success via /models", () => {
const response = { valid: true };
expect(response).toEqual({ valid: true });
});
it("should return correct format for success via chat", () => {
const response = { valid: true, error: null, method: "chat" };
expect(response.valid).toBe(true);
expect(response.method).toBe("chat");
expect(response.error).toBeNull();
});
it("should return correct format for failure with error", () => {
const response = {
valid: false,
error: "API key unauthorized or model unavailable",
method: "chat"
};
expect(response.valid).toBe(false);
expect(response.error).toBeDefined();
});
});
});