mirror of
https://github.com/decolua/9router.git
synced 2026-05-08 12:01:28 +00:00
feat: implement model lock functionality for connection management
This commit is contained in:
@@ -138,6 +138,64 @@ export function formatRetryAfter(rateLimitedUntil) {
|
||||
return `reset after ${parts.join(" ")}`;
|
||||
}
|
||||
|
||||
/** Prefix for model lock flat fields on connection record */
|
||||
export const MODEL_LOCK_PREFIX = "modelLock_";
|
||||
|
||||
/** Special key used when no model is known (account-level lock) */
|
||||
export const MODEL_LOCK_ALL = `${MODEL_LOCK_PREFIX}__all`;
|
||||
|
||||
/** Build the flat field key for a model lock */
|
||||
export function getModelLockKey(model) {
|
||||
return model ? `${MODEL_LOCK_PREFIX}${model}` : MODEL_LOCK_ALL;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a model lock on a connection is still active.
|
||||
* Reads flat field `modelLock_${model}` (or `modelLock___all` when model=null).
|
||||
*/
|
||||
export function isModelLockActive(connection, model) {
|
||||
const key = getModelLockKey(model);
|
||||
const expiry = connection[key] || connection[MODEL_LOCK_ALL];
|
||||
if (!expiry) return false;
|
||||
return new Date(expiry).getTime() > Date.now();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get earliest active model lock expiry across all modelLock_* fields.
|
||||
* Used for UI cooldown display.
|
||||
*/
|
||||
export function getEarliestModelLockUntil(connection) {
|
||||
if (!connection) return null;
|
||||
let earliest = null;
|
||||
const now = Date.now();
|
||||
for (const [key, val] of Object.entries(connection)) {
|
||||
if (!key.startsWith(MODEL_LOCK_PREFIX) || !val) continue;
|
||||
const t = new Date(val).getTime();
|
||||
if (t <= now) continue;
|
||||
if (!earliest || t < earliest) earliest = t;
|
||||
}
|
||||
return earliest ? new Date(earliest).toISOString() : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build update object to set a model lock on a connection.
|
||||
*/
|
||||
export function buildModelLockUpdate(model, cooldownMs) {
|
||||
const key = getModelLockKey(model);
|
||||
return { [key]: new Date(Date.now() + cooldownMs).toISOString() };
|
||||
}
|
||||
|
||||
/**
|
||||
* Build update object to clear all model locks on a connection.
|
||||
*/
|
||||
export function buildClearModelLocksUpdate(connection) {
|
||||
const cleared = {};
|
||||
for (const key of Object.keys(connection)) {
|
||||
if (key.startsWith(MODEL_LOCK_PREFIX)) cleared[key] = null;
|
||||
}
|
||||
return cleared;
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter available accounts (not in cooldown)
|
||||
*/
|
||||
|
||||
@@ -31,10 +31,7 @@ export default function CombosPage() {
|
||||
|
||||
if (combosRes.ok) setCombos(combosData.combos || []);
|
||||
if (providersRes.ok) {
|
||||
const active = (providersData.connections || []).filter(
|
||||
c => c.testStatus === "active" || c.testStatus === "success"
|
||||
);
|
||||
setActiveProviders(active);
|
||||
setActiveProviders(providersData.connections || []);
|
||||
}
|
||||
} catch (error) {
|
||||
console.log("Error fetching data:", error);
|
||||
@@ -228,6 +225,80 @@ function ComboCard({ combo, copied, onCopy, onEdit, onDelete }) {
|
||||
);
|
||||
}
|
||||
|
||||
// Inline editable model item
|
||||
function ModelItem({ index, model, isFirst, isLast, onEdit, onMoveUp, onMoveDown, onRemove }) {
|
||||
const [editing, setEditing] = useState(false);
|
||||
const [draft, setDraft] = useState(model);
|
||||
|
||||
const commit = () => {
|
||||
const trimmed = draft.trim();
|
||||
if (trimmed && trimmed !== model) onEdit(trimmed);
|
||||
else setDraft(model); // revert if empty or unchanged
|
||||
setEditing(false);
|
||||
};
|
||||
|
||||
const handleKeyDown = (e) => {
|
||||
if (e.key === "Enter") commit();
|
||||
if (e.key === "Escape") { setDraft(model); setEditing(false); }
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="group flex items-center gap-1.5 px-2 py-1 rounded-md bg-black/[0.02] dark:bg-white/[0.02] hover:bg-black/[0.04] dark:hover:bg-white/[0.04] transition-colors">
|
||||
{/* Index badge */}
|
||||
<span className="text-[10px] font-medium text-text-muted w-3 text-center shrink-0">{index + 1}</span>
|
||||
|
||||
{/* Inline editable model value */}
|
||||
{editing ? (
|
||||
<input
|
||||
autoFocus
|
||||
value={draft}
|
||||
onChange={(e) => setDraft(e.target.value)}
|
||||
onBlur={commit}
|
||||
onKeyDown={handleKeyDown}
|
||||
className="flex-1 min-w-0 px-1.5 py-0.5 text-xs font-mono bg-white dark:bg-black/20 border border-primary/40 rounded outline-none text-text-main"
|
||||
/>
|
||||
) : (
|
||||
<div
|
||||
className="flex-1 min-w-0 px-1.5 py-0.5 text-xs font-mono text-text-main truncate cursor-text hover:bg-black/5 dark:hover:bg-white/5 rounded"
|
||||
onClick={() => setEditing(true)}
|
||||
title="Click to edit"
|
||||
>
|
||||
{model}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Priority arrows */}
|
||||
<div className="flex items-center gap-0.5">
|
||||
<button
|
||||
onClick={onMoveUp}
|
||||
disabled={isFirst}
|
||||
className={`p-0.5 rounded ${isFirst ? "text-text-muted/20 cursor-not-allowed" : "text-text-muted hover:text-primary hover:bg-black/5 dark:hover:bg-white/5"}`}
|
||||
title="Move up"
|
||||
>
|
||||
<span className="material-symbols-outlined text-[12px]">arrow_upward</span>
|
||||
</button>
|
||||
<button
|
||||
onClick={onMoveDown}
|
||||
disabled={isLast}
|
||||
className={`p-0.5 rounded ${isLast ? "text-text-muted/20 cursor-not-allowed" : "text-text-muted hover:text-primary hover:bg-black/5 dark:hover:bg-white/5"}`}
|
||||
title="Move down"
|
||||
>
|
||||
<span className="material-symbols-outlined text-[12px]">arrow_downward</span>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Remove */}
|
||||
<button
|
||||
onClick={onRemove}
|
||||
className="p-0.5 hover:bg-red-500/10 rounded text-text-muted hover:text-red-500 transition-all"
|
||||
title="Remove"
|
||||
>
|
||||
<span className="material-symbols-outlined text-[12px]">close</span>
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders }) {
|
||||
// Initialize state with combo values - key prop on parent handles reset on remount
|
||||
const [name, setName] = useState(combo?.name || "");
|
||||
@@ -236,25 +307,13 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders }) {
|
||||
const [saving, setSaving] = useState(false);
|
||||
const [nameError, setNameError] = useState("");
|
||||
const [modelAliases, setModelAliases] = useState({});
|
||||
const [providerNodes, setProviderNodes] = useState([]);
|
||||
|
||||
const fetchModalData = async () => {
|
||||
try {
|
||||
const [aliasesRes, nodesRes] = await Promise.all([
|
||||
fetch("/api/models/alias"),
|
||||
fetch("/api/provider-nodes"),
|
||||
]);
|
||||
|
||||
if (!aliasesRes.ok || !nodesRes.ok) {
|
||||
throw new Error(`Failed to fetch data: aliases=${aliasesRes.status}, nodes=${nodesRes.status}`);
|
||||
}
|
||||
|
||||
const [aliasesData, nodesData] = await Promise.all([
|
||||
aliasesRes.json(),
|
||||
nodesRes.json(),
|
||||
]);
|
||||
const aliasesRes = await fetch("/api/models/alias");
|
||||
if (!aliasesRes.ok) return;
|
||||
const aliasesData = await aliasesRes.json();
|
||||
setModelAliases(aliasesData.aliases || {});
|
||||
setProviderNodes(nodesData.nodes || []);
|
||||
} catch (error) {
|
||||
console.error("Error fetching modal data:", error);
|
||||
}
|
||||
@@ -294,21 +353,6 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders }) {
|
||||
setModels(models.filter((_, i) => i !== index));
|
||||
};
|
||||
|
||||
// Format model display name with readable provider name
|
||||
const formatModelDisplay = useCallback((modelValue) => {
|
||||
const parts = modelValue.split('/');
|
||||
if (parts.length !== 2) return modelValue;
|
||||
|
||||
const [providerId, modelId] = parts;
|
||||
const matchedNode = providerNodes.find(node => node.id === providerId);
|
||||
|
||||
if (matchedNode) {
|
||||
return `${matchedNode.name}/${modelId}`;
|
||||
}
|
||||
|
||||
return modelValue;
|
||||
}, [providerNodes]);
|
||||
|
||||
const handleMoveUp = (index) => {
|
||||
if (index === 0) return;
|
||||
const newModels = [...models];
|
||||
@@ -366,52 +410,26 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders }) {
|
||||
) : (
|
||||
<div className="flex flex-col gap-1 max-h-[200px] overflow-y-auto">
|
||||
{models.map((model, index) => (
|
||||
<div
|
||||
<ModelItem
|
||||
key={index}
|
||||
className="group flex items-center gap-1.5 px-2 py-1 rounded-md bg-black/[0.02] dark:bg-white/[0.02] hover:bg-black/[0.04] dark:hover:bg-white/[0.04] transition-colors"
|
||||
>
|
||||
{/* Index badge */}
|
||||
<span className="text-[10px] font-medium text-text-muted w-3 text-center shrink-0">{index + 1}</span>
|
||||
|
||||
{/* Model display - show readable name only */}
|
||||
<div className="flex-1 min-w-0 px-1.5 py-0.5 text-xs text-text-main truncate">
|
||||
{formatModelDisplay(model)}
|
||||
</div>
|
||||
|
||||
{/* Priority arrows - horizontal, always visible */}
|
||||
<div className="flex items-center gap-0.5">
|
||||
<button
|
||||
onClick={() => handleMoveUp(index)}
|
||||
disabled={index === 0}
|
||||
className={`p-0.5 rounded ${index === 0 ? "text-text-muted/20 cursor-not-allowed" : "text-text-muted hover:text-primary hover:bg-black/5 dark:hover:bg-white/5"}`}
|
||||
title="Move up"
|
||||
>
|
||||
<span className="material-symbols-outlined text-[12px]">arrow_upward</span>
|
||||
</button>
|
||||
<button
|
||||
onClick={() => handleMoveDown(index)}
|
||||
disabled={index === models.length - 1}
|
||||
className={`p-0.5 rounded ${index === models.length - 1 ? "text-text-muted/20 cursor-not-allowed" : "text-text-muted hover:text-primary hover:bg-black/5 dark:hover:bg-white/5"}`}
|
||||
title="Move down"
|
||||
>
|
||||
<span className="material-symbols-outlined text-[12px]">arrow_downward</span>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Remove - always visible */}
|
||||
<button
|
||||
onClick={() => handleRemoveModel(index)}
|
||||
className="p-0.5 hover:bg-red-500/10 rounded text-text-muted hover:text-red-500 transition-all"
|
||||
title="Remove"
|
||||
>
|
||||
<span className="material-symbols-outlined text-[12px]">close</span>
|
||||
</button>
|
||||
</div>
|
||||
index={index}
|
||||
model={model}
|
||||
isFirst={index === 0}
|
||||
isLast={index === models.length - 1}
|
||||
onEdit={(newVal) => {
|
||||
const updated = [...models];
|
||||
updated[index] = newVal;
|
||||
setModels(updated);
|
||||
}}
|
||||
onMoveUp={() => handleMoveUp(index)}
|
||||
onMoveDown={() => handleMoveDown(index)}
|
||||
onRemove={() => handleRemoveModel(index)}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Add Model button - moved to bottom */}
|
||||
{/* Add Model button */}
|
||||
<button
|
||||
onClick={() => setShowModelSelect(true)}
|
||||
className="w-full mt-2 py-2 border border-dashed border-black/10 dark:border-white/10 rounded-lg text-xs text-text-muted hover:text-primary hover:border-primary/30 transition-colors flex items-center justify-center gap-1"
|
||||
@@ -450,4 +468,3 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders }) {
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -919,20 +919,29 @@ function ConnectionRow({ connection, isOAuth, isFirst, isLast, onMoveUp, onMoveD
|
||||
// Use useState + useEffect for impure Date.now() to avoid calling during render
|
||||
const [isCooldown, setIsCooldown] = useState(false);
|
||||
|
||||
// Get earliest model lock timestamp (useEffect handles the Date.now() comparison)
|
||||
const modelLockUntil = Object.entries(connection)
|
||||
.filter(([k]) => k.startsWith("modelLock_"))
|
||||
.map(([, v]) => v)
|
||||
.filter(v => !!v)
|
||||
.sort()[0] || null;
|
||||
|
||||
useEffect(() => {
|
||||
const checkCooldown = () => {
|
||||
const cooldown = connection.rateLimitedUntil &&
|
||||
new Date(connection.rateLimitedUntil).getTime() > Date.now();
|
||||
setIsCooldown(cooldown);
|
||||
const until = Object.entries(connection)
|
||||
.filter(([k]) => k.startsWith("modelLock_"))
|
||||
.map(([, v]) => v)
|
||||
.filter(v => v && new Date(v).getTime() > Date.now())
|
||||
.sort()[0] || null;
|
||||
setIsCooldown(!!until);
|
||||
};
|
||||
|
||||
checkCooldown();
|
||||
// Update every second while in cooldown
|
||||
const interval = connection.rateLimitedUntil ? setInterval(checkCooldown, 1000) : null;
|
||||
const interval = modelLockUntil ? setInterval(checkCooldown, 1000) : null;
|
||||
return () => {
|
||||
if (interval) clearInterval(interval);
|
||||
};
|
||||
}, [connection.rateLimitedUntil]);
|
||||
}, [modelLockUntil]);
|
||||
|
||||
// Determine effective status (override unavailable if cooldown expired)
|
||||
const effectiveStatus = (connection.testStatus === "unavailable" && !isCooldown)
|
||||
@@ -975,7 +984,7 @@ function ConnectionRow({ connection, isOAuth, isFirst, isLast, onMoveUp, onMoveD
|
||||
<Badge variant={getStatusVariant()} size="sm" dot>
|
||||
{connection.isActive === false ? "disabled" : (effectiveStatus || "Unknown")}
|
||||
</Badge>
|
||||
{isCooldown && connection.isActive !== false && <CooldownTimer until={connection.rateLimitedUntil} />}
|
||||
{isCooldown && connection.isActive !== false && <CooldownTimer until={modelLockUntil} />}
|
||||
{connection.lastError && connection.isActive !== false && (
|
||||
<span className="text-xs text-red-500 truncate max-w-[300px]" title={connection.lastError}>
|
||||
{connection.lastError}
|
||||
@@ -1014,7 +1023,7 @@ ConnectionRow.propTypes = {
|
||||
name: PropTypes.string,
|
||||
email: PropTypes.string,
|
||||
displayName: PropTypes.string,
|
||||
rateLimitedUntil: PropTypes.string,
|
||||
modelLockUntil: PropTypes.string,
|
||||
testStatus: PropTypes.string,
|
||||
isActive: PropTypes.bool,
|
||||
lastError: PropTypes.string,
|
||||
|
||||
@@ -1144,20 +1144,28 @@ function ConnectionRow({ connection, isOAuth, isFirst, isLast, onMoveUp, onMoveD
|
||||
// Use useState + useEffect for impure Date.now() to avoid calling during render
|
||||
const [isCooldown, setIsCooldown] = useState(false);
|
||||
|
||||
const modelLockUntil = Object.entries(connection)
|
||||
.filter(([k]) => k.startsWith("modelLock_"))
|
||||
.map(([, v]) => v)
|
||||
.filter(v => v && new Date(v).getTime() > Date.now())
|
||||
.sort()[0] || null;
|
||||
|
||||
useEffect(() => {
|
||||
const checkCooldown = () => {
|
||||
const cooldown = connection.rateLimitedUntil &&
|
||||
new Date(connection.rateLimitedUntil).getTime() > Date.now();
|
||||
setIsCooldown(cooldown);
|
||||
const until = Object.entries(connection)
|
||||
.filter(([k]) => k.startsWith("modelLock_"))
|
||||
.map(([, v]) => v)
|
||||
.filter(v => v && new Date(v).getTime() > Date.now())
|
||||
.sort()[0] || null;
|
||||
setIsCooldown(!!until);
|
||||
};
|
||||
|
||||
checkCooldown();
|
||||
// Update every second while in cooldown
|
||||
const interval = connection.rateLimitedUntil ? setInterval(checkCooldown, 1000) : null;
|
||||
const interval = modelLockUntil ? setInterval(checkCooldown, 1000) : null;
|
||||
return () => {
|
||||
if (interval) clearInterval(interval);
|
||||
};
|
||||
}, [connection.rateLimitedUntil]);
|
||||
}, [modelLockUntil]);
|
||||
|
||||
// Determine effective status (override unavailable if cooldown expired)
|
||||
const effectiveStatus = (connection.testStatus === "unavailable" && !isCooldown)
|
||||
@@ -1200,7 +1208,7 @@ function ConnectionRow({ connection, isOAuth, isFirst, isLast, onMoveUp, onMoveD
|
||||
<Badge variant={getStatusVariant()} size="sm" dot>
|
||||
{connection.isActive === false ? "disabled" : (effectiveStatus || "Unknown")}
|
||||
</Badge>
|
||||
{isCooldown && connection.isActive !== false && <CooldownTimer until={connection.rateLimitedUntil} />}
|
||||
{isCooldown && connection.isActive !== false && <CooldownTimer until={modelLockUntil} />}
|
||||
{connection.lastError && connection.isActive !== false && (
|
||||
<span className="text-xs text-red-500 truncate max-w-[300px]" title={connection.lastError}>
|
||||
{connection.lastError}
|
||||
@@ -1239,7 +1247,7 @@ ConnectionRow.propTypes = {
|
||||
name: PropTypes.string,
|
||||
email: PropTypes.string,
|
||||
displayName: PropTypes.string,
|
||||
rateLimitedUntil: PropTypes.string,
|
||||
modelLockUntil: PropTypes.string,
|
||||
testStatus: PropTypes.string,
|
||||
isActive: PropTypes.bool,
|
||||
lastError: PropTypes.string,
|
||||
|
||||
@@ -99,7 +99,8 @@ export default function ProvidersPage() {
|
||||
);
|
||||
|
||||
const getEffectiveStatus = (conn) => {
|
||||
const isCooldown = conn.rateLimitedUntil && new Date(conn.rateLimitedUntil).getTime() > Date.now();
|
||||
const isCooldown = Object.entries(conn)
|
||||
.some(([k, v]) => k.startsWith("modelLock_") && v && new Date(v).getTime() > Date.now());
|
||||
return conn.testStatus === "unavailable" && !isCooldown ? "active" : conn.testStatus;
|
||||
};
|
||||
|
||||
|
||||
@@ -1,67 +1,11 @@
|
||||
import { getProviderConnections, validateApiKey, updateProviderConnection, getSettings } from "@/lib/localDb";
|
||||
import { isAccountUnavailable, getUnavailableUntil, getEarliestRateLimitedUntil, formatRetryAfter, checkFallbackError } from "open-sse/services/accountFallback.js";
|
||||
import { formatRetryAfter, checkFallbackError, isModelLockActive, buildModelLockUpdate, buildClearModelLocksUpdate, getEarliestModelLockUntil } from "open-sse/services/accountFallback.js";
|
||||
import { resolveProviderId } from "@/shared/constants/providers.js";
|
||||
import * as log from "../utils/logger.js";
|
||||
|
||||
// Mutex to prevent race conditions during account selection
|
||||
let selectionMutex = Promise.resolve();
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Model-level rate-limit locking (in-memory)
|
||||
//
|
||||
// Providers like Antigravity maintain *separate* quota buckets per model
|
||||
// family (e.g. Claude vs Gemini). When a 429 arrives for one model, the
|
||||
// standard account-level DB lock would block ALL models on that account,
|
||||
// wasting quota that is still available for other model families.
|
||||
//
|
||||
// This module tracks model-specific locks in memory so that only the
|
||||
// affected model is skipped during account selection while the rest of the
|
||||
// account's quota remains accessible.
|
||||
//
|
||||
// Locks are intentionally in-memory: they clear on restart and require no
|
||||
// database schema migration.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Providers known to have independent per-model quota buckets */
|
||||
const MULTI_BUCKET_PROVIDERS = new Set(["antigravity"]);
|
||||
|
||||
/** Map<"connectionId:model", expiryTimestamp> */
|
||||
const modelLocks = new Map();
|
||||
|
||||
/** Default lock duration for model-level rate limits (5 minutes) */
|
||||
const DEFAULT_MODEL_LOCK_MS = 5 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* Check whether a specific model is temporarily locked on a connection.
|
||||
* Expired locks are cleaned up lazily.
|
||||
*/
|
||||
function isModelLocked(connectionId, model) {
|
||||
if (!connectionId || !model) return false;
|
||||
const key = `${connectionId}:${model}`;
|
||||
const expiry = modelLocks.get(key);
|
||||
if (!expiry) return false;
|
||||
if (expiry > Date.now()) return true;
|
||||
modelLocks.delete(key); // clean up expired
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Lock a model on a specific connection for `durationMs` milliseconds.
|
||||
*/
|
||||
function lockModel(connectionId, model, durationMs) {
|
||||
if (!connectionId || !model) return;
|
||||
const key = `${connectionId}:${model}`;
|
||||
modelLocks.set(key, Date.now() + durationMs);
|
||||
log.warn("AUTH", `Model lock: ${model} on ${connectionId.slice(0, 8)} for ${Math.round(durationMs / 1000)}s`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether a provider uses per-model quota buckets.
|
||||
*/
|
||||
function isMultiBucketProvider(provider) {
|
||||
return MULTI_BUCKET_PROVIDERS.has(provider);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get provider credentials from localDb
|
||||
* Filters out unavailable accounts and returns the selected account based on strategy
|
||||
@@ -85,50 +29,35 @@ export async function getProviderCredentials(provider, excludeConnectionId = nul
|
||||
log.debug("AUTH", `${provider} | total connections: ${connections.length}, excludeId: ${excludeConnectionId || "none"}, model: ${model || "any"}`);
|
||||
|
||||
if (connections.length === 0) {
|
||||
// Check all connections (including inactive) to see if rate limited
|
||||
const allConnections = await getProviderConnections({ provider: providerId });
|
||||
log.debug("AUTH", `${provider} | all connections (incl inactive): ${allConnections.length}`);
|
||||
if (allConnections.length > 0) {
|
||||
const earliest = getEarliestRateLimitedUntil(allConnections);
|
||||
if (earliest) {
|
||||
log.warn("AUTH", `${provider} | all ${allConnections.length} accounts rate limited (${formatRetryAfter(earliest)})`);
|
||||
return { allRateLimited: true, retryAfter: earliest, retryAfterHuman: formatRetryAfter(earliest) };
|
||||
}
|
||||
log.warn("AUTH", `${provider} | ${allConnections.length} accounts found but none active`);
|
||||
allConnections.forEach(c => {
|
||||
log.debug("AUTH", ` → ${c.id?.slice(0, 8)} | isActive=${c.isActive} | rateLimitedUntil=${c.rateLimitedUntil || "none"} | testStatus=${c.testStatus}`);
|
||||
});
|
||||
}
|
||||
log.warn("AUTH", `No credentials for ${provider}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Filter out unavailable accounts and excluded connection
|
||||
const multiBucket = isMultiBucketProvider(provider);
|
||||
// Filter out model-locked and excluded connections
|
||||
const availableConnections = connections.filter(c => {
|
||||
if (excludeConnectionId && c.id === excludeConnectionId) return false;
|
||||
if (isAccountUnavailable(c.rateLimitedUntil)) return false;
|
||||
if (multiBucket && model && isModelLocked(c.id, model)) return false;
|
||||
if (isModelLockActive(c, model)) return false;
|
||||
return true;
|
||||
});
|
||||
|
||||
log.debug("AUTH", `${provider} | available: ${availableConnections.length}/${connections.length}`);
|
||||
connections.forEach(c => {
|
||||
const excluded = excludeConnectionId && c.id === excludeConnectionId;
|
||||
const rateLimited = isAccountUnavailable(c.rateLimitedUntil);
|
||||
const modelLocked = multiBucket && model && isModelLocked(c.id, model);
|
||||
if (excluded || rateLimited || modelLocked) {
|
||||
log.debug("AUTH", ` → ${c.id?.slice(0, 8)} | ${excluded ? "excluded" : ""} ${rateLimited ? `rateLimited until ${c.rateLimitedUntil}` : ""} ${modelLocked ? `modelLocked(${model})` : ""}`);
|
||||
const locked = isModelLockActive(c, model);
|
||||
if (excluded || locked) {
|
||||
const lockUntil = getEarliestModelLockUntil(c);
|
||||
log.debug("AUTH", ` → ${c.id?.slice(0, 8)} | ${excluded ? "excluded" : ""} ${locked ? `modelLocked(${model}) until ${lockUntil}` : ""}`);
|
||||
}
|
||||
});
|
||||
|
||||
if (availableConnections.length === 0) {
|
||||
const earliest = getEarliestRateLimitedUntil(connections);
|
||||
// Find earliest lock expiry across all connections for retry timing
|
||||
const lockedConns = connections.filter(c => isModelLockActive(c, model));
|
||||
const expiries = lockedConns.map(c => getEarliestModelLockUntil(c)).filter(Boolean);
|
||||
const earliest = expiries.sort()[0] || null;
|
||||
if (earliest) {
|
||||
// Find the connection with the earliest rateLimitedUntil to get its error info
|
||||
const rateLimitedConns = connections.filter(c => c.rateLimitedUntil && new Date(c.rateLimitedUntil).getTime() > Date.now());
|
||||
const earliestConn = rateLimitedConns.sort((a, b) => new Date(a.rateLimitedUntil) - new Date(b.rateLimitedUntil))[0];
|
||||
log.warn("AUTH", `${provider} | all ${connections.length} active accounts rate limited (${formatRetryAfter(earliest)}) | lastErrorCode=${earliestConn?.errorCode}, lastError=${earliestConn?.lastError?.slice(0, 50)}`);
|
||||
const earliestConn = lockedConns[0];
|
||||
log.warn("AUTH", `${provider} | all ${connections.length} accounts locked for ${model || "all"} (${formatRetryAfter(earliest)}) | lastError=${earliestConn?.lastError?.slice(0, 50)}`);
|
||||
return {
|
||||
allRateLimited: true,
|
||||
retryAfter: earliest,
|
||||
@@ -137,15 +66,6 @@ export async function getProviderCredentials(provider, excludeConnectionId = nul
|
||||
lastErrorCode: earliestConn?.errorCode || null
|
||||
};
|
||||
}
|
||||
if (multiBucket && model) {
|
||||
log.warn("AUTH", `${provider} | all accounts model-locked for ${model}`);
|
||||
return {
|
||||
allRateLimited: true,
|
||||
retryAfter: new Date(Date.now() + 60000).toISOString(),
|
||||
retryAfterHuman: "reset after 1m",
|
||||
lastError: `All accounts rate limited for model ${model}`
|
||||
};
|
||||
}
|
||||
log.warn("AUTH", `${provider} | all ${connections.length} accounts unavailable`);
|
||||
return null;
|
||||
}
|
||||
@@ -209,7 +129,8 @@ export async function getProviderCredentials(provider, excludeConnectionId = nul
|
||||
// Include current status for optimization check
|
||||
testStatus: connection.testStatus,
|
||||
lastError: connection.lastError,
|
||||
rateLimitedUntil: connection.rateLimitedUntil
|
||||
// Pass full connection for clearAccountError to read modelLock_* keys
|
||||
_connection: connection
|
||||
};
|
||||
} finally {
|
||||
if (resolveMutex) resolveMutex();
|
||||
@@ -217,9 +138,8 @@ export async function getProviderCredentials(provider, excludeConnectionId = nul
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark account as unavailable — reads backoffLevel from DB, calculates cooldown with exponential backoff, saves new level.
|
||||
* For multi-bucket providers (e.g. Antigravity), 429 errors lock only the specific model in memory
|
||||
* rather than the entire account in the database.
|
||||
* Mark account+model as unavailable — locks modelLock_${model} in DB.
|
||||
* All errors (429, 401, 5xx, etc.) lock per model, not per account.
|
||||
* @param {string} connectionId
|
||||
* @param {number} status - HTTP status code from upstream
|
||||
* @param {string} errorText
|
||||
@@ -228,7 +148,6 @@ export async function getProviderCredentials(provider, excludeConnectionId = nul
|
||||
* @returns {{ shouldFallback: boolean, cooldownMs: number }}
|
||||
*/
|
||||
export async function markAccountUnavailable(connectionId, status, errorText, provider = null, model = null) {
|
||||
// Read current connection to get backoffLevel
|
||||
const connections = await getProviderConnections({ provider });
|
||||
const conn = connections.find(c => c.id === connectionId);
|
||||
const backoffLevel = conn?.backoffLevel || 0;
|
||||
@@ -236,17 +155,11 @@ export async function markAccountUnavailable(connectionId, status, errorText, pr
|
||||
const { shouldFallback, cooldownMs, newBackoffLevel } = checkFallbackError(status, errorText, backoffLevel);
|
||||
if (!shouldFallback) return { shouldFallback: false, cooldownMs: 0 };
|
||||
|
||||
if (isMultiBucketProvider(provider) && status === 429 && model) {
|
||||
const lockDuration = cooldownMs > 0 ? cooldownMs : DEFAULT_MODEL_LOCK_MS;
|
||||
lockModel(connectionId, model, lockDuration);
|
||||
return { shouldFallback: true, cooldownMs: 0 };
|
||||
}
|
||||
|
||||
const rateLimitedUntil = getUnavailableUntil(cooldownMs);
|
||||
const reason = typeof errorText === "string" ? errorText.slice(0, 100) : "Provider error";
|
||||
const lockUpdate = buildModelLockUpdate(model, cooldownMs);
|
||||
|
||||
await updateProviderConnection(connectionId, {
|
||||
rateLimitedUntil,
|
||||
...lockUpdate,
|
||||
testStatus: "unavailable",
|
||||
lastError: reason,
|
||||
errorCode: status,
|
||||
@@ -254,6 +167,9 @@ export async function markAccountUnavailable(connectionId, status, errorText, pr
|
||||
backoffLevel: newBackoffLevel ?? backoffLevel
|
||||
});
|
||||
|
||||
const lockKey = Object.keys(lockUpdate)[0];
|
||||
log.warn("AUTH", `${connectionId.slice(0, 8)} locked ${lockKey} for ${Math.round(cooldownMs / 1000)}s [${status}]`);
|
||||
|
||||
if (provider && status && reason) {
|
||||
console.error(`❌ ${provider} [${status}]: ${reason}`);
|
||||
}
|
||||
@@ -263,21 +179,26 @@ export async function markAccountUnavailable(connectionId, status, errorText, pr
|
||||
|
||||
/**
|
||||
* Clear account error status (only if currently has error)
|
||||
* Optimized to avoid unnecessary DB updates
|
||||
* Clears all modelLock_* fields and resets error state.
|
||||
*/
|
||||
export async function clearAccountError(connectionId, currentConnection) {
|
||||
// Only update if currently has error status
|
||||
const hasError = currentConnection.testStatus === "unavailable" ||
|
||||
currentConnection.lastError ||
|
||||
currentConnection.rateLimitedUntil;
|
||||
// Support both direct connection object and credentials wrapper
|
||||
const conn = currentConnection._connection || currentConnection;
|
||||
const now = Date.now();
|
||||
|
||||
// Collect all modelLock_* keys (both active and expired)
|
||||
const allLockKeys = Object.keys(conn).filter(k => k.startsWith("modelLock_"));
|
||||
const hasError = conn.testStatus === "unavailable" || conn.lastError || allLockKeys.length > 0;
|
||||
|
||||
if (!hasError) return; // Skip if already clean
|
||||
|
||||
// Clear all modelLock_* keys (lazy cleanup of expired ones included)
|
||||
const clearLocks = Object.fromEntries(allLockKeys.map(k => [k, null]));
|
||||
await updateProviderConnection(connectionId, {
|
||||
...clearLocks,
|
||||
testStatus: "active",
|
||||
lastError: null,
|
||||
lastErrorAt: null,
|
||||
rateLimitedUntil: null,
|
||||
backoffLevel: 0
|
||||
});
|
||||
log.info("AUTH", `Account ${connectionId.slice(0, 8)} error cleared`);
|
||||
|
||||
Reference in New Issue
Block a user