AI网关架构设计：统一管理多LLM提供商的工程实践

引言当一个企业同时使用OpenAI、Anthropic、Azure OpenAI、本地部署的LLaMA……如何统一管理这些提供商如何实现智能路由、故障转移、成本控制和访问审计AI网关AI Gateway正是为这一需求而生的基础设施组件。它在业务应用和LLM提供商之间架设统一的代理层解决多提供商管理的复杂性。本文将深度解析AI网关的设计架构和工程实现。—## 一、AI网关的核心价值### 1.1 没有AI网关的痛苦现状没有AI网关:应用A → OpenAI API各自独立的SDK应用B → Anthropic API各自独立的SDK应用C → Azure OpenAI各自独立的SDK应用D → 本地LLM各自独立的SDK问题- 每个应用独立管理API密钥和成本- 某个提供商宕机所有依赖它的应用都挂- 无法全局限速和配额管理- 没有统一的访问日志和审计能力- 切换提供商需要修改每个应用的代码### 1.2 AI网关的核心能力理想状态有AI网关:应用A ─┐应用B ─┤→ [AI Gateway] ─→ OpenAI应用C ─┤ ├─→ Anthropic应用D ─┘ ├─→ Azure OpenAI └─→ 本地LLMAI网关提供✅ 统一API接口OpenAI兼容格式✅ 智能路由基于成本、延迟、能力✅ 自动故障转移✅ 全局速率限制✅ 访问审计和成本归因✅ 响应缓存✅ 敏感信息脱敏—## 二、核心架构设计### 2.1 网关整体架构pythonfrom fastapi import FastAPI, Request, HTTPException, Dependsfrom fastapi.responses import StreamingResponsefrom typing import Optional, AsyncIteratorimport asyncioimport timeimport jsonimport loggingapp FastAPI(titleAI Gateway)logger logging.getLogger(__name__)class AIGateway: AI网关核心组件 def __init__(self): self.router ProviderRouter() self.rate_limiter RateLimiter() self.cache ResponseCache() self.auditor AuditLogger() self.cost_tracker CostTracker()### 2.2 提供商路由器pythonfrom dataclasses import dataclass, fieldfrom enum import Enumimport httpximport randomclass RoutingStrategy(Enum): COST_OPTIMIZED cost_optimized # 最低成本 LATENCY_OPTIMIZED latency_optimized # 最低延迟 CAPABILITY_BASED capability_based # 按能力路由 ROUND_ROBIN round_robin # 轮询 FAILOVER failover # 故障转移dataclassclass ProviderConfig: name: str api_base: str api_key: str model_mapping: dict # 统一模型名 → 提供商模型名 # 能力声明 max_context_tokens: int 128000 supports_vision: bool False supports_function_calling: bool True # 价格per 1M tokens input_price_per_1m: float 3.0 output_price_per_1m: float 15.0 # 实时指标动态更新 avg_latency_ms: float 1000 error_rate: float 0.0 is_available: bool True # 权重用于加权轮询 weight: int 1class ProviderRouter: 智能提供商路由器 def __init__(self): self.providers: dict[str, ProviderConfig] {} self._round_robin_idx 0 def register_provider(self, config: ProviderConfig): self.providers[config.name] config def select_provider( self, request_context: dict, strategy: RoutingStrategy RoutingStrategy.COST_OPTIMIZED ) - Optional[ProviderConfig]: 根据策略选择最优提供商 available [ p for p in self.providers.values() if p.is_available and self._meets_requirements(p, request_context) ] if not available: return None if strategy RoutingStrategy.COST_OPTIMIZED: return min(available, keylambda p: p.input_price_per_1m) elif strategy RoutingStrategy.LATENCY_OPTIMIZED: return min(available, keylambda p: p.avg_latency_ms) elif strategy RoutingStrategy.ROUND_ROBIN: provider available[self._round_robin_idx % len(available)] self._round_robin_idx 1 return provider elif strategy RoutingStrategy.CAPABILITY_BASED: return self._route_by_capability(available, request_context) elif strategy RoutingStrategy.FAILOVER: # 按权重排序选第一个可用的 return sorted(available, keylambda p: -p.weight)[0] return available[0] def _meets_requirements( self, provider: ProviderConfig, request_context: dict ) - bool: 检查提供商是否满足请求需求 # 检查context长度 token_estimate request_context.get(estimated_tokens, 0) if token_estimate provider.max_context_tokens: return False # 检查是否需要视觉能力 if request_context.get(has_images) and not provider.supports_vision: return False # 检查是否需要函数调用 if request_context.get(has_tools) and not provider.supports_function_calling: return False return True def _route_by_capability( self, providers: list, context: dict ) - ProviderConfig: 按任务复杂度路由 complexity context.get(complexity_score, 0.5) if complexity 0.8: # 复杂任务选最强模型通常也最贵 return max(providers, keylambda p: p.max_context_tokens) elif complexity 0.3: # 简单任务选最便宜的 return min(providers, keylambda p: p.input_price_per_1m) else: # 中等任务按延迟 return min(providers, keylambda p: p.avg_latency_ms) async def update_provider_metrics( self, provider_name: str, latency_ms: float, success: bool ): 更新提供商实时指标指数移动平均 if provider_name not in self.providers: return p self.providers[provider_name] alpha 0.1 # 平滑系数 # 更新平均延迟 p.avg_latency_ms alpha * latency_ms (1 - alpha) * p.avg_latency_ms # 更新错误率 error_val 0 if success else 1 p.error_rate alpha * error_val (1 - alpha) * p.error_rate # 如果错误率过高标记为不可用 if p.error_rate 0.5: p.is_available False logger.warning(f提供商 {provider_name} 错误率过高({p.error_rate:.1%})暂时下线) # 5分钟后自动重试 asyncio.create_task(self._schedule_recovery(provider_name, 300)) async def _schedule_recovery(self, provider_name: str, delay_seconds: int): await asyncio.sleep(delay_seconds) if provider_name in self.providers: self.providers[provider_name].is_available True self.providers[provider_name].error_rate 0.0 logger.info(f提供商 {provider_name} 已恢复上线)### 2.3 故障转移机制pythonclass FailoverHandler: 自动故障转移处理器 def __init__(self, router: ProviderRouter, max_retries: int 3): self.router router self.max_retries max_retries async def execute_with_failover( self, request: dict, excluded_providers: set None ) - dict: 执行请求自动故障转移 excluded excluded_providers or set() errors [] for attempt in range(self.max_retries): provider self.router.select_provider( request, strategyRoutingStrategy.FAILOVER ) if provider is None or provider.name in excluded: break try: start time.time() result await self._call_provider(provider, request) latency (time.time() - start) * 1000 # 更新成功指标 await self.router.update_provider_metrics( provider.name, latency, successTrue ) result[_provider_used] provider.name result[_latency_ms] latency return result except Exception as e: latency (time.time() - start) * 1000 await self.router.update_provider_metrics( provider.name, latency, successFalse ) errors.append({ provider: provider.name, error: str(e) }) excluded.add(provider.name) logger.warning( f提供商 {provider.name} 失败: {e} f尝试故障转移{attempt1}/{self.max_retries} ) raise Exception(f所有提供商均失败: {errors}) async def _call_provider( self, provider: ProviderConfig, request: dict ) - dict: 调用指定提供商 # 转换模型名称 model request.get(model, gpt-4o) provider_model provider.model_mapping.get(model, model) provider_request {**request, model: provider_model} async with httpx.AsyncClient(timeout30.0) as client: response await client.post( f{provider.api_base}/chat/completions, jsonprovider_request, headers{ Authorization: fBearer {provider.api_key}, Content-Type: application/json } ) response.raise_for_status() return response.json()—## 三、速率限制与配额管理pythonimport redis.asyncio as redisfrom typing import Optionalclass RateLimiter: 基于Redis的分布式速率限制 def __init__(self, redis_url: str): self.redis redis.from_url(redis_url) async def check_and_consume( self, tenant_id: str, tokens_needed: int, limits: dict ) - dict: 检查并消耗配额 Returns: {allowed: bool, remaining: int, reset_at: int} pipe self.redis.pipeline() now int(time.time()) # 分钟级限制 minute_key frl:{tenant_id}:min:{now // 60} # 天级限制 day_key frl:{tenant_id}:day:{now // 86400} # 月级Token配额 month_key frl:{tenant_id}:tokens:month:{now // (86400 * 30)} async with self.redis.pipeline() as pipe: pipe.incrby(minute_key, tokens_needed) pipe.expire(minute_key, 120) pipe.incrby(day_key, 1) pipe.expire(day_key, 172800) pipe.incrby(month_key, tokens_needed) pipe.expire(month_key, 5184000) results await pipe.execute() minute_total results[0] day_requests results[2] month_tokens results[4] # 检查是否超限 if minute_total limits.get(tokens_per_minute, 100000): return { allowed: False, reason: minute_token_limit, remaining: 0, reset_at: (now // 60 1) * 60 } if day_requests limits.get(requests_per_day, 10000): return { allowed: False, reason: daily_request_limit, remaining: 0, reset_at: (now // 86400 1) * 86400 } if month_tokens limits.get(tokens_per_month, 10000000): return { allowed: False, reason: monthly_token_limit, remaining: 0, reset_at: (now // (86400 * 30) 1) * 86400 * 30 } return { allowed: True, remaining: limits.get(tokens_per_minute, 100000) - minute_total, reset_at: (now // 60 1) * 60 }—## 四、FastAPI接口层pythonfrom fastapi import FastAPI, Request, Headerfrom fastapi.responses import JSONResponse, StreamingResponseapp FastAPI(titleAI Gateway v1.0)app.post(/v1/chat/completions)async def chat_completions( request: Request, authorization: str Header(...)): OpenAI兼容的统一入口 # 解析租户 api_key authorization.replace(Bearer , ) tenant await validate_api_key(api_key) if not tenant: raise HTTPException(status_code401, detailInvalid API key) body await request.json() # 速率限制检查 rate_check await gateway.rate_limiter.check_and_consume( tenant[id], tokens_neededestimate_tokens(body), limitstenant.get(limits, {}) ) if not rate_check[allowed]: return JSONResponse( status_code429, content{ error: { type: rate_limit_exceeded, message: f配额超限: {rate_check[reason]}, reset_at: rate_check[reset_at] } } ) # 路由并执行 failover FailoverHandler(gateway.router) try: result await failover.execute_with_failover(body) # 审计日志 await gateway.auditor.log({ tenant_id: tenant[id], model: body.get(model), provider: result.get(_provider_used), tokens: result.get(usage, {}), latency_ms: result.get(_latency_ms) }) return JSONResponse(contentresult) except Exception as e: raise HTTPException(status_code503, detailstr(e))—## 五、总结AI网关是企业AI基础设施的核心中间件。关键设计要点1.统一接口采用OpenAI兼容格式业务应用无需改动2.多策略路由成本、延迟、能力三维度智能选择3.自动故障转移指数移动平均监测健康状态自动切换4.分布式限速Redis支撑多实例部署支持分钟/天/月三级配额5.完整审计每次调用记录租户、提供商、成本支持费用分摊从长远看AI网关会逐渐向AI可观测性平台演进集成Prompt管理、A/B测试、模型评估等能力成为企业AI工程化的核心基础设施。