LLM 게이트웨이: 멀티 프로바이더 라우팅·폴백

클라이언트
    ↓
LLM 게이트웨이 (라우팅 / 폴백 / 캐싱 / 로깅)
    ↓        ↓        ↓
  OpenAI  Anthropic  Google

from litellm import completion

# 모델명만 바꾸면 모든 프로바이더 동일하게 사용
def llm_call(model, messages):
    response = completion(model=model, messages=messages)
    return response.choices[0].message.content

llm_call("gpt-4o",              [{"role": "user", "content": "안녕"}])
llm_call("claude-sonnet-4-6",   [{"role": "user", "content": "안녕"}])
llm_call("gemini/gemini-1.5-pro",[{"role": "user", "content": "안녕"}])

from litellm import completion

FALLBACK_CHAIN = [
    "gpt-4o",
    "claude-sonnet-4-6",
    "gemini/gemini-1.5-pro"
]

def resilient_llm_call(messages, preferred_model=None):
    chain = ([preferred_model] + FALLBACK_CHAIN) if preferred_model else FALLBACK_CHAIN
    chain = list(dict.fromkeys(chain))

    last_error = None
    for model in chain:
        try:
            response = completion(model=model, messages=messages, timeout=10, num_retries=1)
            return {
                "content":      response.choices[0].message.content,
                "model_used":   model,
                "was_fallback": model != chain[0]
            }
        except Exception as e:
            last_error = e
            print(f"[WARN] {model} 실패, 다음 모델로 시도")

    raise RuntimeError(f"모든 모델 실패: {last_error}")

ROUTING_TABLE = {
    "low":    {"primary": "gpt-4o-mini",       "fallback": "claude-haiku-4-5-20251001"},
    "medium": {"primary": "claude-sonnet-4-6", "fallback": "gpt-4o"},
    "high":   {"primary": "claude-opus-4-7",   "fallback": "gpt-4o"},
}

def classify_task(messages):
    user_text = " ".join(m["content"] for m in messages if m["role"] == "user")
    if any(kw in user_text for kw in ["분석", "설계", "아키텍처", "전략"]):
        return "high"
    if any(kw in user_text for kw in ["번역", "요약", "분류", "추출"]):
        return "low"
    return "medium"

def smart_route(messages):
    complexity = classify_task(messages)
    route = ROUTING_TABLE[complexity]
    return resilient_llm_call(messages, preferred_model=route["primary"])

# litellm_config.yaml
model_list:
  - model_name: gpt-4o
    litellm_params:
      model: gpt-4o
      api_key: os.environ/OPENAI_API_KEY

  - model_name: claude-sonnet
    litellm_params:
      model: claude-sonnet-4-6
      api_key: os.environ/ANTHROPIC_API_KEY

router_settings:
  routing_strategy: cost-based-routing
  fallbacks:
    - {"gpt-4o": ["claude-sonnet", "gemini-pro"]}

general_settings:
  master_key: sk-my-master-key
  database_url: os.environ/DATABASE_URL

litellm --config litellm_config.yaml --port 4000

client → gateway → [openai | anthropic | local]
                 ↳ cache / rate-limit

# 프록시에 요청을 보내 어떤 모델이 실제로 응답했는지 확인
curl -s http://localhost:4000/v1/chat/completions \
  -H "Authorization: Bearer $LITELLM_KEY" -H "Content-Type: application/json" \
  -d '{"model":"default-chat","messages":[{"role":"user","content":"ping"}]}' | jq '.model'