AI 에이전트 신뢰성 3편: 프로덕션 모니터링과 SLA

from enum import Enum
from datetime import datetime, timedelta

class CircuitState(Enum):
    CLOSED = "closed"
    OPEN = "open"
    HALF_OPEN = "half_open"

class AgentCircuitBreaker:
    def __init__(self, failure_threshold=5, recovery_timeout=60):
        self.state = CircuitState.CLOSED
        self.failure_count = 0
        self.failure_threshold = failure_threshold
        self.recovery_timeout = recovery_timeout
        self.last_failure_time = None

    async def call(self, agent_fn, *args, **kwargs):
        if self.state == CircuitState.OPEN:
            if self._should_recover():
                self.state = CircuitState.HALF_OPEN
            else:
                raise Exception("Circuit breaker OPEN")
        try:
            result = await agent_fn(*args, **kwargs)
            self.failure_count = 0
            self.state = CircuitState.CLOSED
            return result
        except Exception:
            self.failure_count += 1
            self.last_failure_time = datetime.now()
            if self.failure_count >= self.failure_threshold:
                self.state = CircuitState.OPEN
                print(f"서킷 OPEN: {self.failure_count}회 연속 실패")
            raise

    def _should_recover(self):
        elapsed = datetime.now() - self.last_failure_time
        return elapsed > timedelta(seconds=self.recovery_timeout)

async def query_with_fallback(query: str) -> str:
    chain = [
        lambda q: claude_opus(q),      # 1순위: 최고 품질
        lambda q: claude_sonnet(q),    # 2순위: 빠른 모델
        lambda q: cached_response(q),  # 3순위: 캐시
        lambda q: rule_based(q),       # 4순위: 규칙 기반
    ]
    for i, fn in enumerate(chain):
        try:
            return await asyncio.wait_for(fn(query), timeout=10.0)
        except Exception as e:
            if i > 0:
                print(f"Fallback level {i} 사용")
    return "잠시 후 다시 시도해주세요."

class DegradedService:
    def __init__(self):
        self.features = {
            "tool_use": True,
            "memory": True,
            "personalization": True
        }

    async def handle(self, request):
        if not self.features["tool_use"]:
            return await self.knowledge_only(request)
        return await self.full_agent(request)

    def disable(self, feature: str):
        self.features[feature] = False
        print(f"기능 비활성화: {feature}")

MAX_STEPS = 15

async def safe_run(agent, query):
    steps = 0
    async for step in agent.stream(query):
        steps += 1
        if steps >= MAX_STEPS:
            return {
                "result": step.partial_result,
                "warning": f"최대 스텝({MAX_STEPS}) 도달 — 중간 결과 반환"
            }
    return step.final_result

trace: request_id → llm_calls → tool_calls → outcome

-- 최근 1시간 오류 유형별 건수
SELECT coalesce(provider_status::text, error_type) AS kind, count(*)
FROM agent_steps
WHERE created_at > now() - interval '1 hour' AND NOT success
GROUP BY 1 ORDER BY 2 DESC;