Evaluating and Debugging LLM Agents: Strategies for Reliability in Non-Deterministic Systems

import time, json
from dataclasses import dataclass, field, asdict

@dataclass
class ToolCall:
    tool_name: str
    arguments: dict
    result: object
    duration_ms: float
    error: str = None

@dataclass
class AgentTrace:
    session_id: str
    user_input: str
    final_output: str
    tool_calls: list = field(default_factory=list)
    total_duration_ms: float = 0
    success: bool = True

class TracedAgent:
    def __init__(self, agent):
        self.agent = agent

    def run(self, user_input):
        session_id = f"trace_{int(time.time())}"
        start = time.time()
        trace = AgentTrace(session_id=session_id, user_input=user_input, final_output="")

        original_dispatch = self.agent.dispatch_tool

        def traced_dispatch(name, args):
            t0 = time.time()
            try:
                result = original_dispatch(name, args)
                trace.tool_calls.append(ToolCall(
                    tool_name=name, arguments=args, result=result,
                    duration_ms=(time.time() - t0) * 1000
                ))
                return result
            except Exception as e:
                trace.tool_calls.append(ToolCall(
                    tool_name=name, arguments=args, result=None,
                    duration_ms=(time.time() - t0) * 1000, error=str(e)
                ))
                raise

        self.agent.dispatch_tool = traced_dispatch

        try:
            trace.final_output = self.agent.run(user_input)
        except Exception as e:
            trace.success = False
            trace.final_output = str(e)

        trace.total_duration_ms = (time.time() - start) * 1000
        return trace

import pytest
from unittest.mock import patch

class TestToolSelection:
    def setup_method(self):
        self.agent = MyAgent()

    def test_stock_query_calls_stock_tool(self):
        """주가 관련 질문은 반드시 get_stock_price를 호출해야 함"""
        with patch.object(self.agent, 'get_stock_price', return_value={"price": 100}) as mock:
            self.agent.run("삼성전자 주가 알려줘")
            mock.assert_called_once()

    def test_no_tool_for_greeting(self):
        """단순 인사에는 툴을 호출하지 않아야 함"""
        with patch.object(self.agent, 'dispatch_tool') as mock:
            self.agent.run("안녕하세요")
            mock.assert_not_called()

    def test_parallel_calls_for_multi_query(self):
        """여러 종목 동시 조회는 병렬 호출해야 함"""
        with patch.object(self.agent, 'get_stock_price', return_value={"price": 100}) as mock:
            self.agent.run("삼성전자랑 SK하이닉스 주가 비교해줘")
            assert mock.call_count == 2

import anthropic, json

eval_client = anthropic.Anthropic()

EVAL_PROMPT = """다음 기준으로 에이전트 응답을 1-5점으로 평가하세요.

[질문] {question}
[에이전트 응답] {response}
[참고 정답] {ground_truth}

평가 기준:
- 정확성 (40%), 완결성 (30%), 간결성 (30%)

JSON으로만 반환: {{"score": 1-5, "reason": "한 줄 이유"}}"""

def evaluate_response(question, response, ground_truth):
    result = eval_client.messages.create(
        model="claude-sonnet-4-6",
        max_tokens=200,
        messages=[{
            "role": "user",
            "content": EVAL_PROMPT.format(
                question=question, response=response, ground_truth=ground_truth
            )
        }]
    )
    return json.loads(result.content[0].text)

def run_evaluation(test_cases):
    scores = []
    for case in test_cases:
        response = agent.run(case["question"])
        eval_result = evaluate_response(case["question"], response, case["expected"])
        scores.append(eval_result["score"])

    return {
        "avg_score": sum(scores) / len(scores),
        "pass_rate": sum(1 for s in scores if s >= 4) / len(scores)
    }

metrics = {
    "tool_call_rate":    "호출당 평균 툴 사용 횟수 — 급증 시 루프 의심",
    "tool_error_rate":   "툴 호출 실패율 — 외부 서비스 장애 탐지",
    "avg_latency_ms":    "평균 응답 시간 — 성능 저하 탐지",
    "fallback_rate":     "최종 답변 없이 종료된 비율 — 에이전트 미완성 탐지",
    "user_retry_rate":   "같은 질문 재시도율 — 만족도 프록시",
}

# 알람 조건
alerts = {
    "tool_call_rate > 10":   "루프 의심, 즉시 알람",
    "tool_error_rate > 5%":  "외부 서비스 점검",
    "avg_latency_ms > 10000": "SLA 위반 위험",
}