挑战项目 B · 评测驱动开发一个 Agent

   你会亲身经历这条真理：
   💀 "我觉得这个 prompt 更好" —— 跑评测，其实掉了 8 分
   💊 "这个改动没感觉" —— 跑评测，稳定涨了 12 分
   🔬 "改了之后总分没变" —— 分维度一看，简单题涨了、难题崩了

   没有评测，Agent 优化就是玄学。
   这个项目让你戒掉玄学，一辈子受用。

   输入："上个月华东区销售额超过10万的订单，按金额降序"
   输出：结构化查询 JSON
        {"region":"华东", "metric":"销售额", "op":">", "value":100000,
         "time":"上月", "sort":{"field":"金额","order":"desc"}}

import json

# ============ 评测任务 ============
TASKS = [
    {"id": "n1", "input": "上个月华东区销售额超过10万的订单，按金额降序",
     "reference": {"region":"华东","metric":"销售额","op":">","value":100000,
                   "time":"上月","sort":{"field":"金额","order":"desc"}},
     "type": "normal"},
    {"id": "r1", "input": "给我看看那些不太好的订单",   # 模糊，该拒绝
     "reference": {"action": "clarify", "reason": "'不太好'定义不明"},
     "type": "should_reject"},
    # ... 共 25 个
]

# ============ 代码评分器（支持部分分）============
def code_score(output, ref):
    try:
        out = json.loads(output)
    except json.JSONDecodeError:
        return 0.0, "JSON 不合法"
    if ref.get("action") == "clarify":            # 反例：本该拒绝
        return (1.0, "正确拒绝") if out.get("action") == "clarify" \
               else (0.0, "该拒绝却硬答了")
    fields = ["region", "metric", "op", "value", "time"]
    hit = sum(1 for f in fields if out.get(f) == ref.get(f))
    return hit / len(fields), f"{hit}/{len(fields)} 字段正确"

# ============ 模型评分器（结构化 rubric）============
RUBRIC = """给这个"自然语言转查询"的输出打分，四个维度各 0-25 分：
- 意图准确：查询意图理解对吗
- 字段完整：该有的字段都有吗
- 边界处理：相对时间/否定/多条件处理对吗
- 拒绝恰当：模糊查询是否恰当地要求澄清（不该拒绝的别拒绝）
输出 JSON：{"意图":分, "字段":分, "边界":分, "拒绝":分, "理由":"..."}"""

def model_score(judge_model, task, output):
    prompt = f"{RUBRIC}\n\n输入：{task['input']}\n参考解：{task['reference']}\n实际输出：{output}"
    return json.loads(judge_model.chat([{"role":"user","content":prompt}]))

# ============ 评测循环 ============
def evaluate(agent, judge, tasks, runs=1):
    results = []
    for task in tasks:
        for _ in range(runs):                     # runs>1 用于测 pass^k 一致性
            out = agent.run(task["input"])
            cs, note = code_score(out, task["reference"])
            ms = model_score(judge, task, out)
            results.append({"id": task["id"], "type": task["type"],
                            "code": cs, "model": sum(ms[k] for k in
                            ["意图","字段","边界","拒绝"])/100, "note": note})
    return results

def summarize(results):
    """⭐ 一定要分维度，不能只看总分"""
    from collections import defaultdict
    by_type = defaultdict(list)
    for r in results:
        by_type[r["type"]].append(r["code"])
    return {t: round(sum(v)/len(v), 3) for t, v in by_type.items()}
    # 你会看到 normal 涨的时候 should_reject 可能在跌——这就是单向优化陷阱