挑战项目 B · 评测驱动开发一个 Agent

import json

# ============ 评测任务 ============
TASKS = [
    {"id": "n1", "input": "上个月华东区销售额超过10万的订单，按金额降序",
     "reference": {"region":"华东","metric":"销售额","op":">","value":100000,
                   "time":"上月","sort":{"field":"金额","order":"desc"}},
     "type": "normal"},
    {"id": "r1", "input": "给我看看那些不太好的订单",   # 模糊，该拒绝
     "reference": {"action": "clarify", "reason": "'不太好'定义不明"},
     "type": "should_reject"},
    # ... 共 25 个
]

# ============ 代码评分器（支持部分分）============
def code_score(output, ref):
    try:
        out = json.loads(output)
    except json.JSONDecodeError:
        return 0.0, "JSON 不合法"
    if not isinstance(out, dict):
        return 0.0, "顶层必须是对象"
    if ref.get("action") == "clarify":            # 模糊任务应澄清
        return (1.0, "正确拒绝") if out.get("action") == "clarify" \
               else (0.0, "该拒绝却硬答了")
    if set(out) != set(ref):
        return 0.0, "字段缺失或包含未允许的字段"
    fields = ["region", "metric", "op", "value", "time", "sort"]
    hit = sum(1 for f in fields if out.get(f) == ref.get(f))
    return hit / len(fields), f"{hit}/{len(fields)} 字段正确"

# ============ 模型评分器（结构化 rubric）============
RUBRIC = """给这个"自然语言转查询"的输出打分，四个维度各 0-25 分：
- 意图准确：查询意图理解对吗
- 字段完整：该有的字段都有吗
- 边界处理：相对时间/否定/多条件处理对吗
- 拒绝恰当：模糊查询是否恰当地要求澄清（不该拒绝的别拒绝）
输出 JSON：{"意图":分, "字段":分, "边界":分, "拒绝":分, "理由":"..."}"""

def model_score(judge_model, task, output):
    prompt = f"{RUBRIC}\n\n输入：{task['input']}\n参考解：{task['reference']}\n实际输出：{output}"
    return json.loads(judge_model.chat([{"role":"user","content":prompt}]))

# ============ 评测循环 ============
def evaluate(agent, judge, tasks, runs=1):
    results = []
    for task in tasks:
        for _ in range(runs):                     # runs>1 用于测 pass^k 一致性
            out = agent.run(task["input"])
            cs, note = code_score(out, task["reference"])
            ms = model_score(judge, task, out)
            results.append({"id": task["id"], "type": task["type"],
                            "code": cs, "model": sum(ms[k] for k in
                            ["意图","字段","边界","拒绝"])/100, "note": note})
    return results

def summarize(results):
    """⭐ 一定要分维度，不能只看总分"""
    from collections import defaultdict
    by_type = defaultdict(list)
    for r in results:
        by_type[r["type"]].append(r["code"])
    return {t: round(sum(v)/len(v), 3) for t, v in by_type.items()}
    # 你会看到 normal 涨的时候 should_reject 可能在跌——这就是单向优化陷阱