gate.py 2.8 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970
  1. """对 Agent 生成的任务跑门禁:静态检查、参考解、空解。
  2. 用法:python3 synth/gate.py <任务目录>
  3. 有没通过的检查时,会打印细节,可以存成文件交给 run_agent.sh 修改。
  4. """
  5. import re
  6. import subprocess
  7. import sys
  8. import tempfile
  9. from pathlib import Path
  10. def last_error(output):
  11. """从 Harbor 的报错栈里取出最后一条 XxxError: ... 信息。"""
  12. text = re.sub(r"[│╭╮╰╯─]", " ", output)
  13. text = re.sub(r"ExceptionGroup: unhandled errors in a TaskGroup \(\d+ sub-exceptions?\)", " ", text)
  14. text = " ".join(text.split())
  15. errors = re.findall(r"\b\w+(?:Error|Exception): .*?(?=\s\w+(?:Error|Exception): |$)", text)
  16. errors = [e for e in errors if not e.startswith("ExceptionGroup")]
  17. return errors[-1][:500] if errors else text[-500:]
  18. def run_trial(task, agent):
  19. """跑一次 trial,返回 (reward, 输出)。拿不到 reward 时,输出是 Harbor 的报错。"""
  20. with tempfile.TemporaryDirectory() as tmp:
  21. jobs = Path(tmp) / "jobs"
  22. proc = subprocess.run(
  23. ["harbor", "run", "-p", str(task), "-a", agent, "-o", str(jobs), "-y", "-q"],
  24. capture_output=True,
  25. text=True,
  26. )
  27. rewards = list(jobs.glob("*/*/verifier/reward.txt"))
  28. if not rewards:
  29. return None, last_error(proc.stdout + proc.stderr)
  30. stdout = list(jobs.glob("*/*/verifier/test-stdout.txt"))
  31. return float(rewards[0].read_text()), stdout[0].read_text() if stdout else ""
  32. def main(task_dir):
  33. task = Path(task_dir).resolve()
  34. rows, details = [], []
  35. lint = subprocess.run(["harbor", "lint", str(task)], capture_output=True, text=True)
  36. ok = lint.returncode == 0
  37. rows.append(("lint", "no error", "ok" if ok else "error", ok))
  38. if not ok:
  39. details.append("harbor lint 输出:\n" + (lint.stdout + lint.stderr).strip())
  40. for agent, rule, check in (("oracle", "== 1", lambda r: r == 1.0), ("nop", "== 0", lambda r: r == 0.0)):
  41. reward, stdout = run_trial(task, agent)
  42. if reward is None:
  43. # 没拿到 reward 是运行出错,不是任务本身没通过,要先排查环境
  44. rows.append((agent, rule, "infra", False))
  45. details.append(f"{agent} 没有拿到 reward,Harbor 输出:\n{stdout}")
  46. continue
  47. ok = check(reward)
  48. rows.append((agent, rule, f"{reward:.2f}", ok))
  49. if not ok:
  50. details.append(f"{agent} 得分 {reward:.2f},判分脚本输出:\n{stdout.strip()}")
  51. print(f"{'gate':<8}{'expect':<10}{'result':<8}status")
  52. for name, rule, shown, ok in rows:
  53. print(f"{name:<8}{rule:<10}{shown:<8}{'ok' if ok else 'FAIL'}")
  54. for d in details:
  55. print("\n" + d)
  56. return 0 if all(r[3] for r in rows) else 1
  57. if __name__ == "__main__":
  58. sys.exit(main(sys.argv[1]))