1
0

probe.py 1.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354
  1. """用 oracle agent 跑不同的 solve.sh,检查 verifier 会不会判错。
  2. 用法:python3 probe.py <任务目录> [探针目录],探针目录默认是 probes/
  3. """
  4. import shutil
  5. import subprocess
  6. import sys
  7. import tempfile
  8. from pathlib import Path
  9. HERE = Path(__file__).parent
  10. def expectation(name):
  11. if name == "oracle" or name.startswith("variant"):
  12. return "== 1", lambda r: r == 1.0
  13. if name == "nop" or name.startswith("shortcut"):
  14. return "== 0", lambda r: r == 0.0
  15. return "< 1", lambda r: r < 1.0
  16. def run_trial(task_dir, agent, solve_script=None):
  17. with tempfile.TemporaryDirectory() as tmp:
  18. task = Path(tmp) / task_dir.name
  19. shutil.copytree(task_dir, task)
  20. if solve_script:
  21. shutil.copy(solve_script, task / "solution" / "solve.sh")
  22. jobs = Path(tmp) / "jobs"
  23. subprocess.run(
  24. ["harbor", "run", "-p", str(task), "-a", agent, "-o", str(jobs), "-y", "-q"],
  25. capture_output=True,
  26. )
  27. rewards = list(jobs.glob("*/*/verifier/reward.txt"))
  28. return float(rewards[0].read_text()) if rewards else None
  29. def main(task_dir, probe_dir=HERE / "probes"):
  30. probes = [("oracle", "oracle", None), ("nop", "nop", None)]
  31. probes += [(p.stem, "oracle", p) for p in sorted(Path(probe_dir).glob("*.sh"))]
  32. print(f"{'probe':<24}{'expect':<8}{'reward':<8}result")
  33. failures = 0
  34. for name, agent, script in probes:
  35. reward = run_trial(task_dir, agent, script)
  36. rule, ok = expectation(name)
  37. good = reward is not None and ok(reward)
  38. failures += not good
  39. shown = "error" if reward is None else f"{reward:.2f}"
  40. print(f"{name:<24}{rule:<8}{shown:<8}{'ok' if good else 'WRONG'}")
  41. print(f"\n{failures} probe(s) judged wrongly")
  42. return 1 if failures else 0
  43. if __name__ == "__main__":
  44. sys.exit(main(Path(sys.argv[1]).resolve(), *sys.argv[2:3]))