synthesize.py 7.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204
  1. """从种子批量生成 Harbor 任务,再逐个过门禁。
  2. 真实流水线里,“设计任务”和“写任务包”通常由 LLM 完成。这里用模板代替,
  3. 不需要 API key,重点放在门禁:坏任务必须被拦住。
  4. 用法:python3 synth/synthesize.py synth/seeds.json
  5. """
  6. import json
  7. import random
  8. import shutil
  9. import sys
  10. import tempfile
  11. from pathlib import Path
  12. HERE = Path(__file__).resolve().parent
  13. sys.path.insert(0, str(HERE.parent))
  14. from probe import run_trial # noqa: E402
  15. OUT = HERE / "generated"
  16. REQUIRED = ["instruction.md", "task.toml", "environment/Dockerfile", "solution/solve.sh", "tests/test.sh"]
  17. TOML = """schema_version = "1.4"
  18. artifacts = ["/app/stats.py"]
  19. [task]
  20. name = "hello-agents/stats-{id}"
  21. version = "1.0.0"
  22. [verifier]
  23. timeout_sec = 60.0
  24. environment_mode = "separate"
  25. [agent]
  26. timeout_sec = 300.0
  27. [environment]
  28. build_timeout_sec = 300.0
  29. cpus = 1
  30. memory_mb = 1024
  31. storage_mb = 2048
  32. """
  33. DOCKERFILE = "FROM python:3.12-slim\nWORKDIR /app\nCOPY stats.py orders.csv /app/\n"
  34. STATS = """import csv
  35. import json
  36. import sys
  37. MULTIPLIERS = {{"order": 1, "{event}": {multiplier}}}
  38. totals = {{}}
  39. with open(sys.argv[1], newline="") as f:
  40. for row in csv.DictReader(f):
  41. amount = float(row["amount"]) * MULTIPLIERS[row["type"]]
  42. totals[row["user"]] = totals.get(row["user"], 0) + amount
  43. print(json.dumps(totals, sort_keys=True))
  44. """
  45. INSTRUCTION = """`/app/stats.py` 读取一份订单 CSV,按用户统计消费总额,并把结果以 JSON 打印到标准输出。
  46. 用法:`python3 /app/stats.py <csv路径>`。CSV 的表头是 `user,type,amount`,示例数据在 `/app/orders.csv`。
  47. `type` 为 `{event}` 的行表示{meaning}。现在统计结果不对,请修复 `/app/stats.py`。
  48. """
  49. CHECK = """import json
  50. import math
  51. import subprocess
  52. import sys
  53. cases = json.load(open("/tests/cases.json"))
  54. passed = 0
  55. for case in cases:
  56. try:
  57. out = subprocess.run(
  58. [sys.executable, "-I", "/app/stats.py", f"/tests/{case['csv']}"],
  59. capture_output=True, text=True, timeout=10, cwd="/tmp",
  60. )
  61. actual = json.loads(out.stdout)
  62. ok = set(actual) == set(case["expected"]) and all(
  63. math.isclose(actual[k], v, abs_tol=1e-6) for k, v in case["expected"].items()
  64. )
  65. except Exception:
  66. ok = False
  67. print(f"{case['csv']}: {'PASS' if ok else 'FAIL'}")
  68. passed += ok
  69. open("/logs/verifier/reward.txt", "w").write(str(passed / len(cases)))
  70. """
  71. def make_rows(rng, event):
  72. rows = []
  73. for user in rng.sample(["ann", "ben", "cai", "dan", "eve"], 2):
  74. rows.append((user, "order", rng.randint(20, 200)))
  75. rows.append((user, event, rng.randint(1, 19)))
  76. return rows
  77. def gold(rows, event, multiplier):
  78. totals = {}
  79. for user, kind, amount in rows:
  80. totals[user] = totals.get(user, 0) + amount * (multiplier if kind == event else 1)
  81. return totals
  82. def to_csv(rows):
  83. return "user,type,amount\n" + "".join(f"{u},{k},{a}\n" for u, k, a in rows)
  84. def render_task(seed):
  85. """把种子渲染成任务目录。答案由程序算出,而不是由作者手写。"""
  86. event, multiplier = seed["event"], seed["multiplier"]
  87. rng = random.Random(seed["id"])
  88. task = OUT / seed["id"]
  89. shutil.rmtree(task, ignore_errors=True)
  90. for sub in ("environment", "solution", "tests"):
  91. (task / sub).mkdir(parents=True)
  92. public_rows = make_rows(rng, event)
  93. (task / "task.toml").write_text(TOML.format(id=seed["id"]))
  94. (task / "instruction.md").write_text(INSTRUCTION.format(event=event, meaning=seed["meaning"]))
  95. (task / "environment/Dockerfile").write_text(DOCKERFILE)
  96. (task / "environment/orders.csv").write_text(to_csv(public_rows))
  97. # 待修复的脚本把新事件当普通订单处理
  98. (task / "environment/stats.py").write_text(STATS.format(event=event, multiplier=1))
  99. fixed = STATS.format(event=event, multiplier=seed.get("solution_multiplier", multiplier))
  100. (task / "solution/solve.sh").write_text(f"#!/bin/bash\ncat > /app/stats.py <<'PY'\n{fixed}PY\n")
  101. (task / "solution/solve.sh").chmod(0o755)
  102. cases = []
  103. for i in range(3):
  104. rows = make_rows(rng, event)
  105. (task / f"tests/case{i}.csv").write_text(to_csv(rows))
  106. cases.append({"csv": f"case{i}.csv", "expected": gold(rows, event, multiplier)})
  107. (task / "tests/cases.json").write_text(json.dumps(cases))
  108. (task / "tests/check.py").write_text(CHECK)
  109. (task / "tests/test.sh").write_text("#!/bin/bash\npython3 /tests/check.py\n")
  110. (task / "tests/test.sh").chmod(0o755)
  111. return task, gold(public_rows, event, multiplier), cases
  112. def static_gate(task, cases):
  113. if not all((task / f).exists() for f in REQUIRED):
  114. return False
  115. # 隐藏用例的答案不能出现在 agent 看得到的文件里
  116. public = (task / "instruction.md").read_text() + (task / "environment/orders.csv").read_text()
  117. answers = {str(v) for case in cases for v in case["expected"].values()}
  118. return not any(answer in public for answer in answers)
  119. def shortcut_reward(task, public_answer):
  120. """取巧探针:直接打印示例数据的正确答案。"""
  121. with tempfile.TemporaryDirectory() as tmp:
  122. script = Path(tmp) / "solve.sh"
  123. script.write_text(
  124. f"#!/bin/bash\ncat > /app/stats.py <<'PY'\nprint({json.dumps(json.dumps(public_answer))})\nPY\n"
  125. )
  126. return run_trial(task, "oracle", script)
  127. def gate(seed, seen):
  128. """返回 (各门禁的展示值, 结论)。前一道没过,后面就不再跑。"""
  129. row = {"static": "-", "dedup": "-", "oracle": "-", "nop": "-", "shortcut": "-"}
  130. task, public_answer, cases = render_task(seed)
  131. row["static"] = "ok" if static_gate(task, cases) else "FAIL"
  132. if row["static"] == "FAIL":
  133. return row, "reject: leak or missing file"
  134. # 按决定任务的东西去重:换个说法、行为相同,仍然算同一道题
  135. key = seed["multiplier"]
  136. row["dedup"] = "dup" if key in seen else "ok"
  137. if key in seen:
  138. return row, f"reject: same behavior as {seen[key]}"
  139. seen[key] = seed["id"]
  140. oracle = run_trial(task, "oracle")
  141. row["oracle"] = "error" if oracle is None else f"{oracle:.2f}"
  142. if oracle != 1.0:
  143. return row, "reject: reference solution fails"
  144. nop = run_trial(task, "nop")
  145. row["nop"] = "error" if nop is None else f"{nop:.2f}"
  146. if nop != 0.0:
  147. return row, "reject: doing nothing scores"
  148. shortcut = shortcut_reward(task, public_answer)
  149. row["shortcut"] = "error" if shortcut is None else f"{shortcut:.2f}"
  150. if shortcut != 0.0:
  151. return row, "reject: hardcoded answer scores"
  152. return row, "accept"
  153. def main(seed_file):
  154. seeds = json.loads(Path(seed_file).read_text())
  155. seen = {}
  156. print(f"{'seed':<14}{'static':<8}{'dedup':<7}{'oracle':<8}{'nop':<6}{'shortcut':<10}decision")
  157. for seed in seeds:
  158. row, decision = gate(seed, seen)
  159. print(
  160. f"{seed['id']:<14}{row['static']:<8}{row['dedup']:<7}{row['oracle']:<8}"
  161. f"{row['nop']:<6}{row['shortcut']:<10}{decision}"
  162. )
  163. if __name__ == "__main__":
  164. main(sys.argv[1])