extract_schema.py 4.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124
  1. """步骤2 结构化抽取 schema 定义:字段列表、Function Calling schema、提示词构造"""
  2. from __future__ import annotations
  3. from typing import Any
  4. # 抽取的固定字段列表(顺序即重要性顺序,limitations 为重点字段)
  5. EXTRACT_FIELDS: list[str] = [
  6. "title",
  7. "task_problem",
  8. "method",
  9. "key_contributions",
  10. "datasets",
  11. "metrics",
  12. "results",
  13. "limitations",
  14. "future_work",
  15. ]
  16. # 各字段中文描述,用于 schema 与提示词
  17. FIELD_DESCRIPTIONS: dict[str, str] = {
  18. "title": "论文标题",
  19. "task_problem": "研究任务/问题",
  20. "method": "核心方法",
  21. "key_contributions": "主要贡献",
  22. "datasets": "使用的数据集",
  23. "metrics": "评估指标",
  24. "results": "实验结果",
  25. "limitations": "局限性(重点字段,须论文明确承认或可从结果推断,严禁臆造)",
  26. "future_work": "未来工作",
  27. }
  28. def _field_source_schema(description: str) -> dict[str, Any]:
  29. """构造单个 FieldSource 字段的 JSON schema"""
  30. return {
  31. "type": "object",
  32. "description": description,
  33. "properties": {
  34. "value": {
  35. "type": "string",
  36. "description": "字段值",
  37. },
  38. "source_sections": {
  39. "type": "array",
  40. "items": {"type": "string"},
  41. "description": "原文出处章节标题列表,用于回溯核查",
  42. },
  43. "confidence": {
  44. "type": "number",
  45. "description": "置信度 0.0-1.0,信息明确且原文直述取高值,推断取中值,不足取低值",
  46. },
  47. },
  48. "required": ["value", "source_sections", "confidence"],
  49. }
  50. # DeepSeek function calling schema:强制模型输出上述字段的 JSON
  51. EXTRACT_FUNCTION_SCHEMA: dict[str, Any] = {
  52. "type": "function",
  53. "function": {
  54. "name": "extract_paper_fields",
  55. "description": (
  56. "从论文中抽取结构化字段,每个字段含 value/source_sections/confidence。"
  57. "必须返回所有字段,limitations 须真实可溯源。"
  58. ),
  59. "parameters": {
  60. "type": "object",
  61. "properties": {
  62. field: _field_source_schema(desc)
  63. for field, desc in FIELD_DESCRIPTIONS.items()
  64. },
  65. "required": list(EXTRACT_FIELDS),
  66. },
  67. },
  68. }
  69. def build_extraction_prompt(parsed_paper: dict) -> str:
  70. """构造抽取提示词(中文)
  71. 强调三点:
  72. 1. source_sections 必须标注原文出处章节;
  73. 2. limitations 须真实,论文未明确承认时不得臆造;
  74. 3. 严格输出 JSON,不添加额外说明。
  75. """
  76. title = parsed_paper.get("title", "")
  77. sections = parsed_paper.get("sections", [])
  78. # 拼接章节正文,保留 heading 以便 source_sections 标注
  79. sections_text = ""
  80. for i, sec in enumerate(sections, 1):
  81. heading = sec.get("heading", f"章节{i}")
  82. content = sec.get("content", "")
  83. captions = sec.get("figure_captions", []) or []
  84. sections_text += f"\n\n## {heading}\n{content}"
  85. if captions:
  86. sections_text += "\n\n图表标题:\n" + "\n".join(f"- {c}" for c in captions)
  87. fields_list = "、".join(EXTRACT_FIELDS)
  88. return f"""请从以下论文中抽取结构化字段。
  89. 论文标题:{title}
  90. 论文正文:
  91. {sections_text}
  92. 抽取要求:
  93. 1. 必须抽取以下字段:{fields_list}
  94. 2. 每个字段必须是 FieldSource 结构:
  95. {{
  96. "value": "字段值(字符串,尽量简洁完整)",
  97. "source_sections": ["原文出处章节标题列表"],
  98. "confidence": 0.0-1.0
  99. }}
  100. 3. source_sections 必须标注字段信息来源于上面哪个章节(用章节标题原文),便于回溯核查;可标注多个章节。
  101. 4. limitations 字段特别要求:必须是论文明确承认的局限,或可从实验结果直接推断的局限,严禁臆造或泛泛而谈。若论文未明确提及局限,value 填写"论文未明确陈述局限",confidence 设为较低值(如 0.3)。
  102. 5. confidence 反映你对该字段抽取结果的可信度:
  103. - 信息明确且原文直述 → 0.9-1.0
  104. - 需要从原文推断 → 0.6-0.8
  105. - 信息不足或模糊 → 低于 0.6
  106. 6. 严格输出 JSON,不要添加任何额外说明文字或 markdown 代码块标记。
  107. """