extract_paper_structure.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483
  1. """步骤2 单篇论文结构化抽取工具:核心主干
  2. 职责:
  3. - 调用 DeepSeek 通过 function calling 强制 JSON 输出,抽取固定 schema 字段;
  4. - 长论文按章节分块抽取后合并去重;
  5. - 字段完整性校验 + 针对性补抽(只补缺失字段,不重跑整篇);
  6. - limitations 字段单独强调须真实可溯源;
  7. - 自校验输出各字段置信度,低于阈值标记需复核;
  8. - 结果独立缓存(每篇一个 JSON + 追加 papers.jsonl),失败可单独重跑。
  9. """
  10. from __future__ import annotations
  11. import hashlib
  12. import json
  13. import logging
  14. from pathlib import Path
  15. from typing import Any, Optional
  16. from backend.config import settings
  17. from backend.llm.client import DeepSeekClient
  18. from backend.llm.json_utils import parse_json_safe, validate_required_fields
  19. from backend.tools.base import Tool
  20. from backend.tools.extract_schema import (
  21. EXTRACT_FIELDS,
  22. EXTRACT_FUNCTION_SCHEMA,
  23. build_extraction_prompt,
  24. )
  25. logger = logging.getLogger(__name__)
  26. # 长论文分块阈值(章节总字符数超过此值则分块抽取)
  27. CHUNK_CHAR_THRESHOLD = 20000
  28. # 模块级懒加载单例客户端
  29. _client: Optional[DeepSeekClient] = None
  30. def _get_client() -> DeepSeekClient:
  31. """懒加载 DeepSeekClient 单例,首次调用时创建"""
  32. global _client
  33. if _client is None:
  34. _client = DeepSeekClient()
  35. return _client
  36. # --------------------------------------------------------------------------- #
  37. # 内部辅助函数
  38. # --------------------------------------------------------------------------- #
  39. def _normalize_field_source(val: Any) -> Optional[dict]:
  40. """将任意值归一化为 FieldSource dict {value, source_sections, confidence}
  41. 支持三种输入:
  42. - 字符串:包装为低置信度 FieldSource;
  43. - dict:补全缺失键、裁剪 confidence 到 [0,1];
  44. - 其他:返回 None。
  45. """
  46. if val is None:
  47. return None
  48. if isinstance(val, str):
  49. if not val.strip():
  50. return None
  51. return {"value": val, "source_sections": [], "confidence": 0.5}
  52. if isinstance(val, dict):
  53. value = val.get("value", "")
  54. if value is None:
  55. value = ""
  56. value = str(value).strip()
  57. if not value:
  58. return None
  59. source = val.get("source_sections", [])
  60. if isinstance(source, str):
  61. source = [source]
  62. elif not isinstance(source, list):
  63. source = []
  64. else:
  65. source = [str(s) for s in source]
  66. try:
  67. conf = float(val.get("confidence", 0.8))
  68. except (TypeError, ValueError):
  69. conf = 0.8
  70. conf = max(0.0, min(1.0, conf))
  71. return {"value": value, "source_sections": source, "confidence": conf}
  72. return None
  73. def _normalize_record(data: Any) -> dict:
  74. """将模型输出的 dict 归一化为标准 record(所有字段为 FieldSource dict)"""
  75. if not isinstance(data, dict):
  76. return {}
  77. record: dict[str, Any] = {}
  78. for field in EXTRACT_FIELDS:
  79. fs = _normalize_field_source(data.get(field))
  80. if fs is not None:
  81. record[field] = fs
  82. return record
  83. def _chunk_sections(
  84. sections: list[dict], max_chars: int = CHUNK_CHAR_THRESHOLD
  85. ) -> list[list[dict]]:
  86. """按章节分块,每块总字符数不超过 max_chars(不拆分单个章节)"""
  87. chunks: list[list[dict]] = []
  88. current: list[dict] = []
  89. current_chars = 0
  90. for sec in sections:
  91. sec_chars = len(sec.get("content", ""))
  92. # 当前块非空且加入后会超限 → 先落盘当前块
  93. if current and current_chars + sec_chars > max_chars:
  94. chunks.append(current)
  95. current = []
  96. current_chars = 0
  97. current.append(sec)
  98. current_chars += sec_chars
  99. if current:
  100. chunks.append(current)
  101. return chunks
  102. def _merge_records(records: list[dict]) -> dict:
  103. """合并多次抽取结果:每字段取置信度最高的 value,source_sections 去重合并"""
  104. merged: dict[str, Any] = {}
  105. for field in EXTRACT_FIELDS:
  106. candidates: list[dict] = []
  107. for r in records:
  108. fs = r.get(field)
  109. if fs and isinstance(fs, dict) and fs.get("value"):
  110. candidates.append(fs)
  111. if not candidates:
  112. continue
  113. # 取置信度最高的候选作为 value
  114. best = max(candidates, key=lambda x: x.get("confidence", 0.0))
  115. # 合并所有 source_sections 并去重(保序)
  116. all_sources: list[str] = []
  117. seen: set[str] = set()
  118. for c in candidates:
  119. for s in c.get("source_sections", []):
  120. if s not in seen:
  121. seen.add(s)
  122. all_sources.append(s)
  123. merged[field] = {
  124. "value": best["value"],
  125. "source_sections": all_sources,
  126. "confidence": best.get("confidence", 0.8),
  127. }
  128. return merged
  129. async def _call_extract(
  130. client: DeepSeekClient, parsed_paper: dict, focus_fields: Optional[list[str]] = None
  131. ) -> dict:
  132. """单次调用模型抽取字段,返回归一化后的 record dict
  133. Args:
  134. client: DeepSeek 客户端
  135. parsed_paper: 已解析论文 dict
  136. focus_fields: 若指定,则只补抽这些缺失字段(针对性补抽)
  137. """
  138. prompt = build_extraction_prompt(parsed_paper)
  139. if focus_fields:
  140. prompt = (
  141. f"前一次抽取缺失了以下字段:{', '.join(focus_fields)}。\n"
  142. f"请重点从原文中补抽这些字段,其他字段可留空。\n\n{prompt}"
  143. )
  144. messages = [
  145. {
  146. "role": "system",
  147. "content": (
  148. "你是论文结构化抽取助手。严格通过函数调用输出 JSON,"
  149. "每个字段含 value/source_sections/confidence。"
  150. "limitations 必须真实可溯源,严禁臆造。"
  151. ),
  152. },
  153. {"role": "user", "content": prompt},
  154. ]
  155. # 强制调用指定函数以获得结构化 JSON 输出
  156. response = await client.chat(
  157. messages=messages,
  158. tools=[EXTRACT_FUNCTION_SCHEMA],
  159. tool_choice={
  160. "type": "function",
  161. "function": {"name": "extract_paper_fields"},
  162. },
  163. )
  164. msg = response.choices[0].message
  165. data: Any = None
  166. # 优先从 tool_calls 提取参数
  167. tool_calls = getattr(msg, "tool_calls", None)
  168. if tool_calls:
  169. try:
  170. data = parse_json_safe(tool_calls[0].function.arguments)
  171. except Exception:
  172. data = None
  173. # 降级:从 content 解析
  174. if data is None and msg.content:
  175. data = parse_json_safe(msg.content)
  176. if data is None:
  177. logger.warning("模型未返回可解析的 JSON,返回空 record")
  178. return {}
  179. return _normalize_record(data)
  180. # --------------------------------------------------------------------------- #
  181. # 核心抽取函数
  182. # --------------------------------------------------------------------------- #
  183. async def extract_paper_structure(paper_path: str, parsed_paper: dict) -> dict:
  184. """从已解析论文中抽取结构化字段
  185. Args:
  186. paper_path: 论文文件路径
  187. parsed_paper: 已解析论文 dict,含 path/title/sections:[{heading,content,figure_captions}]
  188. Returns:
  189. dict,可序列化为 PaperRecord。所有 EXTRACT_FIELDS 字段为 FieldSource dict。
  190. """
  191. client = _get_client()
  192. sections = parsed_paper.get("sections", [])
  193. total_chars = sum(len(s.get("content", "")) for s in sections)
  194. if total_chars > CHUNK_CHAR_THRESHOLD:
  195. # 长论文:按章节分块抽取后合并去重
  196. logger.info(
  197. "论文 %s 总字符数 %d 超过阈值 %d,按章节分块抽取",
  198. paper_path,
  199. total_chars,
  200. CHUNK_CHAR_THRESHOLD,
  201. )
  202. chunks = _chunk_sections(sections, CHUNK_CHAR_THRESHOLD)
  203. chunk_records: list[dict] = []
  204. for idx, chunk in enumerate(chunks):
  205. chunk_paper = {**parsed_paper, "sections": chunk}
  206. logger.debug("抽取分块 %d/%d(%d 章节)", idx + 1, len(chunks), len(chunk))
  207. rec = await _call_extract(client, chunk_paper)
  208. if rec:
  209. chunk_records.append(rec)
  210. record = _merge_records(chunk_records) if chunk_records else {}
  211. else:
  212. # 短论文:一次性抽取
  213. record = await _call_extract(client, parsed_paper)
  214. # 字段完整性校验:检查必填字段是否齐全
  215. missing = validate_required_fields(record, EXTRACT_FIELDS)
  216. if missing:
  217. logger.info("首次抽取缺失字段 %s,触发针对性补抽", missing)
  218. supplement = await _call_extract(client, parsed_paper, focus_fields=missing)
  219. for field in missing:
  220. fs = supplement.get(field)
  221. if fs:
  222. record[field] = fs
  223. # 确保 path 字段(PaperRecord 顶层字段,非 FieldSource)
  224. record["path"] = paper_path
  225. # 若 title 缺失,用 parsed_paper 的 title 兜底
  226. if "title" not in record or not record["title"].get("value"):
  227. fallback_title = parsed_paper.get("title", "")
  228. if fallback_title:
  229. record["title"] = {
  230. "value": fallback_title,
  231. "source_sections": [],
  232. "confidence": 0.6,
  233. }
  234. return record
  235. # --------------------------------------------------------------------------- #
  236. # 自校验函数
  237. # --------------------------------------------------------------------------- #
  238. async def self_check(
  239. record: dict, parsed_paper: dict, client: DeepSeekClient
  240. ) -> dict:
  241. """让模型对照原文检查各字段,输出每字段置信度
  242. Args:
  243. record: 抽取结果 dict
  244. parsed_paper: 已解析论文 dict
  245. client: DeepSeek 客户端(由调用方传入)
  246. Returns:
  247. {
  248. "record": record,
  249. "field_confidences": {字段名: 置信度},
  250. "overall_confidence": float,
  251. "needs_review": [低于阈值的字段名]
  252. }
  253. """
  254. # 拼接原文节选(截断避免超长)
  255. sections = parsed_paper.get("sections", [])
  256. sections_text = "\n\n".join(
  257. f"## {s.get('heading', '')}\n{s.get('content', '')[:3000]}"
  258. for s in sections
  259. )[:15000]
  260. record_text = json.dumps(record, ensure_ascii=False, indent=2, default=str)
  261. fields_list = "、".join(EXTRACT_FIELDS)
  262. prompt = f"""请对照论文原文,检查以下结构化抽取记录的每个字段是否准确。
  263. 论文原文(节选):
  264. {sections_text}
  265. 抽取记录:
  266. {record_text}
  267. 检查要求:
  268. 1. 对以下每个字段输出 0-1 的置信度:{fields_list}
  269. 2. 输出整体置信度 overall_confidence(所有字段的加权平均,limitations 权重更高)。
  270. 3. 严格以 JSON 输出,格式如下:
  271. {{
  272. "field_confidences": {{
  273. "title": 0.95,
  274. "task_problem": 0.9,
  275. "method": 0.85,
  276. "key_contributions": 0.8,
  277. "datasets": 0.9,
  278. "metrics": 0.85,
  279. "results": 0.8,
  280. "limitations": 0.7,
  281. "future_work": 0.6
  282. }},
  283. "overall_confidence": 0.82,
  284. "issues": ["limitations 字段描述过于模糊,原文未明确提及"]
  285. }}
  286. """
  287. messages = [
  288. {"role": "system", "content": "你是论文抽取质量审核员,负责对照原文核查字段准确性。"},
  289. {"role": "user", "content": prompt},
  290. ]
  291. response = await client.chat(messages=messages)
  292. content = response.choices[0].message.content or ""
  293. result = parse_json_safe(content)
  294. if not isinstance(result, dict):
  295. logger.warning("自校验未返回有效 JSON,降级为空结果")
  296. result = {}
  297. field_confidences: dict[str, float] = {}
  298. raw_confidences = result.get("field_confidences", {})
  299. if isinstance(raw_confidences, dict):
  300. for f in EXTRACT_FIELDS:
  301. try:
  302. field_confidences[f] = float(raw_confidences.get(f, 0.0))
  303. except (TypeError, ValueError):
  304. field_confidences[f] = 0.0
  305. try:
  306. overall = float(result.get("overall_confidence", 0.0))
  307. except (TypeError, ValueError):
  308. overall = 0.0
  309. overall = max(0.0, min(1.0, overall))
  310. # 低于阈值的字段标记需复核
  311. threshold = settings.CONFIDENCE_THRESHOLD
  312. needs_review = [
  313. f for f in EXTRACT_FIELDS if field_confidences.get(f, 0.0) < threshold
  314. ]
  315. return {
  316. "record": record,
  317. "field_confidences": field_confidences,
  318. "overall_confidence": overall,
  319. "needs_review": needs_review,
  320. "issues": result.get("issues", []),
  321. }
  322. # --------------------------------------------------------------------------- #
  323. # 缓存函数
  324. # --------------------------------------------------------------------------- #
  325. def _resolve_cache_dir(cache_dir: Optional[str] = None) -> Path:
  326. """解析缓存目录,默认使用 settings.CACHE_DIR"""
  327. if cache_dir:
  328. p = Path(cache_dir)
  329. else:
  330. p = settings.CACHE_DIR
  331. p.mkdir(parents=True, exist_ok=True)
  332. return p
  333. def _paper_hash(paper_path: str) -> str:
  334. """以论文路径生成 hash,作为缓存文件名(向后兼容旧缓存)"""
  335. return hashlib.md5(paper_path.encode("utf-8")).hexdigest()[:16]
  336. def _cache_file_path(paper_id: Optional[str], paper_path: str, cache_dir: Optional[Path] = None) -> Path:
  337. """解析缓存文件路径,优先用 paper_id,否则回退到路径哈希"""
  338. base = cache_dir or _resolve_cache_dir()
  339. papers_dir = base / "papers"
  340. papers_dir.mkdir(parents=True, exist_ok=True)
  341. key = paper_id if paper_id else _paper_hash(paper_path)
  342. return papers_dir / f"{key}.json"
  343. def save_record(record: dict, cache_dir: Optional[str] = None, paper_id: Optional[str] = None) -> Path:
  344. """保存单篇抽取结果到独立 JSON 文件,并追加写入 papers.jsonl
  345. Args:
  346. record: 抽取结果 dict
  347. cache_dir: 缓存目录,默认 settings.CACHE_DIR
  348. paper_id: 论文内容哈希,优先用作缓存键
  349. Returns:
  350. 独立 JSON 文件路径
  351. """
  352. base = _resolve_cache_dir(cache_dir)
  353. paper_path = record.get("path", "")
  354. file_path = _cache_file_path(paper_id, paper_path, base)
  355. # 独立 JSON 文件(可单独重跑)
  356. with open(file_path, "w", encoding="utf-8") as f:
  357. json.dump(record, f, ensure_ascii=False, indent=2, default=str)
  358. # 追加写入 papers.jsonl(一行一篇)
  359. jsonl_path = base / "papers.jsonl"
  360. with open(jsonl_path, "a", encoding="utf-8") as f:
  361. f.write(json.dumps(record, ensure_ascii=False, default=str) + "\n")
  362. logger.info("抽取结果已缓存至 %s", file_path)
  363. return file_path
  364. def load_record(paper_path: str, cache_dir: Optional[str] = None, paper_id: Optional[str] = None) -> Optional[dict]:
  365. """从缓存加载单篇抽取结果,不存在返回 None
  366. Args:
  367. paper_path: 论文文件路径
  368. cache_dir: 缓存目录,默认 settings.CACHE_DIR
  369. paper_id: 论文内容哈希,优先用作缓存键
  370. Returns:
  371. 抽取结果 dict,或 None
  372. """
  373. base = _resolve_cache_dir(cache_dir)
  374. file_path = _cache_file_path(paper_id, paper_path, base)
  375. if not file_path.exists():
  376. return None
  377. try:
  378. with open(file_path, "r", encoding="utf-8") as f:
  379. return json.load(f)
  380. except (json.JSONDecodeError, OSError) as exc:
  381. logger.warning("读取缓存 %s 失败: %s", file_path, exc)
  382. return None
  383. # --------------------------------------------------------------------------- #
  384. # 工具定义
  385. # --------------------------------------------------------------------------- #
  386. _PARAMETERS_SCHEMA: dict = {
  387. "type": "object",
  388. "properties": {
  389. "paper_path": {"type": "string"},
  390. "parsed_paper": {
  391. "type": "object",
  392. "description": "已解析论文,含 path/title/sections",
  393. },
  394. },
  395. "required": ["paper_path", "parsed_paper"],
  396. }
  397. extract_paper_structure_tool = Tool(
  398. name="extract_paper_structure",
  399. description=(
  400. "从已解析论文中抽取结构化字段"
  401. "(title/task_problem/method/key_contributions/datasets/metrics/"
  402. "results/limitations/future_work),"
  403. "每个字段含来源章节与置信度。长论文自动分块抽取后合并。"
  404. ),
  405. parameters=_PARAMETERS_SCHEMA,
  406. func=extract_paper_structure,
  407. )