paper_library.py 8.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258
  1. """论文库持久化管理:data/papers_library.json
  2. 职责:
  3. - 以 PDF 文件内容 SHA-256 作为 paper_id,跨路径/跨会话唯一标识论文
  4. - 跟踪每篇论文的预处理状态(解析/抽取/索引三步)
  5. - 提供增删查改与缓存状态查询
  6. - 删除论文时联动清理缓存文件与 Chroma chunks
  7. """
  8. from __future__ import annotations
  9. import hashlib
  10. import json
  11. import logging
  12. import threading
  13. from datetime import datetime
  14. from pathlib import Path
  15. from typing import Optional
  16. from backend.config import settings
  17. logger = logging.getLogger(__name__)
  18. # 合法的预处理状态
  19. _STATUSES = ("pending", "done", "failed")
  20. def compute_paper_id(file_path: str) -> str:
  21. """计算 PDF 文件内容的 SHA-256 作为 paper_id
  22. Args:
  23. file_path: PDF 文件路径
  24. Returns:
  25. 64 字符的十六进制 SHA-256 哈希
  26. """
  27. h = hashlib.sha256()
  28. with open(file_path, "rb") as f:
  29. for chunk in iter(lambda: f.read(8192), b""):
  30. h.update(chunk)
  31. return h.hexdigest()
  32. class PaperLibrary:
  33. """论文库持久化管理器
  34. 数据存储在 data/papers_library.json,线程安全(文件锁)。
  35. """
  36. def __init__(self, library_path: Optional[Path] = None) -> None:
  37. self._library_path = library_path or (settings.CACHE_DIR / "papers_library.json")
  38. self._library_path.parent.mkdir(parents=True, exist_ok=True)
  39. self._lock = threading.Lock()
  40. self._papers: dict[str, dict] = self._load()
  41. # ------------------------------------------------------------------ #
  42. # 持久化读写
  43. # ------------------------------------------------------------------ #
  44. def _load(self) -> dict[str, dict]:
  45. """从磁盘加载论文库"""
  46. if not self._library_path.exists():
  47. return {}
  48. try:
  49. with open(self._library_path, "r", encoding="utf-8") as f:
  50. data = json.load(f)
  51. if isinstance(data, dict) and isinstance(data.get("papers"), list):
  52. # 兼容 {"papers": [...]} 格式
  53. return {p["paper_id"]: p for p in data["papers"] if "paper_id" in p}
  54. if isinstance(data, dict):
  55. return data
  56. except (json.JSONDecodeError, OSError) as exc:
  57. logger.warning("加载论文库失败: %s", exc)
  58. return {}
  59. def _save(self) -> None:
  60. """保存论文库到磁盘"""
  61. data = {"papers": list(self._papers.values())}
  62. with open(self._library_path, "w", encoding="utf-8") as f:
  63. json.dump(data, f, ensure_ascii=False, indent=2, default=str)
  64. # ------------------------------------------------------------------ #
  65. # 增删查改
  66. # ------------------------------------------------------------------ #
  67. def register(
  68. self,
  69. paper_id: str,
  70. filename: str,
  71. source: str,
  72. original_path: str,
  73. ) -> dict:
  74. """注册论文到库中。若已存在则返回现有记录(不覆盖状态)。
  75. Args:
  76. paper_id: 内容哈希
  77. filename: 文件名
  78. source: "upload" 或 "folder"
  79. original_path: 存储路径(上传文件为 data/uploads/{paper_id}.pdf)
  80. Returns:
  81. 论文元数据 dict
  82. """
  83. with self._lock:
  84. if paper_id in self._papers:
  85. return self._papers[paper_id]
  86. entry = {
  87. "paper_id": paper_id,
  88. "filename": filename,
  89. "source": source,
  90. "original_path": original_path,
  91. "upload_time": datetime.now().isoformat(),
  92. "title": "",
  93. "parse_status": "pending",
  94. "extract_status": "pending",
  95. "index_status": "pending",
  96. "parsed_at": None,
  97. "extracted_at": None,
  98. "indexed_at": None,
  99. }
  100. self._papers[paper_id] = entry
  101. self._save()
  102. logger.info("注册论文: paper_id=%s, filename=%s", paper_id[:12], filename)
  103. return entry
  104. def get(self, paper_id: str) -> Optional[dict]:
  105. """获取单篇论文元数据,不存在返回 None"""
  106. with self._lock:
  107. return self._papers.get(paper_id)
  108. def list_all(self) -> list[dict]:
  109. """列出全部论文元数据"""
  110. with self._lock:
  111. return list(self._papers.values())
  112. def update_status(
  113. self,
  114. paper_id: str,
  115. title: Optional[str] = None,
  116. parse_status: Optional[str] = None,
  117. extract_status: Optional[str] = None,
  118. index_status: Optional[str] = None,
  119. ) -> Optional[dict]:
  120. """更新论文预处理状态
  121. Args:
  122. paper_id: 论文 ID
  123. title: 解析得到的标题(可选)
  124. parse_status: 解析状态
  125. extract_status: 抽取状态
  126. index_status: 索引状态
  127. Returns:
  128. 更新后的元数据 dict,或 None(论文不存在)
  129. """
  130. with self._lock:
  131. entry = self._papers.get(paper_id)
  132. if entry is None:
  133. return None
  134. now = datetime.now().isoformat()
  135. if title is not None and title:
  136. entry["title"] = title
  137. if parse_status is not None:
  138. if parse_status not in _STATUSES:
  139. raise ValueError(f"非法状态: {parse_status}")
  140. entry["parse_status"] = parse_status
  141. if parse_status == "done":
  142. entry["parsed_at"] = now
  143. if extract_status is not None:
  144. if extract_status not in _STATUSES:
  145. raise ValueError(f"非法状态: {extract_status}")
  146. entry["extract_status"] = extract_status
  147. if extract_status == "done":
  148. entry["extracted_at"] = now
  149. if index_status is not None:
  150. if index_status not in _STATUSES:
  151. raise ValueError(f"非法状态: {index_status}")
  152. entry["index_status"] = index_status
  153. if index_status == "done":
  154. entry["indexed_at"] = now
  155. self._save()
  156. return entry
  157. def delete(self, paper_id: str) -> bool:
  158. """删除论文:移除库记录 + 删除缓存文件 + 删除 Chroma chunks
  159. Args:
  160. paper_id: 论文 ID
  161. Returns:
  162. 是否删除成功
  163. """
  164. with self._lock:
  165. entry = self._papers.pop(paper_id, None)
  166. if entry is None:
  167. return False
  168. # 删除缓存文件(解析结果)
  169. parsed_cache = settings.CACHE_DIR / "parsed" / f"{paper_id}.json"
  170. if parsed_cache.exists():
  171. try:
  172. parsed_cache.unlink()
  173. except OSError as exc:
  174. logger.warning("删除解析缓存失败: %s", exc)
  175. # 删除缓存文件(抽取结果)
  176. extract_cache = settings.CACHE_DIR / "papers" / f"{paper_id}.json"
  177. if extract_cache.exists():
  178. try:
  179. extract_cache.unlink()
  180. except OSError as exc:
  181. logger.warning("删除抽取缓存失败: %s", exc)
  182. # 删除 Chroma chunks
  183. original_path = entry.get("original_path", "")
  184. if original_path:
  185. try:
  186. from backend.tools.build_vector_index import delete_paper_chunks
  187. delete_paper_chunks(original_path)
  188. except Exception as exc:
  189. logger.warning("删除 Chroma chunks 失败: %s", exc)
  190. # 删除上传的原文件(仅 upload 来源)
  191. if entry.get("source") == "upload":
  192. upload_file = Path(entry.get("original_path", ""))
  193. if upload_file.exists():
  194. try:
  195. upload_file.unlink()
  196. except OSError as exc:
  197. logger.warning("删除上传文件失败: %s", exc)
  198. with self._lock:
  199. self._save()
  200. logger.info("删除论文: paper_id=%s", paper_id[:12])
  201. return True
  202. def is_preprocessed(self, paper_id: str) -> bool:
  203. """检查论文是否已完成全部三步预处理"""
  204. entry = self.get(paper_id)
  205. if entry is None:
  206. return False
  207. return (
  208. entry.get("parse_status") == "done"
  209. and entry.get("extract_status") == "done"
  210. and entry.get("index_status") == "done"
  211. )
  212. # 模块级单例
  213. _library: Optional[PaperLibrary] = None
  214. def get_paper_library() -> PaperLibrary:
  215. """获取 PaperLibrary 单例"""
  216. global _library
  217. if _library is None:
  218. _library = PaperLibrary()
  219. return _library