paper_library.py 6.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175
  1. """论文库服务:包装官方 RAGTool + MemoryTool + NoteTool
  2. PDF → MarkItDown → Markdown → 智能分块 → 向量化 → Qdrant 存储
  3. """
  4. from __future__ import annotations
  5. import hashlib
  6. import json
  7. import logging
  8. from datetime import datetime
  9. from pathlib import Path
  10. from typing import Any, Dict, Optional
  11. from hello_agents.tools import RAGTool
  12. logger = logging.getLogger(__name__)
  13. class PaperLibraryService:
  14. """论文库:管理 PDF 的 RAG 索引与检索"""
  15. def __init__(
  16. self,
  17. library_path: str = "./workspace/paper_library",
  18. rag_namespace: Optional[str] = None,
  19. pdf_dir: Optional[str] = None,
  20. ):
  21. self.library_path = Path(library_path)
  22. self.library_path.mkdir(parents=True, exist_ok=True)
  23. self.pdf_dir = Path(pdf_dir) if pdf_dir else self.library_path / "pdfs"
  24. self.pdf_dir.mkdir(parents=True, exist_ok=True)
  25. self.rag_dir = self.library_path / "rag"
  26. self.rag_dir.mkdir(parents=True, exist_ok=True)
  27. if not rag_namespace:
  28. h = hashlib.md5(str(self.library_path.resolve()).encode()).hexdigest()[:8]
  29. rag_namespace = f"papers_{h}"
  30. self.rag_namespace = rag_namespace
  31. # 参照第八章使用 RAGTool
  32. try:
  33. self.rag_tool = RAGTool(
  34. knowledge_base_path=str(self.rag_dir),
  35. collection_name="paper_library",
  36. rag_namespace=rag_namespace,
  37. )
  38. logger.info("PaperLibrary RAGTool 就绪: namespace=%s", rag_namespace)
  39. except Exception as exc:
  40. logger.exception("RAGTool 初始化失败: %s", exc)
  41. self.rag_tool = None
  42. self._manifest_path = self.rag_dir / "index_manifest.json"
  43. self._manifest: Dict[str, Dict[str, Any]] = self._load_manifest()
  44. # ------------------------------------------------------------------
  45. def index_paper(
  46. self,
  47. pdf_path: str,
  48. paper_id: str,
  49. metadata: Optional[Dict[str, Any]] = None,
  50. chunk_size: int = 1000,
  51. chunk_overlap: int = 200,
  52. ) -> Dict[str, Any]:
  53. """将下载的 PDF 加入 RAG 索引(内部走 MarkItDown → Markdown → 分块 → 向量化)"""
  54. if not self.rag_tool:
  55. return {"success": False, "error": "RAGTool 未初始化"}
  56. if paper_id in self._manifest:
  57. return {"success": True, "skipped": True, "paper_id": paper_id,
  58. "chunks": self._manifest[paper_id].get("chunks", 0)}
  59. path = Path(pdf_path)
  60. if not path.exists():
  61. return {"success": False, "error": f"PDF 不存在: {pdf_path}"}
  62. try:
  63. # 参照第八章 8.4.2 的调用方式
  64. result = self.rag_tool.execute(
  65. "add_document",
  66. file_path=str(path),
  67. document_id=paper_id,
  68. chunk_size=chunk_size,
  69. chunk_overlap=chunk_overlap,
  70. )
  71. if isinstance(result, dict):
  72. success = result.get("success", True)
  73. chunks = result.get("chunks") or result.get("chunk_count") or 0
  74. else:
  75. success = True
  76. chunks = 0
  77. if success:
  78. self._manifest[paper_id] = {
  79. "pdf_path": str(path),
  80. "chunks": chunks,
  81. "indexed_at": datetime.now().isoformat(),
  82. "metadata": metadata or {},
  83. }
  84. self._save_manifest()
  85. logger.info("索引 %s 成功 (%d chunks)", paper_id, chunks)
  86. return {"success": success, "paper_id": paper_id, "chunks": chunks}
  87. except Exception as exc:
  88. logger.exception("索引 %s 失败", paper_id)
  89. return {"success": False, "error": str(exc)}
  90. # ------------------------------------------------------------------
  91. def search(
  92. self,
  93. query: str,
  94. limit: int = 5,
  95. min_score: float = 0.1,
  96. enable_advanced: bool = False,
  97. ) -> str:
  98. """检索论文库(参照第八章 8.4.3 的 RAGTool.execute("search") 方式)"""
  99. if not self.rag_tool:
  100. return "❌ RAGTool 未初始化"
  101. try:
  102. kwargs: Dict[str, Any] = {
  103. "query": query, "limit": limit, "min_score": min_score,
  104. }
  105. if enable_advanced:
  106. kwargs.update({
  107. "enable_advanced_search": True,
  108. "enable_mqe": True,
  109. "enable_hyde": True,
  110. })
  111. result = self.rag_tool.execute("search", **kwargs)
  112. return str(result) if not isinstance(result, str) else result
  113. except Exception as exc:
  114. logger.exception("论文库检索失败")
  115. return f"❌ 检索失败: {exc}"
  116. def ask(self, question: str, limit: int = 5) -> str:
  117. """基于论文库直接回答(RAG 增强生成,参照第八章 8.4.3)"""
  118. if not self.rag_tool:
  119. return "❌ RAGTool 未初始化"
  120. try:
  121. result = self.rag_tool.execute("ask", question=question, limit=limit)
  122. return str(result) if not isinstance(result, str) else result
  123. except Exception as exc:
  124. return f"❌ 问答失败: {exc}"
  125. def stats(self) -> Dict[str, Any]:
  126. return {
  127. "library_path": str(self.library_path),
  128. "rag_namespace": self.rag_namespace,
  129. "indexed_papers": len(self._manifest),
  130. "papers": [
  131. {"paper_id": pid, "chunks": info.get("chunks", 0),
  132. "indexed_at": info.get("indexed_at")}
  133. for pid, info in self._manifest.items()
  134. ],
  135. }
  136. # ------------------------------------------------------------------
  137. def _load_manifest(self) -> Dict[str, Dict[str, Any]]:
  138. if self._manifest_path.exists():
  139. try:
  140. with open(self._manifest_path, "r", encoding="utf-8") as f:
  141. return json.load(f)
  142. except Exception as exc:
  143. logger.warning("manifest 读取失败: %s", exc)
  144. return {}
  145. def _save_manifest(self) -> None:
  146. try:
  147. with open(self._manifest_path, "w", encoding="utf-8") as f:
  148. json.dump(self._manifest, f, ensure_ascii=False, indent=2)
  149. except Exception as exc:
  150. logger.warning("manifest 写入失败: %s", exc)