build_vector_index.py 6.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208
  1. """步骤3 索引构建工具:将解析后的论文加入混合检索向量库"""
  2. from __future__ import annotations
  3. import logging
  4. import time
  5. from backend.tools.base import Tool
  6. logger = logging.getLogger(__name__)
  7. # 模块级单例:懒加载,避免导入即触发重型依赖(Embedder/Chroma/BM25)
  8. _hybrid_searcher = None
  9. def _get_searcher():
  10. """懒加载模块级 HybridSearcher 单例。
  11. 供 build_vector_index 与 hybrid_search_tool 共享同一实例,
  12. 保证索引写入与检索读取状态一致。
  13. """
  14. global _hybrid_searcher
  15. if _hybrid_searcher is None:
  16. _t0 = time.perf_counter()
  17. from backend.rag.hybrid_search import HybridSearcher
  18. _hybrid_searcher = HybridSearcher()
  19. logger.info(
  20. "[_get_searcher] HybridSearcher 实例化完成: %.3fs(含 Embedder/ChromaStore/BM25Store 构造,未加载模型)",
  21. time.perf_counter() - _t0,
  22. )
  23. return _hybrid_searcher
  24. def _get_vector_store():
  25. """获取底层 ChromaStore(用于去重查询与删除)"""
  26. searcher = _get_searcher()
  27. return searcher.vector_store
  28. def is_indexed(paper_path: str) -> bool:
  29. """检查指定论文是否已索引(Chroma 中已有该 paper_path 的 chunks)
  30. Args:
  31. paper_path: 论文文件路径
  32. Returns:
  33. 是否已存在 chunks
  34. """
  35. try:
  36. _t0 = time.perf_counter()
  37. store = _get_vector_store()
  38. _t_get_store = time.perf_counter()
  39. store._ensure() # 首次调用会初始化 chromadb client
  40. _t_ensure = time.perf_counter()
  41. collection = store._collection
  42. result = collection.get(where={"paper_path": paper_path}, limit=1)
  43. _t_query = time.perf_counter()
  44. ids = result.get("ids", []) if result else []
  45. logger.info(
  46. "[is_indexed] %s: get_store=%.3fs ensure=%.3fs query=%.3fs total=%.3fs result=%s",
  47. paper_path, _t_get_store - _t0, _t_ensure - _t_get_store,
  48. _t_query - _t_ensure, _t_query - _t0, len(ids) > 0,
  49. )
  50. return len(ids) > 0
  51. except Exception as exc:
  52. logger.warning("查询索引状态失败: %s", exc)
  53. return False
  54. def delete_paper_chunks(paper_path: str) -> int:
  55. """删除指定论文在 Chroma 中的所有 chunks
  56. Args:
  57. paper_path: 论文文件路径
  58. Returns:
  59. 删除的 chunk 数量
  60. """
  61. try:
  62. store = _get_vector_store()
  63. store._ensure()
  64. collection = store._collection
  65. result = collection.get(where={"paper_path": paper_path})
  66. ids = result.get("ids", []) if result else []
  67. if ids:
  68. collection.delete(ids=ids)
  69. logger.info("已删除论文 %s 的 %d 个 chunks", paper_path, len(ids))
  70. return len(ids)
  71. return 0
  72. except Exception as exc:
  73. logger.warning("删除论文 chunks 失败: %s", exc)
  74. return 0
  75. async def build_vector_index(papers_json: list[dict]) -> dict:
  76. """将已解析论文列表加入向量库与 BM25 库。
  77. 对每篇论文先检查是否已索引,已索引则跳过(去重)。
  78. Args:
  79. papers_json: 已解析论文列表,每项含 path/title/sections
  80. Returns:
  81. {"indexed_count": N, "skipped_count": M, "status": "done"}
  82. """
  83. from backend.models.schemas import ParsedPaper, Section
  84. _t_fn_start = time.perf_counter()
  85. logger.info("[build_vector_index] 入口: %d papers", len(papers_json))
  86. _t0 = time.perf_counter()
  87. searcher = _get_searcher()
  88. logger.info("[build_vector_index] _get_searcher: %.3fs", time.perf_counter() - _t0)
  89. indexed = 0
  90. skipped = 0
  91. for paper_dict in papers_json:
  92. try:
  93. paper_path = paper_dict.get("path", "")
  94. _t_paper0 = time.perf_counter()
  95. # 去重:已索引的论文跳过
  96. if paper_path and is_indexed(paper_path):
  97. logger.info("论文 %s 已索引,跳过", paper_path)
  98. skipped += 1
  99. continue
  100. _t_dedup = time.perf_counter()
  101. sections = [
  102. Section(
  103. heading=s.get("heading", ""),
  104. content=s.get("content", ""),
  105. figure_captions=s.get("figure_captions", []),
  106. )
  107. for s in paper_dict.get("sections", [])
  108. ]
  109. paper = ParsedPaper(
  110. path=paper_path,
  111. title=paper_dict.get("title", ""),
  112. sections=sections,
  113. )
  114. _t_construct = time.perf_counter()
  115. logger.info(
  116. "[build_vector_index] %s: dedup_check=%.3fs paper_construct=%.3fs sections=%d",
  117. paper_path, _t_dedup - _t_paper0, _t_construct - _t_dedup, len(sections),
  118. )
  119. await searcher.async_add_paper(paper)
  120. _t_add = time.perf_counter()
  121. logger.info(
  122. "[build_vector_index] %s: async_add_paper=%.3fs (paper total=%.3fs)",
  123. paper_path, _t_add - _t_construct, _t_add - _t_paper0,
  124. )
  125. indexed += 1
  126. except Exception as exc:
  127. # 单篇失败隔离,继续处理其余论文
  128. logger.exception("[build_vector_index] 单篇索引失败: %s", exc)
  129. continue
  130. _t_done = time.perf_counter()
  131. logger.info(
  132. "[build_vector_index] 完成: indexed=%d skipped=%d total=%.3fs",
  133. indexed, skipped, _t_done - _t_fn_start,
  134. )
  135. return {"indexed_count": indexed, "skipped_count": skipped, "status": "done"}
  136. # ---------- Tool 定义 ----------
  137. _PARAMETERS = {
  138. "type": "object",
  139. "properties": {
  140. "papers_json": {
  141. "type": "array",
  142. "description": "已解析论文列表,每项含 path/title/sections",
  143. "items": {
  144. "type": "object",
  145. "properties": {
  146. "path": {"type": "string"},
  147. "title": {"type": "string"},
  148. "sections": {
  149. "type": "array",
  150. "items": {
  151. "type": "object",
  152. "properties": {
  153. "heading": {"type": "string"},
  154. "content": {"type": "string"},
  155. "figure_captions": {
  156. "type": "array",
  157. "items": {"type": "string"},
  158. },
  159. },
  160. },
  161. },
  162. },
  163. },
  164. }
  165. },
  166. "required": ["papers_json"],
  167. }
  168. build_vector_index_tool = Tool(
  169. name="build_vector_index",
  170. description=(
  171. "步骤3:将已解析论文列表加入向量库与 BM25 库,支持后续混合搜索。"
  172. "返回 {indexed_count, status}。"
  173. ),
  174. parameters=_PARAMETERS,
  175. func=build_vector_index,
  176. )