hybrid_search.py 6.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197
  1. """混合搜索:组合向量检索 + BM25 检索,加权分数归一化重排"""
  2. from __future__ import annotations
  3. import asyncio
  4. import logging
  5. import time
  6. from typing import Optional
  7. from backend.models.schemas import ParsedPaper, Section
  8. from backend.rag.bm25_store import BM25Store
  9. from backend.rag.embedding import Embedder, get_default_embedder
  10. from backend.rag.vector_store import ChromaStore
  11. logger = logging.getLogger(__name__)
  12. # 分块目标字符数
  13. CHUNK_SIZE = 500
  14. class HybridSearcher:
  15. """混合检索器:ChromaStore + BM25Store + Embedder"""
  16. def __init__(
  17. self,
  18. vector_store: Optional[ChromaStore] = None,
  19. bm25_store: Optional[BM25Store] = None,
  20. embedder: Optional[Embedder] = None,
  21. chunk_size: int = CHUNK_SIZE,
  22. ):
  23. self.embedder = embedder or get_default_embedder()
  24. self.vector_store = vector_store or ChromaStore(embedder=self.embedder)
  25. self.bm25_store = bm25_store or BM25Store()
  26. self.chunk_size = chunk_size
  27. # ---------- 分块 ----------
  28. def _chunk_section(self, section: Section) -> list[dict]:
  29. """将单章节按段落分块,块大小约 chunk_size 字符。
  30. Returns:
  31. list[dict],每项 {section, text}
  32. """
  33. content = section.content or ""
  34. heading = section.heading or ""
  35. # 按换行切段落,丢弃空段
  36. paragraphs = [p.strip() for p in content.split("\n") if p.strip()]
  37. chunks: list[dict] = []
  38. buf = ""
  39. for para in paragraphs:
  40. # 累计超过阈值则落块
  41. if buf and len(buf) + len(para) > self.chunk_size:
  42. chunks.append({"section": heading, "text": buf})
  43. buf = para
  44. else:
  45. buf = f"{buf}\n{para}".strip() if buf else para
  46. if buf:
  47. chunks.append({"section": heading, "text": buf})
  48. # 处理单段过长的情况:按 chunk_size 强制切分
  49. final: list[dict] = []
  50. for c in chunks:
  51. text = c["text"]
  52. if len(text) <= self.chunk_size * 2:
  53. final.append(c)
  54. else:
  55. for i in range(0, len(text), self.chunk_size):
  56. final.append(
  57. {"section": c["section"], "text": text[i : i + self.chunk_size]}
  58. )
  59. return final
  60. # ---------- 写入 ----------
  61. def add_paper(self, paper: ParsedPaper):
  62. """把单篇论文章节分块后加入向量库与 BM25 库"""
  63. t_start = time.perf_counter()
  64. all_chunks: list[dict] = []
  65. for section in paper.sections:
  66. all_chunks.extend(self._chunk_section(section))
  67. t_chunk = time.perf_counter()
  68. logger.info(
  69. "[add_paper] %s 分块完成: %d sections → %d chunks, 用时 %.2fs",
  70. paper.path, len(paper.sections), len(all_chunks), t_chunk - t_start,
  71. )
  72. if not all_chunks:
  73. return
  74. # 向量库:传入 paper_path/title + chunks[{section, text}]
  75. self.vector_store.add_chunks(paper.path, paper.title, all_chunks)
  76. t_vec = time.perf_counter()
  77. logger.info(
  78. "[add_paper] %s 向量库写入完成: 用时 %.2fs (累计 %.2fs)",
  79. paper.path, t_vec - t_chunk, t_vec - t_start,
  80. )
  81. # BM25:chunks 需带 paper_path/title
  82. bm25_chunks = [
  83. {**c, "paper_path": paper.path, "title": paper.title} for c in all_chunks
  84. ]
  85. self.bm25_store.add_chunks(bm25_chunks)
  86. t_bm25 = time.perf_counter()
  87. logger.info(
  88. "[add_paper] %s BM25 写入完成: 用时 %.2fs (累计 %.2fs)",
  89. paper.path, t_bm25 - t_vec, t_bm25 - t_start,
  90. )
  91. def load_all(self, papers: list[ParsedPaper]):
  92. """批量加载多篇论文"""
  93. for paper in papers:
  94. self.add_paper(paper)
  95. # ---------- 检索 ----------
  96. def search(
  97. self,
  98. query: str,
  99. top_k: int = 5,
  100. vector_weight: float = 0.6,
  101. bm25_weight: float = 0.4,
  102. ) -> list[dict]:
  103. """混合检索:向量 + BM25 各取 top_k*2,加权分数归一化重排。
  104. Returns:
  105. list[dict],每项 {text, paper_path, title, section, score},按融合分数降序
  106. """
  107. fetch_k = top_k * 2
  108. vector_results = self.vector_store.vector_search(query, top_k=fetch_k)
  109. bm25_results = self.bm25_store.bm25_search(query, top_k=fetch_k)
  110. # 各路结果做 min-max 归一化到 [0,1]
  111. vector_results = self._normalize(vector_results)
  112. bm25_results = self._normalize(bm25_results)
  113. # 以 text 为 key 合并去重,加权累加分数
  114. merged: dict[str, dict] = {}
  115. for r in vector_results:
  116. key = r["text"]
  117. s = vector_weight * r["norm_score"]
  118. if key not in merged:
  119. merged[key] = {
  120. "text": r["text"],
  121. "paper_path": r["paper_path"],
  122. "title": r["title"],
  123. "section": r["section"],
  124. "score": s,
  125. }
  126. else:
  127. merged[key]["score"] += s
  128. for r in bm25_results:
  129. key = r["text"]
  130. s = bm25_weight * r["norm_score"]
  131. if key not in merged:
  132. merged[key] = {
  133. "text": r["text"],
  134. "paper_path": r["paper_path"],
  135. "title": r["title"],
  136. "section": r["section"],
  137. "score": s,
  138. }
  139. else:
  140. merged[key]["score"] += s
  141. # 按融合分数降序取 top_k
  142. out = sorted(merged.values(), key=lambda x: x["score"], reverse=True)[:top_k]
  143. return out
  144. @staticmethod
  145. def _normalize(results: list[dict]) -> list[dict]:
  146. """对结果列表的 score 做 min-max 归一化,写入 norm_score 字段"""
  147. if not results:
  148. return results
  149. scores = [r["score"] for r in results]
  150. lo, hi = min(scores), max(scores)
  151. rng = hi - lo if hi > lo else 1.0
  152. for r in results:
  153. r["norm_score"] = (r["score"] - lo) / rng
  154. return results
  155. # ---------- 异步友好封装 ----------
  156. async def async_add_paper(self, paper: ParsedPaper):
  157. """异步写入单篇论文"""
  158. await asyncio.to_thread(self.add_paper, paper)
  159. async def async_load_all(self, papers: list[ParsedPaper]):
  160. """异步批量加载"""
  161. await asyncio.to_thread(self.load_all, papers)
  162. async def async_search(
  163. self,
  164. query: str,
  165. top_k: int = 5,
  166. vector_weight: float = 0.6,
  167. bm25_weight: float = 0.4,
  168. ) -> list[dict]:
  169. """异步混合检索"""
  170. return await asyncio.to_thread(
  171. self.search, query, top_k, vector_weight, bm25_weight
  172. )