bm25_store.py 3.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105
  1. """BM25 关键词检索:基于 rank_bm25 的内存检索,中英文兼容分词"""
  2. from __future__ import annotations
  3. import re
  4. from typing import Optional
  5. from rank_bm25 import BM25Okapi
  6. class BM25Store:
  7. """BM25 内存文档库,懒构建索引"""
  8. def __init__(self):
  9. # 原始文档与对应分词结果并行维护
  10. self._docs: list[dict] = []
  11. self._tokenized: list[list[str]] = []
  12. self._bm25: Optional[BM25Okapi] = None
  13. self._dirty = False # 标记索引是否需要重建
  14. @staticmethod
  15. def _tokenize(text: str) -> list[str]:
  16. """中英文兼容分词(不依赖 jieba)。
  17. - 英文/数字:按连续字母数字切词,统一小写
  18. - 中文:按单字切分(字粒度,兼容无词典场景)
  19. """
  20. if not text:
  21. return []
  22. tokens: list[str] = []
  23. # 英文单词与数字
  24. for m in re.findall(r"[A-Za-z0-9]+", text):
  25. tokens.append(m.lower())
  26. # 中文字符逐字切分
  27. for ch in re.findall(r"[\u4e00-\u9fff]", text):
  28. tokens.append(ch)
  29. return tokens
  30. def add_chunks(self, chunks: list[dict]):
  31. """添加文档。
  32. 每个 chunk 形如 {text, paper_path, title, section}。
  33. """
  34. for c in chunks:
  35. doc = {
  36. "text": c.get("text", ""),
  37. "paper_path": c.get("paper_path", ""),
  38. "title": c.get("title", ""),
  39. "section": c.get("section", ""),
  40. }
  41. self._docs.append(doc)
  42. self._tokenized.append(self._tokenize(doc["text"]))
  43. self._dirty = True
  44. def _ensure_index(self):
  45. """懒构建 BM25 索引:有新增时重建"""
  46. if self._dirty or self._bm25 is None:
  47. if self._tokenized:
  48. self._bm25 = BM25Okapi(self._tokenized)
  49. else:
  50. self._bm25 = None
  51. self._dirty = False
  52. def bm25_search(self, query: str, top_k: int = 5) -> list[dict]:
  53. """BM25 检索,返回 top_k 结果。
  54. Returns:
  55. list[dict],每项 {text, paper_path, title, section, score}
  56. """
  57. self._ensure_index()
  58. if self._bm25 is None or not self._docs:
  59. return []
  60. q_tokens = self._tokenize(query)
  61. if not q_tokens:
  62. return []
  63. scores = self._bm25.get_scores(q_tokens)
  64. q_set = set(q_tokens)
  65. # 仅保留至少包含一个查询词的文档,避免返回无关结果
  66. # (注意:小语料下 BM25Okapi 的 IDF 可能为 0,不能再用 score>0 过滤)
  67. candidate_idx = [
  68. i for i, toks in enumerate(self._tokenized) if q_set & set(toks)
  69. ]
  70. if not candidate_idx:
  71. return []
  72. # 在候选中按分数降序取 top_k
  73. candidate_idx.sort(key=lambda i: scores[i], reverse=True)
  74. out: list[dict] = []
  75. for i in candidate_idx[:top_k]:
  76. doc = self._docs[i]
  77. out.append(
  78. {
  79. "text": doc["text"],
  80. "paper_path": doc["paper_path"],
  81. "title": doc["title"],
  82. "section": doc["section"],
  83. "score": float(scores[i]),
  84. }
  85. )
  86. return out
  87. def clear(self):
  88. """清空文档库"""
  89. self._docs = []
  90. self._tokenized = []
  91. self._bm25 = None
  92. self._dirty = False