vector_store.py 4.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131
  1. """Chroma 向量库:基于 chromadb 的本地持久化向量存储"""
  2. from __future__ import annotations
  3. import logging
  4. import time
  5. import uuid
  6. from typing import Optional
  7. from backend.config import settings
  8. from backend.rag.embedding import Embedder
  9. logger = logging.getLogger(__name__)
  10. class ChromaStore:
  11. """Chroma 向量库封装,懒初始化 client 与 collection"""
  12. def __init__(
  13. self,
  14. chroma_path: Optional[str] = None,
  15. embedder: Optional[Embedder] = None,
  16. collection_name: str = "paper_chunks",
  17. ):
  18. # 路径默认取配置中的 CHROMA_PATH
  19. self.chroma_path = str(chroma_path or settings.CHROMA_PATH)
  20. self.embedder = embedder or Embedder()
  21. self.collection_name = collection_name
  22. self._client = None
  23. self._collection = None
  24. def _ensure(self):
  25. """懒初始化 client 和 collection,首次访问时建立"""
  26. if self._client is None:
  27. _t0 = time.perf_counter()
  28. # 延迟导入,避免模块导入即触发 chromadb 重型初始化
  29. import chromadb
  30. _t_import = time.perf_counter()
  31. self._client = chromadb.PersistentClient(path=self.chroma_path)
  32. _t_client = time.perf_counter()
  33. self._collection = self._client.get_or_create_collection(
  34. name=self.collection_name
  35. )
  36. _t_coll = time.perf_counter()
  37. logger.info(
  38. "[ChromaStore._ensure] 首次初始化: import_chromadb=%.3fs client=%.3fs collection=%.3fs total=%.3fs path=%s",
  39. _t_import - _t0, _t_client - _t_import, _t_coll - _t_client,
  40. _t_coll - _t0, self.chroma_path,
  41. )
  42. def add_chunks(self, paper_path: str, title: str, chunks: list[dict]):
  43. """批量入库。
  44. Args:
  45. paper_path: 论文文件路径
  46. title: 论文标题
  47. chunks: 每个 chunk 形如 {section, text}
  48. """
  49. if not chunks:
  50. return
  51. self._ensure()
  52. t0 = time.perf_counter()
  53. texts = [c["text"] for c in chunks]
  54. embeddings = self.embedder.embed_texts(texts)
  55. t_emb = time.perf_counter()
  56. logger.info(
  57. "[add_chunks] %s embedding 完成: %d chunks, 用时 %.2fs",
  58. paper_path, len(chunks), t_emb - t0,
  59. )
  60. # 唯一 id,避免冲突
  61. ids = [str(uuid.uuid4()) for _ in chunks]
  62. metadatas = [
  63. {
  64. "paper_path": paper_path,
  65. "title": title,
  66. "section": c.get("section", ""),
  67. }
  68. for c in chunks
  69. ]
  70. self._collection.add(
  71. ids=ids,
  72. embeddings=embeddings,
  73. documents=texts,
  74. metadatas=metadatas,
  75. )
  76. t_add = time.perf_counter()
  77. logger.info(
  78. "[add_chunks] %s chroma.add 完成: 用时 %.2fs (累计 %.2fs)",
  79. paper_path, t_add - t_emb, t_add - t0,
  80. )
  81. def vector_search(self, query: str, top_k: int = 5) -> list[dict]:
  82. """向量检索,返回 top_k 结果。
  83. Returns:
  84. list[dict],每项 {text, paper_path, title, section, score}
  85. score 越大越相似(由距离转换得到)。
  86. """
  87. self._ensure()
  88. query_emb = self.embedder.embed_query(query)
  89. results = self._collection.query(
  90. query_embeddings=[query_emb],
  91. n_results=top_k,
  92. )
  93. out: list[dict] = []
  94. # chroma 返回结构:每个字段是“外层 list=查询数,内层 list=结果数”
  95. docs = (results.get("documents") or [[]])[0]
  96. metas = (results.get("metadatas") or [[]])[0]
  97. dists = (results.get("distances") or [[]])[0]
  98. for doc, meta, dist in zip(docs, metas, dists):
  99. # 距离越小越相似,转换为 (0,1] 的相似度分数
  100. score = 1.0 / (1.0 + float(dist)) if dist is not None else 0.0
  101. out.append(
  102. {
  103. "text": doc,
  104. "paper_path": meta.get("paper_path", ""),
  105. "title": meta.get("title", ""),
  106. "section": meta.get("section", ""),
  107. "score": score,
  108. }
  109. )
  110. return out
  111. def clear(self):
  112. """清空 collection(删除后重建空 collection)"""
  113. self._ensure()
  114. self._client.delete_collection(name=self.collection_name)
  115. self._collection = self._client.get_or_create_collection(
  116. name=self.collection_name
  117. )