embedding.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272
  1. """本地 Embedding:直接用 transformers 加载 BGE 模型(绕开 sentence-transformers 与 pyarrow)。
  2. 为什么不用 sentence-transformers:
  3. 你的环境里 pyarrow 在 import datasets 时触发 0xC0000005 访问冲突,
  4. 而 sentence-transformers 强依赖 datasets → pyarrow,整条链路崩溃。
  5. 这里直接用 transformers 加载 BERT backbone + CLS Pooling,向量结果与 sentence-transformers 等价。
  6. """
  7. from __future__ import annotations
  8. import os
  9. import threading
  10. import logging
  11. from typing import List, Optional
  12. # tokenizers 并行告警抑制;CUDA 由 torch 自行检测(GPU 优先,CPU 兜底)
  13. os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
  14. # 强制 transformers / huggingface_hub 走离线模式,不再请求 Hub
  15. os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
  16. os.environ.setdefault("HF_HUB_OFFLINE", "1")
  17. from backend.config import settings
  18. logger = logging.getLogger(__name__)
  19. # mini-batch 大小:GPU 16,CPU 8(避免内存爆 + 减少 padding 浪费)
  20. EMBED_BATCH_SIZE = 16
  21. class Embedder:
  22. """本地 Embedder,懒加载 BGE 系列模型。
  23. 支持任意 HuggingFace 上有 BertModel + AutoTokenizer 结构的嵌入模型
  24. (bge-large-zh-v1.5 / bge-base-zh / bge-small-zh / bge-m3 等)。
  25. """
  26. def __init__(self, model_name: Optional[str] = None):
  27. self.model_name = model_name or settings.EMBEDDING_MODEL
  28. self._model = None
  29. self._tokenizer = None
  30. self._lock = threading.Lock()
  31. # 自动检测:GPU 可用则用 GPU,否则回退 CPU
  32. self._device = self._detect_device()
  33. self._batch_size = EMBED_BATCH_SIZE if self._device != "cpu" else 8
  34. @staticmethod
  35. def _detect_device() -> str:
  36. """检测可用设备:CUDA 优先,CPU 兜底。"""
  37. try:
  38. import torch
  39. if torch.cuda.is_available():
  40. return "cuda"
  41. except Exception:
  42. pass
  43. return "cpu"
  44. def _resolve_local_snapshot(self, model_name: str) -> Optional[str]:
  45. """如果 HF 本地缓存已有该模型的 snapshot,直接返回快照路径。"""
  46. try:
  47. from pathlib import Path as _P
  48. from huggingface_hub.constants import HF_HUB_CACHE
  49. repo_dir_name = "models--" + model_name.replace("/", "--")
  50. snapshots_dir = _P(HF_HUB_CACHE) / repo_dir_name / "snapshots"
  51. if snapshots_dir.is_dir():
  52. snapshots = sorted(
  53. [p for p in snapshots_dir.iterdir() if p.is_dir()],
  54. key=lambda p: p.stat().st_mtime,
  55. reverse=True,
  56. )
  57. if snapshots:
  58. snap = snapshots[0]
  59. if (snap / "config.json").exists():
  60. return str(snap)
  61. except Exception:
  62. pass
  63. return None
  64. def _load(self):
  65. """懒加载模型:首次调用时加载,双重检查锁保证线程安全。"""
  66. import time as _time_load
  67. _t_entry = _time_load.perf_counter()
  68. logger.info("[_load] 进入 _load: model_is_none=%s", self._model is None)
  69. if self._model is None:
  70. _t_lock0 = _time_load.perf_counter()
  71. logger.info("[_load] 等待 self._lock...")
  72. with self._lock:
  73. _t_lock1 = _time_load.perf_counter()
  74. logger.info("[_load] 获得 self._lock: 等待 %.3fs", _t_lock1 - _t_lock0)
  75. if self._model is None:
  76. import time as _time
  77. _t_imp_t = _time.perf_counter()
  78. import torch
  79. _t_imp_torch = _time.perf_counter()
  80. from transformers import AutoModel, AutoTokenizer
  81. _t_imp_tf = _time.perf_counter()
  82. logger.info(
  83. "[_load] imports 完成: 总耗时 %.3fs (torch=%.3fs transformers=%.3fs, 从获得锁算起)",
  84. _t_imp_tf - _t_lock1,
  85. _t_imp_torch - _t_imp_t,
  86. _t_imp_tf - _t_imp_torch,
  87. )
  88. _t0 = _time.perf_counter()
  89. local_snapshot = self._resolve_local_snapshot(self.model_name)
  90. if local_snapshot:
  91. load_path = local_snapshot
  92. else:
  93. load_path = self.model_name
  94. _t_resolve = _time.perf_counter()
  95. # 注意:bge 官方建议 query 加指令前缀 "为这个句子生成表示以用于检索中文文档:"
  96. self._tokenizer = AutoTokenizer.from_pretrained(
  97. load_path, local_files_only=bool(local_snapshot)
  98. )
  99. _t_tok = _time.perf_counter()
  100. self._model = AutoModel.from_pretrained(
  101. load_path, local_files_only=bool(local_snapshot)
  102. )
  103. _t_model_load = _time.perf_counter()
  104. # 关键:将模型迁移到 device,并验证实际驻留设备
  105. self._model = self._model.to(self._device)
  106. _t_to_device = _time.perf_counter()
  107. # 探测模型参数实际所在设备(防止 .to("cuda") 静默失败)
  108. try:
  109. param = next(self._model.parameters())
  110. actual_device = str(param.device)
  111. except Exception:
  112. actual_device = "unknown"
  113. self._model.eval()
  114. _t_eval = _time.perf_counter()
  115. # 注意:不做 CUDA warmup。
  116. # 实测在 uvicorn 长驻进程里,dummy forward 反而要 24s(首次 CUDA 上下文初始化)。
  117. # 让第一次真实 forward 自然完成初始化即可,避免额外开销。
  118. _t_warmup = _t_eval
  119. logger.info(
  120. "Embedder 模型加载完成: model=%s, device=%s, actual_param_device=%s, batch_size=%d | "
  121. "resolve=%.2fs tokenizer=%.2fs model_load=%.2fs to_device=%.2fs eval=%.2fs cuda_warmup=%.2fs total=%.2fs",
  122. self.model_name, self._device, actual_device, self._batch_size,
  123. _t_resolve - _t0, _t_tok - _t_resolve, _t_model_load - _t_tok,
  124. _t_to_device - _t_model_load, _t_eval - _t_to_device,
  125. _t_warmup - _t_eval, _t_warmup - _t0,
  126. )
  127. @staticmethod
  128. def _last_token_pool(last_hidden_states, attention_mask):
  129. """BGE 系列使用的 last-token pooling(等价于 sentence-transformers 的 cls + 末尾 token)。"""
  130. left_padding = attention_mask[:, -1].sum() == attention_mask.shape[0]
  131. if left_padding:
  132. return last_hidden_states[:, -1]
  133. sequence_lengths = attention_mask.sum(dim=1) - 1
  134. batch_size = last_hidden_states.shape[0]
  135. return last_hidden_states[
  136. torch.arange(batch_size, device=last_hidden_states.device),
  137. sequence_lengths,
  138. ]
  139. def _encode(self, texts: List[str]):
  140. import time as _time
  141. import torch
  142. # BGE 官方推荐:query 加指令前缀,passage 不加
  143. is_query = getattr(self, "_is_query", True)
  144. if is_query and "bge" in self.model_name.lower() and "zh" in self.model_name.lower():
  145. prefix = "为这个句子生成表示以用于检索中文文档:"
  146. texts = [prefix + t for t in texts]
  147. # 探测模型实际驻留设备(防止 .to("cuda") 静默失败导致实际跑 CPU)
  148. try:
  149. _actual_dev = str(next(self._model.parameters()).device)
  150. except Exception:
  151. _actual_dev = "unknown"
  152. # mini-batching:避免一次性把全部 chunk 喂入模型(CPU/显存压力 + padding 浪费)
  153. all_embeddings: List[List[float]] = []
  154. bs = max(1, self._batch_size)
  155. n_batches = (len(texts) + bs - 1) // bs
  156. _t_enc_start = _time.perf_counter()
  157. logger.info(
  158. "[_encode] 开始编码: %d texts, batch_size=%d, %d batches, model_device=%s, declared=%s",
  159. len(texts), bs, n_batches, _actual_dev, self._device,
  160. )
  161. for i in range(0, len(texts), bs):
  162. bi = i // bs
  163. batch = texts[i : i + bs]
  164. _t_b0 = _time.perf_counter()
  165. inputs = self._tokenizer(
  166. batch,
  167. padding=True,
  168. truncation=True,
  169. max_length=512,
  170. return_tensors="pt",
  171. ).to(self._device)
  172. _t_tok = _time.perf_counter()
  173. with torch.no_grad():
  174. outputs = self._model(**inputs)
  175. # bge 用 last_token_pool;其它 BERT 类模型退化为 [CLS] pooling
  176. try:
  177. embeddings = self._last_token_pool(
  178. outputs.last_hidden_state, inputs["attention_mask"]
  179. )
  180. except Exception:
  181. embeddings = outputs.last_hidden_state[:, 0]
  182. embeddings = torch.nn.functional.normalize(embeddings, p=2, dim=1)
  183. # GPU 同步,确保 forward 真正完成后再计时
  184. if "cuda" in _actual_dev:
  185. torch.cuda.synchronize()
  186. _t_fwd = _time.perf_counter()
  187. all_embeddings.extend(embeddings.cpu().numpy().tolist())
  188. _t_cpu = _time.perf_counter()
  189. # 每个批次都打印(首次诊断用;定位后可改为每 10 批打印一次)
  190. if bi < 3 or bi % 10 == 0 or bi == n_batches - 1:
  191. logger.info(
  192. "[_encode] batch %d/%d: size=%d, tok=%.3fs fwd=%.3fs to_cpu=%.3fs (elapsed %.2fs)",
  193. bi + 1, n_batches, len(batch),
  194. _t_tok - _t_b0, _t_fwd - _t_tok, _t_cpu - _t_fwd, _t_cpu - _t_enc_start,
  195. )
  196. logger.info(
  197. "[_encode] 完成: %d texts, total=%.2fs, avg=%.3fs/text",
  198. len(texts), _time.perf_counter() - _t_enc_start,
  199. (_time.perf_counter() - _t_enc_start) / max(1, len(texts)),
  200. )
  201. return all_embeddings
  202. def embed_texts(self, texts: List[str]) -> List[List[float]]:
  203. """批量向量化(passage/documents):不加 query 前缀。"""
  204. import time as _time_et
  205. logger.info("[embed_texts] 入口: %d texts, model_loaded=%s", len(texts), self._model is not None)
  206. if not texts:
  207. return []
  208. _t_et0 = _time_et.perf_counter()
  209. self._load()
  210. logger.info("[embed_texts] _load 返回: %.3fs", _time_et.perf_counter() - _t_et0)
  211. old = getattr(self, "_is_query", True)
  212. self._is_query = False
  213. try:
  214. return self._encode(texts)
  215. finally:
  216. self._is_query = old
  217. def embed_query(self, text: str) -> List[float]:
  218. """单条查询向量化:加 query 前缀。"""
  219. self._load()
  220. self._is_query = True
  221. return self._encode([text])[0]
  222. # ---------- 模块级单例 ----------
  223. _default_embedder: Optional["Embedder"] = None
  224. _embedder_lock = threading.Lock()
  225. def get_default_embedder() -> "Embedder":
  226. """获取模块级单例 Embedder,复用已加载的模型与 CUDA 上下文。
  227. 业务流程应优先调用此函数,而非直接 Embedder(),避免重复加载模型(每次 12s)
  228. 和重复触发 CUDA 上下文初始化(首次 forward 42s)。
  229. """
  230. global _default_embedder
  231. if _default_embedder is None:
  232. with _embedder_lock:
  233. if _default_embedder is None:
  234. _default_embedder = Embedder()
  235. return _default_embedder