utils.py 736 B

12345678910111213141516171819202122232425262728
  1. """通用工具函数"""
  2. from __future__ import annotations
  3. import logging
  4. import re
  5. logger = logging.getLogger(__name__)
  6. def strip_thinking_tokens(text: str) -> str:
  7. """移除 <think>...</think> 段"""
  8. while "<think>" in text and "</think>" in text:
  9. start = text.find("<think>")
  10. end = text.find("</think>") + len("</think>")
  11. text = text[:start] + text[end:]
  12. return text
  13. def strip_tool_calls(text: str) -> str:
  14. """移除残留的 [TOOL_CALL:...] 标记"""
  15. if not text:
  16. return text
  17. return re.sub(r"\[TOOL_CALL:[^\]]+\]", "", text)
  18. def normalize_title(title: str) -> str:
  19. """标题归一化,用于去重"""
  20. return re.sub(r"\s+", " ", (title or "").strip().lower())