web_search.py 3.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134
  1. """联网搜索工具:DuckDuckGo HTML 搜索,用于创新点新颖性去重"""
  2. from __future__ import annotations
  3. import re
  4. from urllib.parse import parse_qs, unquote, urlparse
  5. from backend.tools.base import Tool
  6. # ---------- DuckDuckGo HTML 结果解析 ----------
  7. # 结果标题与链接
  8. _TITLE_PATTERN = re.compile(
  9. r'<a[^>]*class="result__a"[^>]*href="([^"]*)"[^>]*>(.*?)</a>',
  10. re.DOTALL,
  11. )
  12. # 结果摘要
  13. _SNIPPET_PATTERN = re.compile(
  14. r'<a[^>]*class="result__snippet"[^>]*>(.*?)</a>',
  15. re.DOTALL,
  16. )
  17. # HTML 标签
  18. _TAG_PATTERN = re.compile(r"<[^>]+>")
  19. # 常见 HTML 实体
  20. _HTML_ENTITIES = {
  21. "&amp;": "&",
  22. "&lt;": "<",
  23. "&gt;": ">",
  24. "&quot;": '"',
  25. "&#39;": "'",
  26. "&nbsp;": " ",
  27. }
  28. def _strip_html(s: str) -> str:
  29. """去除 HTML 标签与实体,压缩空白"""
  30. s = _TAG_PATTERN.sub("", s)
  31. for entity, char in _HTML_ENTITIES.items():
  32. s = s.replace(entity, char)
  33. s = re.sub(r"\s+", " ", s)
  34. return s.strip()
  35. def _extract_real_url(href: str) -> str:
  36. """从 DuckDuckGo 跳转链接中提取真实 URL"""
  37. if "uddg=" in href:
  38. # 补全协议头以便解析(DDG 链接常以 // 开头)
  39. target = href if href.startswith("http") else "https:" + href
  40. parsed = urlparse(target)
  41. qs = parse_qs(parsed.query)
  42. if "uddg" in qs and qs["uddg"]:
  43. return unquote(qs["uddg"][0])
  44. return href
  45. def _parse_ddg_html(html: str, max_results: int) -> list[dict]:
  46. """解析 DuckDuckGo HTML 结果页,提取标题、摘要、URL"""
  47. titles = _TITLE_PATTERN.findall(html)
  48. snippets = _SNIPPET_PATTERN.findall(html)
  49. results: list[dict] = []
  50. for i, (href, raw_title) in enumerate(titles):
  51. if i >= max_results:
  52. break
  53. title = _strip_html(raw_title)
  54. snippet = _strip_html(snippets[i]) if i < len(snippets) else ""
  55. url = _extract_real_url(href)
  56. if title:
  57. results.append({"title": title, "snippet": snippet, "url": url})
  58. return results
  59. # ---------- 异步工具函数 ----------
  60. async def web_search(query: str, max_results: int = 5) -> dict:
  61. """DuckDuckGo 联网搜索,返回标题与摘要。
  62. 失败时降级返回空结果,不崩溃。
  63. Returns:
  64. {"query": query, "results": [{title, snippet, url}]}
  65. """
  66. import httpx
  67. from urllib.parse import quote
  68. try:
  69. url = f"https://html.duckduckgo.com/html/?q={quote(query)}"
  70. headers = {
  71. "User-Agent": (
  72. "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
  73. "AppleWebKit/537.36 (KHTML, like Gecko) "
  74. "Chrome/120.0.0.0 Safari/537.36"
  75. )
  76. }
  77. async with httpx.AsyncClient(timeout=15.0, follow_redirects=True) as client:
  78. resp = await client.get(url, headers=headers)
  79. resp.raise_for_status()
  80. html = resp.text
  81. results = _parse_ddg_html(html, max_results)
  82. return {"query": query, "results": results}
  83. except Exception as e:
  84. # 降级:返回空结果,不崩溃
  85. return {"query": query, "results": [], "error": f"联网搜索失败: {e}"}
  86. # ---------- Tool 定义 ----------
  87. _PARAMETERS = {
  88. "type": "object",
  89. "properties": {
  90. "query": {
  91. "type": "string",
  92. "description": "搜索查询文本",
  93. },
  94. "max_results": {
  95. "type": "integer",
  96. "description": "最大返回结果数,默认 5",
  97. "default": 5,
  98. },
  99. },
  100. "required": ["query"],
  101. }
  102. web_search_tool = Tool(
  103. name="web_search",
  104. description=(
  105. "联网搜索(DuckDuckGo),用于创新点新颖性去重。"
  106. "返回 {query, results: [{title, snippet, url}]}。失败时降级返回空结果。"
  107. ),
  108. parameters=_PARAMETERS,
  109. func=web_search,
  110. )