quality.py 9.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261
  1. """输出质量评估:基于 LLM 对创新点与最终报告进行质量评分"""
  2. from __future__ import annotations
  3. import logging
  4. from typing import Optional
  5. from backend.llm.client import DeepSeekClient
  6. from backend.llm.json_utils import parse_json_safe
  7. logger = logging.getLogger(__name__)
  8. # 模块级懒加载单例
  9. _client: Optional[DeepSeekClient] = None
  10. def get_client() -> DeepSeekClient:
  11. """懒加载 DeepSeek 客户端单例"""
  12. global _client
  13. if _client is None:
  14. _client = DeepSeekClient()
  15. return _client
  16. def _clamp_score(value: float) -> float:
  17. """将评分限制在 0-10 范围内,非法值降级为 0.0"""
  18. try:
  19. v = float(value)
  20. except (TypeError, ValueError):
  21. return 0.0
  22. return max(0.0, min(10.0, v))
  23. class QualityEvaluator:
  24. """输出质量评估器:调用 LLM 对创新点与报告做三维/多维评分"""
  25. def __init__(self, client: Optional[DeepSeekClient] = None) -> None:
  26. # 优先使用注入的 client,否则使用模块级懒加载单例
  27. self.client = client or get_client()
  28. async def evaluate_innovations(self, innovations: list[dict]) -> list[dict]:
  29. """对每个创新点用 LLM 评估三维评分(novelty/feasibility/significance 0-10)
  30. 若创新点已有 score 则校验/调整,否则新生成。
  31. 返回 [{innovation_title, score: {novelty, feasibility, significance}, reasoning}]
  32. """
  33. if not innovations:
  34. return []
  35. results: list[dict] = []
  36. for inv in innovations:
  37. if not isinstance(inv, dict):
  38. continue
  39. results.append(await self._evaluate_single_innovation(inv))
  40. return results
  41. async def _evaluate_single_innovation(self, inv: dict) -> dict:
  42. """评估单个创新点:构造提示词→调用 LLM→解析三维评分"""
  43. title = inv.get("title", "")
  44. idea = inv.get("idea", "")
  45. existing_score = inv.get("score")
  46. # 已有分数则校验/调整,否则新生成
  47. if isinstance(existing_score, dict) and existing_score:
  48. score_hint = (
  49. f"该创新点已有初步评分:novelty={existing_score.get('novelty', 0)}, "
  50. f"feasibility={existing_score.get('feasibility', 0)}, "
  51. f"significance={existing_score.get('significance', 0)}。"
  52. "请基于以上信息进行校验与调整。"
  53. )
  54. else:
  55. score_hint = "该创新点暂无评分,请生成新的三维评分。"
  56. prompt = f"""你是一位严谨的学术评审专家,请对以下创新点进行三维评分(0-10 分,保留一位小数)。
  57. ## 创新点信息
  58. - 标题:{title}
  59. - 思路:{idea}
  60. - 论据支撑:{inv.get('supporting_evidence', '')}
  61. - 新颖性检查:{inv.get('novelty_check', '') or '无'}
  62. ## 评分维度
  63. - novelty(新颖性):与现有工作的差异程度
  64. - feasibility(可行性):技术实现与资源获取的难度
  65. - significance(显著性):学术或应用价值
  66. ## 当前状态
  67. {score_hint}
  68. ## 输出要求
  69. 严格输出 JSON,不要包含任何额外说明文本或代码块标记。格式如下:
  70. {{
  71. "innovation_title": "{title}",
  72. "score": {{
  73. "novelty": 0.0到10.0,
  74. "feasibility": 0.0到10.0,
  75. "significance": 0.0到10.0
  76. }},
  77. "reasoning": "评分理由,简明扼要"
  78. }}
  79. """
  80. messages = [
  81. {
  82. "role": "system",
  83. "content": "你是一位学术评审专家,擅长对创新点进行多维评分。只输出严格 JSON,不要输出任何其他内容。",
  84. },
  85. {"role": "user", "content": prompt},
  86. ]
  87. try:
  88. response = await self.client.chat(messages=messages)
  89. content = response.choices[0].message.content or ""
  90. data = parse_json_safe(content)
  91. if isinstance(data, dict):
  92. score = data.get("score", {})
  93. if not isinstance(score, dict):
  94. score = {}
  95. return {
  96. "innovation_title": data.get("innovation_title", title),
  97. "score": {
  98. "novelty": _clamp_score(score.get("novelty", 0.0)),
  99. "feasibility": _clamp_score(score.get("feasibility", 0.0)),
  100. "significance": _clamp_score(score.get("significance", 0.0)),
  101. },
  102. "reasoning": data.get("reasoning", ""),
  103. }
  104. logger.warning("创新点评分 JSON 解析失败: %s", content[:200])
  105. except Exception as exc:
  106. logger.warning("创新点 '%s' 评分失败: %s", title, exc)
  107. # 降级:使用已有分数或全 0
  108. fallback = existing_score if isinstance(existing_score, dict) else {}
  109. return {
  110. "innovation_title": title,
  111. "score": {
  112. "novelty": _clamp_score(fallback.get("novelty", 0.0)),
  113. "feasibility": _clamp_score(fallback.get("feasibility", 0.0)),
  114. "significance": _clamp_score(fallback.get("significance", 0.0)),
  115. },
  116. "reasoning": "LLM 评分解析失败,使用降级分数",
  117. }
  118. async def evaluate_report(self, report: dict) -> dict:
  119. """对最终报告整体质量评估:结构完整性、论证严谨性、可执行性、可追溯性
  120. 返回 {overall_score, dimensions: {...}, suggestions: [...]}
  121. """
  122. if not report:
  123. return {
  124. "overall_score": 0.0,
  125. "dimensions": {
  126. "structural_completeness": 0.0,
  127. "argument_rigor": 0.0,
  128. "executability": 0.0,
  129. "traceability": 0.0,
  130. },
  131. "suggestions": ["报告为空,无法评估"],
  132. }
  133. # 提取报告关键内容用于评估
  134. background = report.get("background_review", "")
  135. innovations = report.get("innovations", [])
  136. plans = report.get("experiment_plans", [])
  137. references = report.get("references", [])
  138. innovations_text = "\n".join(
  139. f"- {inv.get('title', '')}: {inv.get('idea', '')[:100]}"
  140. for inv in innovations
  141. if isinstance(inv, dict)
  142. )
  143. plans_text = "\n".join(
  144. f"- {p.get('innovation_title', '')}: 假设={p.get('hypothesis', '')[:80]}"
  145. for p in plans
  146. if isinstance(p, dict)
  147. )
  148. refs_count = len(references) if isinstance(references, list) else 0
  149. prompt = f"""你是一位学术报告评审专家,请对以下最终报告进行整体质量评估。
  150. ## 报告概览
  151. - 背景综述长度:{len(background) if isinstance(background, str) else 0} 字符
  152. - 创新点数量:{len(innovations) if isinstance(innovations, list) else 0}
  153. - 实验方案数量:{len(plans) if isinstance(plans, list) else 0}
  154. - 参考文献数量:{refs_count}
  155. ## 创新点列表
  156. {innovations_text or '无'}
  157. ## 实验方案列表
  158. {plans_text or '无'}
  159. ## 评估维度(0-10 分,保留一位小数)
  160. - structural_completeness(结构完整性):背景/创新/方案/参考是否齐全
  161. - argument_rigor(论证严谨性):论据是否充分、逻辑是否自洽
  162. - executability(可执行性):实验方案是否可落地
  163. - traceability(可追溯性):是否标注来源、可回溯原文
  164. ## 输出要求
  165. 严格输出 JSON,不要包含任何额外说明文本或代码块标记。格式如下:
  166. {{
  167. "overall_score": 0.0到10.0,
  168. "dimensions": {{
  169. "structural_completeness": 0.0到10.0,
  170. "argument_rigor": 0.0到10.0,
  171. "executability": 0.0到10.0,
  172. "traceability": 0.0到10.0
  173. }},
  174. "suggestions": ["改进建议1", "改进建议2"]
  175. }}
  176. """
  177. messages = [
  178. {
  179. "role": "system",
  180. "content": "你是一位学术报告评审专家,擅长整体质量评估。只输出严格 JSON,不要输出任何其他内容。",
  181. },
  182. {"role": "user", "content": prompt},
  183. ]
  184. try:
  185. response = await self.client.chat(messages=messages)
  186. content = response.choices[0].message.content or ""
  187. data = parse_json_safe(content)
  188. if isinstance(data, dict):
  189. dimensions = data.get("dimensions", {})
  190. if not isinstance(dimensions, dict):
  191. dimensions = {}
  192. suggestions = data.get("suggestions", [])
  193. if not isinstance(suggestions, list):
  194. suggestions = []
  195. return {
  196. "overall_score": _clamp_score(data.get("overall_score", 0.0)),
  197. "dimensions": {
  198. "structural_completeness": _clamp_score(
  199. dimensions.get("structural_completeness", 0.0)
  200. ),
  201. "argument_rigor": _clamp_score(
  202. dimensions.get("argument_rigor", 0.0)
  203. ),
  204. "executability": _clamp_score(
  205. dimensions.get("executability", 0.0)
  206. ),
  207. "traceability": _clamp_score(
  208. dimensions.get("traceability", 0.0)
  209. ),
  210. },
  211. "suggestions": [str(s) for s in suggestions],
  212. }
  213. logger.warning("报告评估 JSON 解析失败: %s", content[:200])
  214. except Exception as exc:
  215. logger.warning("报告评估失败: %s", exc)
  216. # 降级返回
  217. return {
  218. "overall_score": 0.0,
  219. "dimensions": {
  220. "structural_completeness": 0.0,
  221. "argument_rigor": 0.0,
  222. "executability": 0.0,
  223. "traceability": 0.0,
  224. },
  225. "suggestions": ["评估调用失败,请检查 LLM 服务"],
  226. }