1
0

benchmark_index.py 1.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051
  1. """端到端索引构建 benchmark:解析 PDF → 构建索引,带分阶段计时"""
  2. import asyncio
  3. import glob
  4. import logging
  5. import time
  6. logging.basicConfig(
  7. level=logging.INFO,
  8. format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
  9. )
  10. from backend.tools.parse_pdf import parse_pdf
  11. from backend.tools.build_vector_index import build_vector_index
  12. async def main():
  13. pdfs = sorted(glob.glob("data/uploads/*.pdf"))
  14. if not pdfs:
  15. print("没有找到 PDF")
  16. return
  17. pdf_path = pdfs[0]
  18. print(f"\n=== 测试 PDF: {pdf_path} ===\n")
  19. # 1. 解析 PDF
  20. t0 = time.perf_counter()
  21. parsed = await parse_pdf(file_path=pdf_path)
  22. t_parse = time.perf_counter()
  23. print(f"\n>>> parse_pdf: {t_parse - t0:.2f}s")
  24. print(f" sections: {len(parsed.get('sections', []))}")
  25. total_chars = sum(len(s.get('content', '')) for s in parsed.get('sections', []))
  26. print(f" total content chars: {total_chars}")
  27. # 2. 构建索引(带去重检查)
  28. print(f"\n>>> 开始 build_vector_index...")
  29. t1 = time.perf_counter()
  30. result = await build_vector_index(papers_json=[parsed])
  31. t_index = time.perf_counter()
  32. print(f"\n>>> build_vector_index: {t_index - t1:.2f}s")
  33. print(f" result: {result}")
  34. # 3. 第二次(测试去重是否生效)
  35. print(f"\n>>> 第二次 build_vector_index(应命中去重)...")
  36. t2 = time.perf_counter()
  37. result2 = await build_vector_index(papers_json=[parsed])
  38. t_index2 = time.perf_counter()
  39. print(f"\n>>> build_vector_index (cached): {t_index2 - t2:.2f}s")
  40. print(f" result: {result2}")
  41. if __name__ == "__main__":
  42. asyncio.run(main())