session-timeline-benchmark.fixture.ts 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488
  1. import { base64Encode } from "@opencode-ai/core/util/encode"
  2. import type { Page } from "@playwright/test"
  3. import { mockOpenCodeServer } from "../../utils/mock-server"
  4. import { expectAppVisible, expectSessionTitle } from "../../utils/waits"
  5. import { expect } from "../benchmark"
  6. const directory = "C:/OpenCode/TimelineStateRegression"
  7. const projectID = "proj_timeline_state_regression"
  8. const sessionID = "ses_timeline_state_regression"
  9. const userMessageID = "msg_user_regression"
  10. const assistantMessageID = "msg_assistant_regression"
  11. const editPartID = "prt_0001_edit"
  12. export const textPartID = "prt_9999_text"
  13. const title = "Timeline collapse state regression"
  14. const model = { providerID: "opencode", modelID: "claude-opus-4-6", variant: "max" }
  15. type EventPayload = {
  16. directory: string
  17. payload: Record<string, unknown>
  18. }
  19. const userMessage = {
  20. info: {
  21. id: userMessageID,
  22. sessionID,
  23. role: "user",
  24. time: { created: 1700000000000 },
  25. summary: { diffs: [] },
  26. agent: "build",
  27. model,
  28. },
  29. parts: [
  30. {
  31. id: "prt_user_text",
  32. sessionID,
  33. messageID: userMessageID,
  34. type: "text",
  35. text: "Please edit the file.",
  36. },
  37. ],
  38. }
  39. const editPart = {
  40. id: editPartID,
  41. sessionID,
  42. messageID: assistantMessageID,
  43. type: "tool",
  44. callID: "call_edit_regression",
  45. tool: "edit",
  46. state: {
  47. status: "completed",
  48. input: { filePath: "src/regression.ts" },
  49. output: "Edited src/regression.ts",
  50. title: "src/regression.ts",
  51. metadata: {
  52. filediff: {
  53. file: "src/regression.ts",
  54. additions: 1,
  55. deletions: 1,
  56. before: "export const value = 'before'\n",
  57. after: "export const value = 'after'\n",
  58. },
  59. diff: "diff --git a/src/regression.ts b/src/regression.ts\n-export const value = 'before'\n+export const value = 'after'\n",
  60. },
  61. time: { start: 1700000001000, end: 1700000002000 },
  62. },
  63. }
  64. const streamedTextPart = {
  65. id: textPartID,
  66. sessionID,
  67. messageID: assistantMessageID,
  68. type: "text",
  69. text: "Streaming added a later assistant text part.",
  70. }
  71. const assistantMessage = {
  72. info: {
  73. id: assistantMessageID,
  74. sessionID,
  75. role: "assistant",
  76. time: { created: 1700000001000 },
  77. parentID: userMessageID,
  78. modelID: model.modelID,
  79. providerID: model.providerID,
  80. mode: "build",
  81. agent: "build",
  82. path: { cwd: directory, root: directory },
  83. cost: 0.01,
  84. tokens: { input: 100, output: 200, reasoning: 0, cache: { read: 0, write: 0 } },
  85. variant: "max",
  86. },
  87. parts: [editPart],
  88. }
  89. export async function setupTimelineBenchmark(page: Page, options: { historyTurns: number; eventBatch: number }) {
  90. const events: EventPayload[] = []
  91. let eventBatch = options.eventBatch
  92. await mockOpenCodeServer(page, {
  93. directory,
  94. project: project(),
  95. provider: provider(),
  96. sessions: [session()],
  97. pageMessages: () => ({
  98. items: [
  99. ...Array.from({ length: options.historyTurns }, (_, index) => performanceTurn(index)).flat(),
  100. userMessage,
  101. assistantMessage,
  102. ],
  103. }),
  104. events: () => events.splice(0, eventBatch),
  105. eventRetry: 16,
  106. })
  107. await page.addInitScript(() => {
  108. localStorage.setItem(
  109. "settings.v3",
  110. JSON.stringify({
  111. general: {
  112. editToolPartsExpanded: true,
  113. shellToolPartsExpanded: true,
  114. showReasoningSummaries: true,
  115. showSessionProgressBar: true,
  116. },
  117. }),
  118. )
  119. })
  120. await page.setViewportSize({ width: 1366, height: 768 })
  121. const scroller = page.locator(".scroll-view__viewport", { has: page.locator("[data-timeline-row]") })
  122. const text = page.locator(`[data-timeline-part-id="${textPartID}"]`).first()
  123. await page.goto(`/${base64Encode(directory)}/session/${sessionID}`)
  124. await expectSessionTitle(page, title)
  125. await expectAppVisible(scroller)
  126. return {
  127. scroller,
  128. text,
  129. transport: {
  130. enqueue(payload: EventPayload | EventPayload[]) {
  131. events.push(...(Array.isArray(payload) ? payload : [payload]))
  132. },
  133. pendingCount() {
  134. return events.length
  135. },
  136. releaseAll() {
  137. eventBatch = events.length
  138. },
  139. },
  140. async scrollToBottom() {
  141. await scroller.evaluate((element) => {
  142. element.scrollTop = element.scrollHeight
  143. })
  144. },
  145. async waitForStableGeometry() {
  146. await expect
  147. .poll(() => scroller.evaluate((element) => element.scrollHeight - element.clientHeight - element.scrollTop))
  148. .toBeLessThanOrEqual(1)
  149. await page.waitForFunction((partID) => {
  150. const root = [...document.querySelectorAll<HTMLElement>(".scroll-view__viewport")].find((element) =>
  151. element.querySelector(`[data-timeline-part-id="${partID}"]`),
  152. )
  153. if (!root) return false
  154. return new Promise<boolean>((resolve) => {
  155. const height = root.scrollHeight
  156. requestAnimationFrame(() =>
  157. requestAnimationFrame(() =>
  158. resolve(root.scrollHeight === height && root.scrollHeight - root.clientHeight - root.scrollTop <= 1),
  159. ),
  160. )
  161. })
  162. }, textPartID)
  163. },
  164. }
  165. }
  166. export function buildInitialStreamEvent(deltaCount: number): EventPayload {
  167. return {
  168. directory,
  169. payload: {
  170. type: "message.part.updated",
  171. properties: {
  172. part: {
  173. ...streamedTextPart,
  174. text: `Streaming${streamChunk(0, deltaCount + 1)}\n\n\`\`\`ts\nconst initial = true\n\`\`\``,
  175. },
  176. },
  177. },
  178. }
  179. }
  180. export function buildStreamDeltaEvents(deltaCount: number): EventPayload[] {
  181. return Array.from({ length: deltaCount }, (_, index) => ({
  182. directory,
  183. payload: {
  184. type: "message.part.delta",
  185. properties: {
  186. messageID: assistantMessageID,
  187. partID: textPartID,
  188. field: "text",
  189. delta: streamChunk(index + 1, deltaCount + 1),
  190. },
  191. },
  192. }))
  193. }
  194. function performanceTurn(index: number) {
  195. const suffix = String(index).padStart(4, "0")
  196. const userID = `msg_0000_${suffix}_a_user`
  197. const assistantID = `msg_0000_${suffix}_b_assistant`
  198. const before = historicalSource(index, false)
  199. const after = historicalSource(index, true)
  200. const parts = [
  201. ...(index % 5 === 0
  202. ? [
  203. {
  204. id: `prt_0000_${suffix}_reasoning`,
  205. sessionID,
  206. messageID: assistantID,
  207. type: "reasoning",
  208. text: `Reviewing the existing implementation. ${"constraint analysis ".repeat(20)}`,
  209. time: { start: 1690000001000 + index * 2_000, end: 1690000001200 + index * 2_000 },
  210. },
  211. ]
  212. : []),
  213. {
  214. id: `prt_0000_${suffix}_assistant`,
  215. sessionID,
  216. messageID: assistantID,
  217. type: "text",
  218. text: historicalMarkdown(index),
  219. },
  220. ...(index % 8 === 0
  221. ? [
  222. {
  223. id: `prt_0000_${suffix}_edit`,
  224. sessionID,
  225. messageID: assistantID,
  226. type: "tool",
  227. callID: `call_0000_${suffix}_edit`,
  228. tool: "edit",
  229. state: {
  230. status: "completed",
  231. input: { filePath: `src/history-${index}.ts` },
  232. output: `Edited src/history-${index}.ts`,
  233. title: `src/history-${index}.ts`,
  234. metadata: {
  235. filediff: { file: `src/history-${index}.ts`, additions: 48, deletions: 48, before, after },
  236. },
  237. time: { start: 1690000001200 + index * 2_000, end: 1690000001400 + index * 2_000 },
  238. },
  239. },
  240. ]
  241. : []),
  242. ...(index % 12 === 0
  243. ? [
  244. {
  245. id: `prt_0000_${suffix}_write`,
  246. sessionID,
  247. messageID: assistantID,
  248. type: "tool",
  249. callID: `call_0000_${suffix}_write`,
  250. tool: "write",
  251. state: {
  252. status: "completed",
  253. input: { filePath: `src/generated-${index}.tsx`, content: after },
  254. output: `Wrote src/generated-${index}.tsx`,
  255. title: `src/generated-${index}.tsx`,
  256. metadata: {
  257. filediff: { file: `src/generated-${index}.tsx`, additions: 32, deletions: 0, before: "", after },
  258. },
  259. time: { start: 1690000001400 + index * 2_000, end: 1690000001500 + index * 2_000 },
  260. },
  261. },
  262. ]
  263. : []),
  264. ...(index % 16 === 0
  265. ? [
  266. {
  267. id: `prt_0000_${suffix}_patch`,
  268. sessionID,
  269. messageID: assistantID,
  270. type: "tool",
  271. callID: `call_0000_${suffix}_patch`,
  272. tool: "apply_patch",
  273. state: {
  274. status: "completed",
  275. input: { patchText: realisticPatch(index) },
  276. output: "Success. Updated src/components/SessionCard.tsx",
  277. title: "src/components/SessionCard.tsx",
  278. metadata: {
  279. files: [
  280. {
  281. filePath: "src/components/SessionCard.tsx",
  282. relativePath: "src/components/SessionCard.tsx",
  283. type: "update",
  284. additions: 8,
  285. deletions: 3,
  286. patch: realisticPatch(index),
  287. before,
  288. after,
  289. },
  290. ],
  291. },
  292. time: { start: 1690000001500 + index * 2_000, end: 1690000001700 + index * 2_000 },
  293. },
  294. },
  295. ]
  296. : []),
  297. ]
  298. return [
  299. {
  300. info: {
  301. id: userID,
  302. sessionID,
  303. role: "user",
  304. time: { created: 1690000000000 + index * 2_000 },
  305. summary: { diffs: [] },
  306. agent: "build",
  307. model,
  308. },
  309. parts: [
  310. {
  311. id: `prt_0000_${suffix}_user`,
  312. sessionID,
  313. messageID: userID,
  314. type: "text",
  315. text: `Historical prompt ${index}`,
  316. },
  317. ],
  318. },
  319. {
  320. info: {
  321. id: assistantID,
  322. sessionID,
  323. role: "assistant",
  324. time: { created: 1690000001000 + index * 2_000, completed: 1690000001500 + index * 2_000 },
  325. parentID: userID,
  326. modelID: model.modelID,
  327. providerID: model.providerID,
  328. mode: "build",
  329. agent: "build",
  330. path: { cwd: directory, root: directory },
  331. cost: 0.01,
  332. tokens: { input: 100, output: 200, reasoning: 0, cache: { read: 0, write: 0 } },
  333. variant: "max",
  334. finish: "stop",
  335. },
  336. parts,
  337. },
  338. ]
  339. }
  340. function historicalMarkdown(index: number) {
  341. const code = `import { For, Show, createSignal } from "solid-js"
  342. type SessionRow = { id: string; title: string; active: boolean }
  343. export function SessionList(props: { rows: SessionRow[] }) {
  344. const [selected, setSelected] = createSignal<string>()
  345. return (
  346. <section aria-label="Sessions">
  347. <For each={props.rows}>{(row) => (
  348. <button classList={{ active: row.active }} onClick={() => setSelected(row.id)}>
  349. <Show when={selected() === row.id} fallback={row.title}>{row.title.toUpperCase()}</Show>
  350. </button>
  351. )}</For>
  352. </section>
  353. )
  354. }`
  355. return `## Session renderer review ${index}
  356. The active session keeps **semantic row identity** while reconciling measured content. See [Solid documentation](https://docs.solidjs.com/) and the inline \`measureElement(node)\` call.
  357. | Concern | Current behavior | Verification |
  358. | --- | --- | --- |
  359. | streaming | appends Markdown blocks | painted frames |
  360. | geometry | anchors visible rows | DOM coordinates |
  361. | tools | preserves expanded state | keyed remount probe |
  362. > Long sessions combine Markdown, syntax highlighting, tool output, and asynchronously rendered diffs.
  363. ${index % 4 === 0 ? `\`\`\`tsx\n${code}\n\`\`\`\n\n\`\`\`bash\nbun typecheck\nbun test --preload ./happydom.ts ./src/pages/session\ngit diff --check\n\`\`\`` : "- preserve the viewport anchor\n- avoid replacing stable Markdown nodes\n- process provider deltas without blocking input"}`
  364. }
  365. function historicalSource(index: number, updated: boolean) {
  366. const method = updated ? "toLocaleUpperCase(props.locale)" : "toUpperCase()"
  367. const limit = updated ? 24 : 20
  368. return `import { createMemo, For } from "solid-js"
  369. type Message = {
  370. id: string
  371. role: "user" | "assistant"
  372. text: string
  373. tokens: { input: number; output: number }
  374. }
  375. export function MessageSummary(props: { messages: Message[]; locale: string }) {
  376. const visible = createMemo(() => props.messages.filter((message) => message.text.trim()).slice(-${limit}))
  377. const total = createMemo(() => visible().reduce((sum, message) => sum + message.tokens.output, 0))
  378. return (
  379. <article data-session-index="${index}">
  380. <header>{total().toLocaleString(props.locale)} output tokens</header>
  381. <For each={visible()}>{(message) => <p data-role={message.role}>{message.text.${method}}</p>}</For>
  382. </article>
  383. )
  384. }
  385. `
  386. }
  387. function realisticPatch(index: number) {
  388. return `*** Begin Patch
  389. *** Update File: src/components/SessionCard.tsx
  390. @@
  391. -const title = props.session.title.toUpperCase()
  392. -const messages = props.messages.slice(-20)
  393. +const title = props.session.title.toLocaleUpperCase(props.locale)
  394. +const messages = props.messages.filter((message) => message.text.trim()).slice(-24)
  395. +const outputTokens = messages.reduce((sum, message) => sum + message.tokens.output, 0)
  396. @@
  397. - <h2>{title}</h2>
  398. + <h2 data-session-index="${index}">{title}</h2>
  399. + <span>{outputTokens.toLocaleString(props.locale)} output tokens</span>
  400. *** End Patch`
  401. }
  402. export function streamChunk(index: number, count: number) {
  403. if (index === 0) return `\n\n## Implementation plan\n\nStreaming **bold analysis`
  404. if (index === count - 1)
  405. return `\n\`\`\`\n\n## Verification\n\n- **Typecheck:** passed\n- **Timeline geometry:** stable\n- **Streaming output:** benchmark-complete <!-- stream-${index} -->`
  406. const section = Math.floor(index / 18) + 1
  407. const fragments = [
  408. ` continues across three`,
  409. ` or four word`,
  410. ` provider deltas and`,
  411. ` closes in this fragment**. <!-- stream-${index} -->\n\n`,
  412. `| Concern | State`,
  413. ` | Verification |\n|`,
  414. ` --- | ---`,
  415. ` | --- |\n|`,
  416. ` markdown | incremental |`,
  417. ` painted frames | <!-- stream-${index} -->\n\n`,
  418. `\`\`\`tsx\nconst row: SessionRow`,
  419. ` = rows[index] ??`,
  420. ` fallback\nconst title =`,
  421. ` row.title.toLocaleUpperCase(locale)\n`,
  422. `const selected = createMemo(()`,
  423. ` => row.id ===`,
  424. ` activeID()) // stream-${index}\n`,
  425. `// stream-${index}\n\`\`\`\n\n### Iteration ${section}\n\nStreaming **bold analysis`,
  426. ]
  427. return fragments[(index - 1) % fragments.length]!
  428. }
  429. function project() {
  430. return {
  431. id: projectID,
  432. worktree: directory,
  433. vcs: "git",
  434. name: "timeline-state-regression",
  435. time: { created: 1700000000000, updated: 1700000000000 },
  436. sandboxes: [],
  437. }
  438. }
  439. function session() {
  440. return {
  441. id: sessionID,
  442. slug: "timeline-state-regression",
  443. projectID,
  444. directory,
  445. title,
  446. version: "dev",
  447. time: { created: 1700000000000, updated: 1700000000000 },
  448. }
  449. }
  450. function provider() {
  451. return {
  452. all: [
  453. {
  454. id: "opencode",
  455. name: "OpenCode",
  456. models: { "claude-opus-4-6": { id: "claude-opus-4-6", name: "Claude Opus 4.6", limit: { context: 200_000 } } },
  457. },
  458. ],
  459. connected: ["opencode"],
  460. default: { providerID: "opencode", modelID: "claude-opus-4-6" },
  461. }
  462. }