opencode/packages/app/e2e/performance/timeline/session-timeline-stream-probe.ts

import type { Page } from "@playwright/test"

const STREAM_MARKER_PATTERN = "stream-(\\d+)"
const STREAM_FRAGMENT_COUNT = 18

type TimelineProbeState = {
  started: number
  ended: number
  profileVisual: boolean
  minimal: boolean
  frames: number[]
  frameAt: number[]
  applied: { at: number; index: number }[]
  geometry: {
    scrollTop: number
    scrollHeight: number
    clientHeight: number
    distance: number
    virtualHeight: number
    headerHeight: number
  }[]
  blanks: number
  longTasks: number[]
  layoutShifts: number[]
  visibleMounts: number
  visibleUnmounts: number
  visibleRows: Set<Element>
  visibleSubtreeMounts: string[]
  visibleSubtreeUnmounts: string[]
  visibleSubtreeReplacements: number
  visibleSubtreeDropouts: string[]
  visibleSubtrees: Map<string, Element>
  subtreeKeys: WeakMap<Element, string>
  maxOverlap: number
  maxGap: number
  maxPartTopMovement: number
  previousPartTop: number
  slowFrames: {
    duration: number
    index: number
    phase: "stream" | "boundary" | "complete" | "unknown"
    tokenSpans: number
    blocks: number
    codeBlocks: number
    height: number
    distance: number
  }[]
  scroll: {
    calls: number
    callNoops: number
    sameFrameCalls: number
    assignments: number
    assignmentNoops: number
    lastCallFrame: number
    frame: number
  }
  row: HTMLElement
  markdown: HTMLElement
  running: boolean
  previous: number
  cleanup: () => void
  start: () => void
}

export async function installTimelineStreamProbe(
  page: Page,
  options: { textPartID: string; finalIndex: number; profileVisual: boolean; minimal: boolean },
) {
  await page.evaluate(
    ({ textPartID, finalIndex, profileVisual, minimal, markerPattern, fragmentCount }) => {
      const part = document.querySelector<HTMLElement>(`[data-timeline-part-id="${textPartID}"]`)
      const row = part?.closest<HTMLElement>("[data-timeline-row]")
      const markdown = part?.querySelector<HTMLElement>('[data-component="markdown"]')
      const root = part?.closest<HTMLElement>(".scroll-view__viewport")
      if (!part || !row || !markdown || !root) throw new Error("missing streaming benchmark nodes")
      const viewport = root.getBoundingClientRect()
      const state: TimelineProbeState = {
        started: 0,
        ended: Infinity,
        profileVisual,
        minimal,
        frames: [],
        frameAt: [],
        applied: [],
        geometry: [],
        blanks: 0,
        longTasks: [],
        layoutShifts: [],
        visibleMounts: 0,
        visibleUnmounts: 0,
        visibleRows: new Set(
          [...root.querySelectorAll("[data-timeline-key]")].filter((element) => {
            const rect = element.getBoundingClientRect()
            return rect.bottom > viewport.top && rect.top < viewport.bottom
          }),
        ),
        visibleSubtreeMounts: [],
        visibleSubtreeUnmounts: [],
        visibleSubtreeReplacements: 0,
        visibleSubtreeDropouts: [],
        visibleSubtrees: new Map<string, Element>(),
        subtreeKeys: new WeakMap<Element, string>(),
        maxOverlap: 0,
        maxGap: 0,
        maxPartTopMovement: 0,
        previousPartTop: part.getBoundingClientRect().top,
        slowFrames: [],
        scroll: {
          calls: 0,
          callNoops: 0,
          sameFrameCalls: 0,
          assignments: 0,
          assignmentNoops: 0,
          lastCallFrame: -1,
          frame: 0,
        },
        row,
        markdown,
        running: false,
        previous: 0,
        cleanup: () => {},
        start: () => {},
      }
      ;(window as Window & { __timelineStreamBenchmark?: TimelineProbeState }).__timelineStreamBenchmark = state
      const scrollTo = Element.prototype.scrollTo
      const scrollTop = Object.getOwnPropertyDescriptor(Element.prototype, "scrollTop")!
      if (profileVisual) {
        Element.prototype.scrollTo = function (...args) {
          state.scroll.calls += 1
          const top = typeof args[0] === "object" ? args[0]?.top : args[1]
          if (typeof top === "number") {
            const target = Math.min(top, this.scrollHeight - this.clientHeight)
            if (Math.abs(this.scrollTop - target) < 1) state.scroll.callNoops += 1
          }
          if (state.scroll.lastCallFrame === state.scroll.frame) state.scroll.sameFrameCalls += 1
          state.scroll.lastCallFrame = state.scroll.frame
          return scrollTo.apply(this, args)
        }
        Object.defineProperty(Element.prototype, "scrollTop", {
          configurable: true,
          get: scrollTop.get,
          set(value) {
            state.scroll.assignments += 1
            if (Math.abs(this.scrollTop - value) < 1) state.scroll.assignmentNoops += 1
            scrollTop.set!.call(this, value)
          },
        })
      }

      const recordLongTasks = (entries: PerformanceEntry[]) => {
        if (!state.running) return
        state.longTasks.push(
          ...entries
            .filter((entry) => entry.startTime >= state.started && entry.startTime <= state.ended)
            .map((entry) => entry.duration),
        )
      }
      const longTaskObserver = new PerformanceObserver((list) => recordLongTasks(list.getEntries()))
      longTaskObserver.observe({ type: "longtask" })
      const recordLayoutShifts = (entries: PerformanceEntry[]) => {
        if (!state.running) return
        state.layoutShifts.push(
          ...entries
            .map((entry) => {
              const shift = entry as LayoutShiftEntry
              if (shift.startTime < state.started || shift.hadRecentInput) return
              return shift.value
            })
            .filter((value): value is number => value !== undefined),
        )
      }
      const layoutShiftObserver = profileVisual
        ? new PerformanceObserver((list) => recordLayoutShifts(list.getEntries()))
        : undefined
      layoutShiftObserver?.observe({ type: "layout-shift", buffered: true })

      const visible = (element: Element) => {
        const rect = element.getBoundingClientRect()
        const viewport = root.getBoundingClientRect()
        const style = getComputedStyle(element)
        return (
          element.isConnected &&
          rect.width > 0 &&
          rect.height > 0 &&
          rect.bottom > viewport.top &&
          rect.top < viewport.bottom &&
          style.display !== "none" &&
          style.visibility !== "hidden" &&
          Number(style.opacity) > 0
        )
      }
      const critical = [
        "[data-timeline-part-id]",
        '[data-component="edit-content"]',
        '[data-component="apply-patch-file-diff"]',
        '[data-component="file"]',
        '[data-component="markdown-code"]',
        "[data-markdown-block]",
      ].join(",")
      const describe = (element: Element) => {
        const cached = state.subtreeKeys.get(element)
        if (!element.isConnected && cached) return cached
        const part = element.closest<HTMLElement>("[data-timeline-part-id]")?.dataset.timelinePartId ?? "unknown"
        const block = element
          .closest<HTMLElement>("[data-markdown-key]")
          ?.dataset.markdownKey?.replace(/:(?:code|full|live)$/, "")
        const component =
          element.getAttribute("data-component") ?? element.getAttribute("data-markdown-block") ?? element.tagName
        const key = `${part}:${block ?? "root"}:${component}`
        state.subtreeKeys.set(element, key)
        return key
      }
      const recordMutations = (records: MutationRecord[]) => {
        if (!state.running) return
        records.forEach((record) => {
          record.addedNodes.forEach((node) => {
            if (node instanceof HTMLElement && node.matches("[data-timeline-key]") && visible(node)) {
              state.visibleMounts += 1
              state.visibleRows.add(node)
            }
            if (!(node instanceof Element)) return
            const added = [node, ...node.querySelectorAll(critical)].filter((element) => element.matches(critical))
            added.forEach((element) => {
              if (visible(element)) state.visibleSubtreeMounts.push(describe(element))
            })
          })
          record.removedNodes.forEach((node) => {
            if (node instanceof HTMLElement && node.matches("[data-timeline-key]") && state.visibleRows.delete(node))
              state.visibleUnmounts += 1
            if (!(node instanceof Element)) return
            const removed = [node, ...node.querySelectorAll(critical)].filter((element) => element.matches(critical))
            removed.forEach((element) => {
              const key = describe(element)
              if (state.visibleSubtrees.get(key) === element) state.visibleSubtreeUnmounts.push(key)
            })
          })
        })
      }
      const mutationObserver = profileVisual ? new MutationObserver(recordMutations) : undefined
      mutationObserver?.observe(root, { childList: true, subtree: true })
      const currentPart = () => root.querySelector<HTMLElement>(`[data-timeline-part-id="${textPartID}"]`)
      const observeProgress = (at: number) => {
        if (!state.running) return
        const content = currentPart()?.textContent ?? ""
        const index = content.includes("benchmark-complete")
          ? finalIndex
          : Number(content.match(new RegExp(markerPattern, "g"))?.at(-1)?.match(/\d+/)?.[0] ?? -1)
        if (index >= 0 && index !== state.applied.at(-1)?.index) state.applied.push({ at, index })
      }
      const progressObserver = new MutationObserver(() => observeProgress(performance.now()))
      progressObserver.observe(root, { characterData: true, childList: true, subtree: true })
      state.cleanup = () => {
        recordLongTasks(longTaskObserver.takeRecords())
        recordLayoutShifts(layoutShiftObserver?.takeRecords() ?? [])
        recordMutations(mutationObserver?.takeRecords() ?? [])
        if (progressObserver.takeRecords().length) observeProgress(performance.now())
        longTaskObserver.disconnect()
        layoutShiftObserver?.disconnect()
        mutationObserver?.disconnect()
        progressObserver.disconnect()
        if (!profileVisual) return
        Element.prototype.scrollTo = scrollTo
        Object.defineProperty(Element.prototype, "scrollTop", scrollTop)
      }

      const sample = (now: number) => {
        if (!state.running) return
        state.frameAt.push(now)
        observeProgress(now)
        if (minimal) {
          state.frames.push(now - state.previous)
          state.previous = now
          requestAnimationFrame(sample)
          return
        }
        setTimeout(() => {
          if (!state.running) return
          state.scroll.frame += 1
          const duration = now - state.previous
          state.frames.push(duration)
          state.previous = now
          const virtualRoot = root.querySelector<HTMLElement>("[data-timeline-virtual-content]")
          const header = root.querySelector<HTMLElement>("[data-session-title]")
          state.geometry.push({
            scrollTop: root.scrollTop,
            scrollHeight: root.scrollHeight,
            clientHeight: root.clientHeight,
            distance: root.scrollHeight - root.clientHeight - root.scrollTop,
            virtualHeight: virtualRoot?.getBoundingClientRect().height ?? 0,
            headerHeight: header?.getBoundingClientRect().height ?? 0,
          })
          const viewport = root.getBoundingClientRect()
          if (profileVisual) {
            const visibleRows = [...root.querySelectorAll<HTMLElement>("[data-timeline-key]")]
              .map((element) => ({ element, rect: element.getBoundingClientRect() }))
              .filter((item) => item.rect.bottom > viewport.top && item.rect.top < viewport.bottom)
              .sort((a, b) => a.rect.top - b.rect.top)
            state.visibleRows = new Set(visibleRows.map((item) => item.element))
            const rows = visibleRows.map((item) => item.rect)
            rows.slice(1).forEach((rect, index) => {
              const previous = rows[index]!
              state.maxOverlap = Math.max(state.maxOverlap, previous.bottom - rect.top)
              state.maxGap = Math.max(state.maxGap, rect.top - previous.bottom)
            })
            const partTop = part.getBoundingClientRect().top
            state.maxPartTopMovement = Math.max(state.maxPartTopMovement, Math.abs(partTop - state.previousPartTop))
            state.previousPartTop = partTop
          }
          const visibleRow = [...root.querySelectorAll<HTMLElement>("[data-timeline-row]")].some((element) => {
            const rect = element.getBoundingClientRect()
            return rect.bottom > viewport.top && rect.top < viewport.bottom
          })
          if (!visibleRow) state.blanks += 1
          if (profileVisual) {
            const subtrees = new Map<string, { element: Element; rendered: boolean }>()
            const visibleSubtrees = new Map<string, Element>()
            root.querySelectorAll(critical).forEach((element) => {
              const key = describe(element)
              const rect = element.getBoundingClientRect()
              const style = getComputedStyle(element)
              const rendered =
                element.isConnected &&
                rect.width > 0 &&
                rect.height > 0 &&
                style.display !== "none" &&
                style.visibility !== "hidden" &&
                Number(style.opacity) > 0
              subtrees.set(key, { element, rendered })
              if (rendered && rect.bottom > viewport.top && rect.top < viewport.bottom) {
                const previous = state.visibleSubtrees.get(key)
                if (previous && previous !== element && key.startsWith(`${textPartID}:`))
                  state.visibleSubtreeReplacements += 1
                visibleSubtrees.set(key, element)
              }
            })
            state.visibleSubtrees.forEach((element, key) => {
              const current = subtrees.get(key)
              if (key.startsWith(`${textPartID}:`) && !current?.rendered) {
                const markdown = part.querySelector<HTMLElement>('[data-component="markdown"]')
                state.visibleSubtreeDropouts.push(
                  `${key}:projection=${markdown?.dataset.markdownProjectionLength}/${markdown?.dataset.markdownProjectionBlocks}:result=${markdown?.dataset.markdownResultLength}/${markdown?.dataset.markdownResultBlocks}:applied=${markdown?.dataset.markdownAppliedBlocks}:dom=${markdown?.children.length}`,
                )
              }
              if (element.matches('[data-component="file"]')) {
                const hadLines = element.hasAttribute("data-profiler-had-lines")
                const hasLines = element.shadowRoot?.querySelector("[data-line]") != null
                if (hasLines) element.setAttribute("data-profiler-had-lines", "")
                if (hadLines && !hasLines) state.visibleSubtreeDropouts.push(`${key}:shadow-lines`)
              }
            })
            state.visibleSubtrees = visibleSubtrees
          }
          if (profileVisual && duration > 33.34) {
            const livePart = currentPart()
            const content = livePart?.textContent ?? ""
            const complete = content.includes("benchmark-complete")
            const index = complete
              ? finalIndex
              : Number(content.match(new RegExp(markerPattern, "g"))?.at(-1)?.match(/\d+/)?.[0] ?? -1)
            state.slowFrames.push({
              duration,
              index,
              phase: complete
                ? "complete"
                : index >= 0 && index % fragmentCount === 0
                  ? "boundary"
                  : index >= 0
                    ? "stream"
                    : "unknown",
              tokenSpans: livePart?.querySelectorAll(".shiki span").length ?? 0,
              blocks: livePart?.querySelectorAll("[data-markdown-block]").length ?? 0,
              codeBlocks: livePart?.querySelectorAll('[data-component="markdown-code"]').length ?? 0,
              height: livePart?.getBoundingClientRect().height ?? 0,
              distance: root.scrollHeight - root.clientHeight - root.scrollTop,
            })
          }
          requestAnimationFrame(sample)
        }, 0)
      }
      state.start = () => {
        state.started = performance.now()
        state.previous = state.started
        state.running = true
        requestAnimationFrame(sample)
      }
    },
    { ...options, markerPattern: STREAM_MARKER_PATTERN, fragmentCount: STREAM_FRAGMENT_COUNT },
  )
}

export function startTimelineStreamProbe(page: Page) {
  return page.evaluate(() => {
    const state = (window as Window & { __timelineStreamBenchmark?: TimelineProbeState }).__timelineStreamBenchmark
    if (!state) throw new Error("missing streaming benchmark state")
    state.start()
  })
}

type LayoutShiftEntry = PerformanceEntry & { value: number; hadRecentInput?: boolean }

export function layoutShiftValue(
  entry: Pick<LayoutShiftEntry, "startTime" | "value" | "hadRecentInput">,
  start: number,
) {
  if (entry.startTime < start || entry.hadRecentInput) return
  return entry.value
}

export function removeVisibleRow<T>(visible: Set<T>, row: T) {
  return visible.delete(row)
}

export function streamProgress(content: string) {
  const index = Number(content.match(new RegExp(STREAM_MARKER_PATTERN, "g"))?.at(-1)?.match(/\d+/)?.[0] ?? -1)
  return {
    index,
    phase: content.includes("benchmark-complete")
      ? ("complete" as const)
      : index >= 0 && index % STREAM_FRAGMENT_COUNT === 0
        ? ("boundary" as const)
        : index >= 0
          ? ("stream" as const)
          : ("unknown" as const),
  }
}

export async function collectTimelineStreamMetrics(
  page: Page,
  options: { textPartID: string; finalIndex: number; navigations: string[] },
) {
  return page.evaluate(({ textPartID, finalIndex, navigations }) => {
    const state = (window as Window & { __timelineStreamBenchmark?: TimelineProbeState }).__timelineStreamBenchmark
    if (!state) throw new Error(`missing streaming benchmark state after navigation: ${JSON.stringify(navigations)}`)
    state.ended = performance.now()
    state.cleanup()
    state.running = false
    const part = document.querySelector<HTMLElement>(`[data-timeline-part-id="${textPartID}"]`)
    const row = part?.closest<HTMLElement>("[data-timeline-row]")
    const markdown = part?.querySelector<HTMLElement>('[data-component="markdown"]')
    const sorted = state.frames.slice().sort((a, b) => a - b)
    const duration = state.frames.reduce((sum, value) => sum + value, 0)
    const longestSlowStreak = state.frames.reduce(
      (result, value) => {
        const current = value > 33.34 ? result.current + 1 : 0
        return { current, longest: Math.max(result.longest, current) }
      },
      { current: 0, longest: 0 },
    ).longest
    const busyStart = state.applied.at(0)?.at
    const completion = state.applied.find((value) => value.index === finalIndex)
    const busyEnd = completion?.at
    const busyFrames =
      busyStart === undefined || busyEnd === undefined
        ? []
        : state.frames.filter((_, index) => state.frameAt[index]! >= busyStart && state.frameAt[index]! <= busyEnd)
    const busySorted = busyFrames.slice().sort((a, b) => a - b)
    const busyDuration = busyFrames.reduce((sum, value) => sum + value, 0)
    const completionObservedMs = (completion?.at ?? NaN) - state.started
    const visual = state.profileVisual
      ? {
          layoutShiftValueSum: state.layoutShifts.reduce((sum, value) => sum + value, 0),
          maxLayoutShiftValue: Math.max(0, ...state.layoutShifts),
          visibleMounts: state.visibleMounts,
          visibleUnmounts: state.visibleUnmounts,
          visibleSubtreeMounts: state.visibleSubtreeMounts,
          visibleSubtreeUnmounts: [...new Set(state.visibleSubtreeUnmounts)],
          visibleSubtreeReplacements: state.visibleSubtreeReplacements,
          visibleSubtreeDropouts: [...new Set(state.visibleSubtreeDropouts)],
          maxOverlapPx: state.maxOverlap,
          maxGapPx: state.maxGap,
          maxPartTopMovementPx: state.maxPartTopMovement,
          slowestRafGaps: state.slowFrames
            .sort((a, b) => b.duration - a.duration)
            .slice(0, 20)
            .map((frame) => ({
              durationMs: frame.duration,
              index: frame.index,
              phase: frame.phase,
              tokenSpans: frame.tokenSpans,
              blocks: frame.blocks,
              codeBlocks: frame.codeBlocks,
              heightPx: frame.height,
              distancePx: frame.distance,
            })),
          slowRafGapPhases: Object.fromEntries(
            ["stream", "boundary", "complete", "unknown"].map((phase) => {
              const frames = state.slowFrames.filter((frame) => frame.phase === phase)
              return [
                phase,
                {
                  count: frames.length,
                  totalMs: frames.reduce((sum, frame) => sum + frame.duration, 0),
                  maxMs: Math.max(0, ...frames.map((frame) => frame.duration)),
                },
              ]
            }),
          ),
          scroll: state.scroll,
        }
      : null
    const geometry = state.minimal
      ? null
      : {
          maxDistancePx: Math.max(0, ...state.geometry.map((sample) => sample.distance)),
          finalDistancePx: state.geometry.at(-1)?.distance ?? 0,
          final: state.geometry.at(-1),
          distanceTransitionsPx: state.geometry
            .map((sample) => Math.round(sample.distance))
            .filter((value, index, values) => index === 0 || value !== values[index - 1]),
          bottomDriftTransitions: state.geometry.slice(1).filter((value, index) => {
            const previous = state.geometry[index]?.distance ?? 0
            return previous <= 1 && value.distance > 1
          }).length,
          blankSamples: state.blanks,
        }
    return {
      capabilities: { visual: state.profileVisual, geometry: !state.minimal },
      completionObservedMs,
      deltasPerSecond: Number.isFinite(completionObservedMs) ? finalIndex / (completionObservedMs / 1_000) : null,
      rafGapSamples: state.frames.length,
      rafCallbackRate: duration ? (state.frames.length * 1000) / duration : 0,
      observedProgressWindowRafCallbackRate: busyDuration ? (busyFrames.length * 1000) / busyDuration : null,
      observedProgressWindowRafGapP95Ms: busySorted[Math.floor(busySorted.length * 0.95)] ?? null,
      observedProgressWindowRafGaps: busyFrames.length,
      maxObservedProgressIndex: Math.max(-1, ...state.applied.map((value) => value.index)),
      observedProgressTransitions: state.applied.length,
      rafGapP50Ms: sorted[Math.floor(sorted.length * 0.5)] ?? 0,
      rafGapP95Ms: sorted[Math.floor(sorted.length * 0.95)] ?? 0,
      rafGapP99Ms: sorted[Math.floor(sorted.length * 0.99)] ?? 0,
      maxRafGapMs: sorted.at(-1) ?? 0,
      rafGapsOver33Ms: state.frames.filter((value) => value > 33.34).length,
      rafGapsOver50Ms: state.frames.filter((value) => value > 50).length,
      missedFrameBudgetEquivalents: state.frames.reduce(
        (sum, value) => sum + Math.max(0, Math.round(value / 16.67) - 1),
        0,
      ),
      longestRafGapOver33MsStreak: longestSlowStreak,
      longTaskCount: state.longTasks.length,
      longTaskTimeMs: state.longTasks.reduce((sum, value) => sum + value, 0),
      visual,
      geometry,
      rowReplaced: row !== state.row,
      markdownReplaced: markdown !== state.markdown,
      domTextCharacters: part?.textContent?.length ?? 0,
    }
  }, options)
}