test(acp): add compatibility baseline (#29222)

packages/opencode/test/cli/acp/acp-compatibility-baseline.test.ts

@@ -0,0 +1,322 @@
+import { describe, expect } from "bun:test"
+import type {
+  CloseSessionResponse,
+  InitializeResponse,
+  NewSessionResponse,
+  ResumeSessionResponse,
+  SessionNotification,
+  SetSessionConfigOptionResponse,
+} from "@agentclientprotocol/sdk"
+import { Effect } from "effect"
+import { mkdir } from "node:fs/promises"
+import path from "node:path"
+import { cliIt } from "../../lib/cli-process"
+import { testProviderConfig } from "../../lib/test-provider"
+import {
+  createAcpClient,
+  expectOk,
+  firstAlternateValue,
+  flattenSelectOptions,
+  selectConfigOption,
+} from "./acp-test-client"
+
+describe("opencode acp verifier compatibility baseline", () => {
+  cliIt.live(
+    "initialize advertises close and resume capabilities",
+    ({ opencode }) =>
+      Effect.gen(function* () {
+        const acp = createAcpClient(yield* opencode.acp())
+        const initialized = expectOk(
+          yield* acp.request<InitializeResponse>("initialize", {
+            protocolVersion: 1,
+          }),
+        )
+
+        expect(initialized.protocolVersion).toBe(1)
+        expect(initialized.agentCapabilities?.sessionCapabilities?.close).toEqual({})
+        expect(initialized.agentCapabilities?.sessionCapabilities?.resume).toEqual({})
+      }),
+    60_000,
+  )
+
+  cliIt.live(
+    "first session timing diagnostic stays bounded and returns model options",
+    ({ home, llm, opencode }) =>
+      Effect.gen(function* () {
+        const acp = createAcpClient(
+          yield* opencode.acp({
+            env: {
+              OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url)),
+            },
+          }),
+        )
+        const started = Date.now()
+        yield* acp.request<InitializeResponse>("initialize", {
+          protocolVersion: 1,
+          clientCapabilities: {},
+          clientInfo: { name: "opencode-local-acp-baseline", version: "0.1.0" },
+        })
+        const session = expectOk(
+          yield* acp.request<NewSessionResponse>("session/new", {
+            cwd: home,
+            mcpServers: [],
+          }),
+        )
+        const durationMs = Date.now() - started
+        expect(durationMs).toBeLessThan(15_000)
+
+        const model = selectConfigOption(session.configOptions, "model")
+        expect(model?.category).toBe("model")
+        expect(model?.currentValue).toBe("test/test-model")
+        expect(model ? flattenSelectOptions(model).length : 0).toBeGreaterThanOrEqual(2)
+      }),
+    60_000,
+  )
+
+  cliIt.live(
+    "warm newSession timing diagnostic stays bounded",
+    ({ home, llm, opencode }) =>
+      Effect.gen(function* () {
+        const acp = createAcpClient(
+          yield* opencode.acp({
+            env: {
+              OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url)),
+            },
+          }),
+        )
+        yield* acp.request<InitializeResponse>("initialize", { protocolVersion: 1 })
+        yield* acp.request<NewSessionResponse>("session/new", { cwd: home, mcpServers: [] })
+
+        const started = Date.now()
+        const session = expectOk(
+          yield* acp.request<NewSessionResponse>("session/new", {
+            cwd: home,
+            mcpServers: [],
+          }),
+        )
+        const durationMs = Date.now() - started
+        expect(durationMs).toBeLessThan(15_000)
+        expect(session.sessionId).toBeTruthy()
+      }),
+    60_000,
+  )
+
+  cliIt.live(
+    "model switch timing diagnostic updates currentValue",
+    ({ home, llm, opencode }) =>
+      Effect.gen(function* () {
+        const acp = createAcpClient(
+          yield* opencode.acp({
+            env: {
+              OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url)),
+            },
+          }),
+        )
+        yield* acp.request<InitializeResponse>("initialize", { protocolVersion: 1 })
+        const session = expectOk(yield* acp.request<NewSessionResponse>("session/new", { cwd: home, mcpServers: [] }))
+        const model = selectConfigOption(session.configOptions, "model")
+        expect(model).toBeDefined()
+        const nextModel = model
+          ? flattenSelectOptions(model).find((option) => option.value === "test/second-model")?.value
+          : undefined
+        expect(nextModel).toBe("test/second-model")
+
+        const started = Date.now()
+        const updated = expectOk(
+          yield* acp.request<SetSessionConfigOptionResponse>("session/set_config_option", {
+            sessionId: session.sessionId,
+            configId: "model",
+            value: nextModel,
+          }),
+        )
+        const durationMs = Date.now() - started
+
+        expect(durationMs).toBeLessThan(15_000)
+        expect(selectConfigOption(updated.configOptions, "model")?.currentValue).toBe(nextModel)
+      }),
+    60_000,
+  )
+
+  cliIt.live(
+    "effort option is listed for variant-capable models and can switch",
+    ({ home, llm, opencode }) =>
+      Effect.gen(function* () {
+        const acp = createAcpClient(
+          yield* opencode.acp({
+            env: {
+              OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url)),
+            },
+          }),
+        )
+        yield* acp.request<InitializeResponse>("initialize", { protocolVersion: 1 })
+        const session = expectOk(yield* acp.request<NewSessionResponse>("session/new", { cwd: home, mcpServers: [] }))
+        const effort = selectConfigOption(session.configOptions, "effort")
+        expect(effort?.category).toBe("thought_level")
+        const nextEffort = effort ? firstAlternateValue(effort) : undefined
+        expect(nextEffort).toBe("high")
+
+        const updated = expectOk(
+          yield* acp.request<SetSessionConfigOptionResponse>("session/set_config_option", {
+            sessionId: session.sessionId,
+            configId: "effort",
+            value: nextEffort,
+          }),
+        )
+
+        expect(selectConfigOption(updated.configOptions, "effort")?.currentValue).toBe(nextEffort)
+      }),
+    60_000,
+  )
+
+  cliIt.live(
+    "default test provider documents missing effort option when the model has no variants",
+    ({ home, llm, opencode }) =>
+      Effect.gen(function* () {
+        const acp = createAcpClient(
+          yield* opencode.acp({
+            env: {
+              OPENCODE_CONFIG_CONTENT: JSON.stringify(noVariantConfig(llm.url)),
+            },
+          }),
+        )
+        yield* acp.request<InitializeResponse>("initialize", { protocolVersion: 1 })
+        const session = expectOk(yield* acp.request<NewSessionResponse>("session/new", { cwd: home, mcpServers: [] }))
+
+        expect(selectConfigOption(session.configOptions, "model")?.currentValue).toBe("test/test-model")
+        expect(selectConfigOption(session.configOptions, "effort")).toBeUndefined()
+      }),
+    60_000,
+  )
+
+  cliIt.live(
+    "skill slash command timing diagnostic appears through available_commands_update",
+    ({ home, llm, opencode }) =>
+      Effect.gen(function* () {
+        const skills = path.join(home, "skills")
+        yield* Effect.promise(() => mkdir(path.join(skills, "verifier-skill"), { recursive: true }))
+        yield* Effect.promise(() => Bun.write(path.join(skills, "verifier-skill", "SKILL.md"), verifierSkill))
+        const acp = createAcpClient(
+          yield* opencode.acp({
+            env: {
+              OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url, skills)),
+            },
+          }),
+        )
+        yield* acp.request<InitializeResponse>("initialize", { protocolVersion: 1 })
+        const session = expectOk(yield* acp.request<NewSessionResponse>("session/new", { cwd: home, mcpServers: [] }))
+
+        const update = yield* acp.waitForNotification<SessionNotification>(
+          "session/update",
+          (params) =>
+            params.sessionId === session.sessionId &&
+            params.update.sessionUpdate === "available_commands_update" &&
+            params.update.availableCommands.some((command) => command.name === "verifier-skill"),
+        )
+
+        expect(update.params?.sessionId).toBe(session.sessionId)
+
+        const secondSession = expectOk(
+          yield* acp.request<NewSessionResponse>("session/new", { cwd: home, mcpServers: [] }),
+        )
+        const started = Date.now()
+        yield* acp.waitForNotification<SessionNotification>(
+          "session/update",
+          (params) =>
+            params.sessionId === secondSession.sessionId &&
+            params.update.sessionUpdate === "available_commands_update" &&
+            params.update.availableCommands.some((command) => command.name === "verifier-skill"),
+        )
+        const durationMs = Date.now() - started
+        expect(durationMs).toBeLessThan(15_000)
+      }),
+    60_000,
+  )
+
+  cliIt.live(
+    "close request succeeds for a live session",
+    ({ home, opencode }) =>
+      Effect.gen(function* () {
+        const acp = createAcpClient(yield* opencode.acp())
+        yield* acp.request<InitializeResponse>("initialize", { protocolVersion: 1 })
+        const session = expectOk(yield* acp.request<NewSessionResponse>("session/new", { cwd: home, mcpServers: [] }))
+
+        expectOk(yield* acp.request<CloseSessionResponse>("session/close", { sessionId: session.sessionId }))
+      }),
+    60_000,
+  )
+
+  cliIt.live(
+    "resume request succeeds for a created session",
+    ({ home, opencode }) =>
+      Effect.gen(function* () {
+        const acp = createAcpClient(yield* opencode.acp())
+        yield* acp.request<InitializeResponse>("initialize", { protocolVersion: 1 })
+        const session = expectOk(yield* acp.request<NewSessionResponse>("session/new", { cwd: home, mcpServers: [] }))
+
+        const resumed = expectOk(
+          yield* acp.request<ResumeSessionResponse>("session/resume", {
+            sessionId: session.sessionId,
+            cwd: home,
+            mcpServers: [],
+          }),
+        )
+        expect(resumed.configOptions?.length).toBeGreaterThan(0)
+      }),
+    60_000,
+  )
+})
+
+function verifierConfig(llmUrl: string, skills?: string) {
+  const config = testProviderConfig(llmUrl)
+  return {
+    ...config,
+    model: "test/test-model",
+    ...(skills ? { skills: { paths: [skills] } } : {}),
+    provider: {
+      test: {
+        ...config.provider.test,
+        models: {
+          "test-model": {
+            ...config.provider.test.models["test-model"],
+            variants: {
+              low: {},
+              high: {},
+            },
+          },
+          "second-model": {
+            ...config.provider.test.models["test-model"],
+            id: "second-model",
+            name: "Second Test Model",
+          },
+        },
+      },
+    },
+  }
+}
+
+function noVariantConfig(llmUrl: string) {
+  const config = verifierConfig(llmUrl)
+  return {
+    ...config,
+    provider: {
+      test: {
+        ...config.provider.test,
+        models: {
+          "test-model": {
+            ...config.provider.test.models["test-model"],
+            variants: undefined,
+          },
+          "second-model": config.provider.test.models["second-model"],
+        },
+      },
+    },
+  }
+}
+
+const verifierSkill = `---
+name: verifier-skill
+description: Verifier compatibility skill.
+---
+
+# Verifier Skill
+`

packages/opencode/test/cli/acp/acp-test-client.ts

@@ -0,0 +1,96 @@
+import { expect } from "bun:test"
+import type { SessionConfigOption, SessionConfigSelectOption } from "@agentclientprotocol/sdk"
+import { Duration, Effect } from "effect"
+import type { AcpHandle } from "../../lib/cli-process"
+
+type JsonRpcRequest = {
+  readonly jsonrpc: "2.0"
+  readonly id: number
+  readonly method: string
+  readonly params?: unknown
+}
+
+type JsonRpcResponse<T = unknown> = {
+  readonly jsonrpc: "2.0"
+  readonly id: number
+  readonly result?: T
+  readonly error?: unknown
+}
+
+type JsonRpcNotification<T = unknown> = {
+  readonly jsonrpc: "2.0"
+  readonly method: string
+  readonly params?: T
+}
+
+export type AcpClient = {
+  readonly request: <T>(method: string, params?: unknown) => Effect.Effect<JsonRpcResponse<T>, unknown>
+  readonly receive: Effect.Effect<unknown>
+  readonly waitForNotification: <T>(
+    method: string,
+    predicate: (params: T) => boolean,
+    timeoutMs?: number,
+  ) => Effect.Effect<JsonRpcNotification<T>, unknown>
+}
+
+export function createAcpClient(acp: AcpHandle): AcpClient {
+  const state = { nextId: 1 }
+
+  const request = <T>(method: string, params?: unknown) =>
+    Effect.gen(function* () {
+      const id = state.nextId++
+      const message: JsonRpcRequest =
+        params === undefined ? { jsonrpc: "2.0", id, method } : { jsonrpc: "2.0", id, method, params }
+      yield* acp.send(message)
+
+      while (true) {
+        const received = yield* acp.receive.pipe(Effect.timeout(Duration.seconds(15)))
+        if (isJsonRpcResponse<T>(received) && received.id === id) return received
+      }
+    })
+
+  const waitForNotification = <T>(method: string, predicate: (params: T) => boolean, timeoutMs = 15_000) =>
+    Effect.gen(function* () {
+      while (true) {
+        const received = yield* acp.receive.pipe(Effect.timeout(Duration.millis(timeoutMs)))
+        if (!isJsonRpcNotification<T>(received)) continue
+        if (received.method === method && predicate(received.params as T)) return received
+      }
+    })
+
+  return {
+    request,
+    receive: acp.receive,
+    waitForNotification,
+  }
+}
+
+export function expectOk<T>(response: JsonRpcResponse<T>) {
+  expect(response.error).toBeUndefined()
+  expect(response.result).toBeDefined()
+  return response.result as T
+}
+
+export function selectConfigOption(options: SessionConfigOption[] | null | undefined, id: string) {
+  return options?.find(
+    (option): option is Extract<SessionConfigOption, { type: "select" }> => option.id === id && option.type === "select",
+  )
+}
+
+export function firstAlternateValue(option: Extract<SessionConfigOption, { type: "select" }>) {
+  return flattenSelectOptions(option).find((item) => item.value !== option.currentValue)?.value
+}
+
+export function flattenSelectOptions(option: Extract<SessionConfigOption, { type: "select" }>) {
+  return option.options.flatMap((item): SessionConfigSelectOption[] => ("value" in item ? [item] : item.options))
+}
+
+function isJsonRpcResponse<T>(input: unknown): input is JsonRpcResponse<T> {
+  if (!input || typeof input !== "object") return false
+  return "id" in input && "jsonrpc" in input
+}
+
+function isJsonRpcNotification<T>(input: unknown): input is JsonRpcNotification<T> {
+  if (!input || typeof input !== "object") return false
+  return "method" in input && !("id" in input) && "jsonrpc" in input
+}