|
|
@@ -1,8 +1,10 @@
|
|
|
import { NodeFileSystem } from "@effect/platform-node"
|
|
|
+import { AppFileSystem } from "@opencode-ai/core/filesystem"
|
|
|
+import { ModelsDev } from "@opencode-ai/core/models-dev"
|
|
|
import { HttpRecorder, Redactor } from "@opencode-ai/http-recorder"
|
|
|
-import { describe, expect } from "bun:test"
|
|
|
+import { describe, expect, test } from "bun:test"
|
|
|
import { tool, type ModelMessage, type JSONValue } from "ai"
|
|
|
-import { Effect, Layer, Stream } from "effect"
|
|
|
+import { Effect, Layer, Option, Schema, Stream } from "effect"
|
|
|
import { FetchHttpClient } from "effect/unstable/http"
|
|
|
import path from "node:path"
|
|
|
import z from "zod"
|
|
|
@@ -14,12 +16,12 @@ import { ModelID, ProviderID } from "@/provider/schema"
|
|
|
import { Filesystem } from "@/util/filesystem"
|
|
|
import { LLMEvent, LLMResponse } from "@opencode-ai/llm"
|
|
|
import { LLMClient, RequestExecutor, WebSocketExecutor } from "@opencode-ai/llm/route"
|
|
|
+import { Env } from "@/env"
|
|
|
import { RuntimeFlags } from "@/effect/runtime-flags"
|
|
|
import type { Agent } from "../../src/agent/agent"
|
|
|
import { LLM } from "../../src/session/llm"
|
|
|
import { MessageV2 } from "../../src/session/message-v2"
|
|
|
import { MessageID, SessionID } from "../../src/session/schema"
|
|
|
-import type { ModelsDev } from "@opencode-ai/core/models-dev"
|
|
|
import { TestInstance } from "../fixture/fixture"
|
|
|
import { testEffect } from "../lib/effect"
|
|
|
|
|
|
@@ -27,106 +29,232 @@ const FIXTURES_DIR = path.join(import.meta.dir, "../fixtures/recordings")
|
|
|
|
|
|
const zenURL = (connection: string) => `https://console.opencode.ai/proxy/connections/${connection}/v1`
|
|
|
|
|
|
-type ProviderSpec = {
|
|
|
+const replayOpenAIOAuth = {
|
|
|
+ type: "oauth",
|
|
|
+ refresh: "fixture-refresh-token",
|
|
|
+ access: "fixture-access-token",
|
|
|
+ expires: Date.now() + 60 * 60 * 1000,
|
|
|
+ accountId: "fixture-account",
|
|
|
+} satisfies Auth.Info
|
|
|
+
|
|
|
+type RecordedScenario = {
|
|
|
+ readonly id: string
|
|
|
+ readonly name: string
|
|
|
readonly providerID: ProviderID
|
|
|
readonly modelID: string
|
|
|
readonly cassette: string
|
|
|
readonly protocol: string
|
|
|
readonly tags: ReadonlyArray<string>
|
|
|
- readonly canRecord: boolean
|
|
|
+ readonly canRecord: () => boolean
|
|
|
+ readonly recordAuth?: () => Auth.Info | undefined
|
|
|
+ readonly replayAuth?: Auth.Info
|
|
|
+ readonly stableID?: string
|
|
|
readonly config: (model: ModelsDev.Provider["models"][string]) => Partial<Config.Info>
|
|
|
}
|
|
|
|
|
|
-const cloneModel = (model: ModelsDev.Provider["models"][string]) =>
|
|
|
- structuredClone(model) as NonNullable<NonNullable<Config.Info["provider"]>[string]["models"]>[string]
|
|
|
+const cloneModel = (model: ModelsDev.Provider["models"][string]) => {
|
|
|
+ const cloned = structuredClone(model)
|
|
|
+ const { experimental, ...rest } = cloned
|
|
|
+ // oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion -- The config schema accepts the same model shape except object-valued experimental metadata.
|
|
|
+ if (typeof experimental === "boolean")
|
|
|
+ return cloned as NonNullable<NonNullable<Config.Info["provider"]>[string]["models"]>[string]
|
|
|
+ // oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion -- Dropping non-boolean experimental metadata makes the fixture model match config input.
|
|
|
+ return rest as NonNullable<NonNullable<Config.Info["provider"]>[string]["models"]>[string]
|
|
|
+}
|
|
|
+
|
|
|
+const envValue = (...names: string[]) => names.map((name) => process.env[name]).find(Boolean)
|
|
|
+const decodeAuth = Schema.decodeUnknownOption(Auth.Info)
|
|
|
+const recordOpenAIOAuth = (() => {
|
|
|
+ let loaded = false
|
|
|
+ let auth: Auth.Info | undefined
|
|
|
+ return () => {
|
|
|
+ if (loaded) return auth
|
|
|
+ loaded = true
|
|
|
+ auth = decodeRecordOpenAIOAuth()
|
|
|
+ return auth
|
|
|
+ }
|
|
|
+})()
|
|
|
+
|
|
|
+function decodeRecordOpenAIOAuth() {
|
|
|
+ const value = process.env.OPENCODE_RECORD_OPENAI_AUTH
|
|
|
+ if (!value) return undefined
|
|
|
+ try {
|
|
|
+ const auth = Option.getOrUndefined(decodeAuth(JSON.parse(value)))
|
|
|
+ return auth?.type === "oauth" ? auth : undefined
|
|
|
+ } catch {
|
|
|
+ return undefined
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+const providerConfig = (input: {
|
|
|
+ readonly providerID: ProviderID
|
|
|
+ readonly name: string
|
|
|
+ readonly env: string[]
|
|
|
+ readonly npm: string
|
|
|
+ readonly api: string
|
|
|
+ readonly model: ModelsDev.Provider["models"][string]
|
|
|
+ readonly options: Record<string, unknown>
|
|
|
+}): Partial<Config.Info> => ({
|
|
|
+ enabled_providers: [input.providerID],
|
|
|
+ provider: {
|
|
|
+ [input.providerID]: {
|
|
|
+ name: input.name,
|
|
|
+ env: input.env,
|
|
|
+ npm: input.npm,
|
|
|
+ api: input.api,
|
|
|
+ models: { [input.model.id]: cloneModel(input.model) },
|
|
|
+ options: input.options,
|
|
|
+ },
|
|
|
+ },
|
|
|
+})
|
|
|
|
|
|
-const PROVIDERS = {
|
|
|
- openai: {
|
|
|
+const RECORDED_SCENARIOS = [
|
|
|
+ {
|
|
|
+ id: "openai-api-key",
|
|
|
+ name: "OpenAI API key",
|
|
|
providerID: ProviderID.openai,
|
|
|
modelID: "gpt-4.1-mini",
|
|
|
cassette: "session/native-openai-tool-loop",
|
|
|
protocol: "openai-responses",
|
|
|
tags: ["opencode", "native", "tool-loop"],
|
|
|
- canRecord: Boolean(process.env.OPENCODE_RECORD_OPENAI_API_KEY ?? process.env.OPENAI_API_KEY),
|
|
|
- config: (model) => ({
|
|
|
- enabled_providers: ["openai"],
|
|
|
- provider: {
|
|
|
- openai: {
|
|
|
- name: "OpenAI",
|
|
|
- env: ["OPENAI_API_KEY"],
|
|
|
- npm: "@ai-sdk/openai",
|
|
|
- api: "https://api.openai.com/v1",
|
|
|
- models: { [model.id]: cloneModel(model) },
|
|
|
- options: {
|
|
|
- apiKey: process.env.OPENCODE_RECORD_OPENAI_API_KEY ?? process.env.OPENAI_API_KEY ?? "fixture-openai-key",
|
|
|
- baseURL: "https://api.openai.com/v1",
|
|
|
- },
|
|
|
+ canRecord: () => Boolean(envValue("OPENCODE_RECORD_OPENAI_API_KEY", "OPENAI_API_KEY")),
|
|
|
+ config: (model) =>
|
|
|
+ providerConfig({
|
|
|
+ providerID: ProviderID.openai,
|
|
|
+ name: "OpenAI",
|
|
|
+ env: ["OPENAI_API_KEY"],
|
|
|
+ npm: "@ai-sdk/openai",
|
|
|
+ api: "https://api.openai.com/v1",
|
|
|
+ model,
|
|
|
+ options: {
|
|
|
+ apiKey: envValue("OPENCODE_RECORD_OPENAI_API_KEY", "OPENAI_API_KEY") ?? "fixture-openai-key",
|
|
|
+ baseURL: "https://api.openai.com/v1",
|
|
|
},
|
|
|
- },
|
|
|
- }),
|
|
|
+ }),
|
|
|
},
|
|
|
- opencode: {
|
|
|
+ {
|
|
|
+ id: "openai-oauth",
|
|
|
+ name: "OpenAI OAuth",
|
|
|
+ providerID: ProviderID.openai,
|
|
|
+ modelID: "gpt-5.5",
|
|
|
+ cassette: "session/native-openai-oauth-tool-loop",
|
|
|
+ protocol: "openai-responses",
|
|
|
+ tags: ["opencode", "native", "oauth", "tool-loop"],
|
|
|
+ canRecord: () => recordOpenAIOAuth() !== undefined,
|
|
|
+ recordAuth: recordOpenAIOAuth,
|
|
|
+ replayAuth: replayOpenAIOAuth,
|
|
|
+ stableID: "openai-oauth",
|
|
|
+ config: (model) =>
|
|
|
+ providerConfig({
|
|
|
+ providerID: ProviderID.openai,
|
|
|
+ name: "OpenAI",
|
|
|
+ env: ["OPENAI_API_KEY"],
|
|
|
+ npm: "@ai-sdk/openai",
|
|
|
+ api: "https://api.openai.com/v1",
|
|
|
+ model,
|
|
|
+ options: { baseURL: "https://api.openai.com/v1" },
|
|
|
+ }),
|
|
|
+ },
|
|
|
+ {
|
|
|
+ id: "opencode-proxy",
|
|
|
+ name: "OpenCode proxy",
|
|
|
providerID: ProviderID.opencode,
|
|
|
modelID: "gpt-5.2-codex",
|
|
|
cassette: "session/native-zen-tool-loop",
|
|
|
protocol: "openai-responses",
|
|
|
tags: ["opencode", "zen", "native", "tool-loop"],
|
|
|
- canRecord: Boolean(process.env.OPENCODE_RECORD_CONSOLE_TOKEN && process.env.OPENCODE_RECORD_ZEN_ORG_ID),
|
|
|
- config: (model) => ({
|
|
|
- enabled_providers: ["opencode"],
|
|
|
- provider: {
|
|
|
- opencode: {
|
|
|
- name: "OpenCode Zen",
|
|
|
- env: ["OPENCODE_CONSOLE_TOKEN"],
|
|
|
- npm: "@ai-sdk/openai-compatible",
|
|
|
- // The connection slug is account-specific; the cassette redactor
|
|
|
- // normalizes it to {connection} for replay. Set during recording.
|
|
|
- api: zenURL(process.env.OPENCODE_RECORD_ZEN_CONNECTION ?? "fixture"),
|
|
|
- models: { [model.id]: cloneModel(model) },
|
|
|
- options: {
|
|
|
- apiKey: process.env.OPENCODE_RECORD_CONSOLE_TOKEN ?? "fixture-console-token",
|
|
|
- headers: { "x-org-id": process.env.OPENCODE_RECORD_ZEN_ORG_ID ?? "fixture-org" },
|
|
|
- },
|
|
|
+ canRecord: () => Boolean(process.env.OPENCODE_RECORD_CONSOLE_TOKEN && process.env.OPENCODE_RECORD_ZEN_ORG_ID),
|
|
|
+ config: (model) =>
|
|
|
+ providerConfig({
|
|
|
+ providerID: ProviderID.opencode,
|
|
|
+ name: "OpenCode Zen",
|
|
|
+ env: ["OPENCODE_CONSOLE_TOKEN"],
|
|
|
+ npm: "@ai-sdk/openai-compatible",
|
|
|
+ api: zenURL(process.env.OPENCODE_RECORD_ZEN_CONNECTION ?? "fixture"),
|
|
|
+ model,
|
|
|
+ options: {
|
|
|
+ apiKey: process.env.OPENCODE_RECORD_CONSOLE_TOKEN ?? "fixture-console-token",
|
|
|
+ headers: { "x-org-id": process.env.OPENCODE_RECORD_ZEN_ORG_ID ?? "fixture-org" },
|
|
|
},
|
|
|
- },
|
|
|
- }),
|
|
|
+ }),
|
|
|
},
|
|
|
- anthropic: {
|
|
|
+ {
|
|
|
+ id: "anthropic-api-key",
|
|
|
+ name: "Anthropic API key",
|
|
|
providerID: ProviderID.anthropic,
|
|
|
modelID: "claude-haiku-4-5-20251001",
|
|
|
cassette: "session/native-anthropic-tool-loop",
|
|
|
protocol: "anthropic-messages",
|
|
|
tags: ["opencode", "native", "tool-loop"],
|
|
|
- canRecord: Boolean(process.env.OPENCODE_RECORD_ANTHROPIC_API_KEY ?? process.env.ANTHROPIC_API_KEY),
|
|
|
- config: (model) => ({
|
|
|
- enabled_providers: ["anthropic"],
|
|
|
- provider: {
|
|
|
- anthropic: {
|
|
|
- name: "Anthropic",
|
|
|
- env: ["ANTHROPIC_API_KEY"],
|
|
|
- npm: "@ai-sdk/anthropic",
|
|
|
- api: "https://api.anthropic.com/v1",
|
|
|
- models: { [model.id]: cloneModel(model) },
|
|
|
- options: {
|
|
|
- apiKey:
|
|
|
- process.env.OPENCODE_RECORD_ANTHROPIC_API_KEY ?? process.env.ANTHROPIC_API_KEY ?? "fixture-anthropic-key",
|
|
|
- baseURL: "https://api.anthropic.com/v1",
|
|
|
- },
|
|
|
+ canRecord: () => Boolean(envValue("OPENCODE_RECORD_ANTHROPIC_API_KEY", "ANTHROPIC_API_KEY")),
|
|
|
+ config: (model) =>
|
|
|
+ providerConfig({
|
|
|
+ providerID: ProviderID.anthropic,
|
|
|
+ name: "Anthropic",
|
|
|
+ env: ["ANTHROPIC_API_KEY"],
|
|
|
+ npm: "@ai-sdk/anthropic",
|
|
|
+ api: "https://api.anthropic.com/v1",
|
|
|
+ model,
|
|
|
+ options: {
|
|
|
+ apiKey: envValue("OPENCODE_RECORD_ANTHROPIC_API_KEY", "ANTHROPIC_API_KEY") ?? "fixture-anthropic-key",
|
|
|
+ baseURL: "https://api.anthropic.com/v1",
|
|
|
},
|
|
|
- },
|
|
|
- }),
|
|
|
+ }),
|
|
|
},
|
|
|
-} satisfies Record<string, ProviderSpec>
|
|
|
+] satisfies ReadonlyArray<RecordedScenario>
|
|
|
|
|
|
const shouldRecord = process.env.RECORD === "true"
|
|
|
+const selectedScenarios = new Set(
|
|
|
+ (envValue("OPENCODE_RECORDED_SCENARIO", "RECORDED_PROVIDER") ?? "")
|
|
|
+ .split(",")
|
|
|
+ .map((item) => item.trim().toLowerCase())
|
|
|
+ .filter(Boolean),
|
|
|
+)
|
|
|
|
|
|
-const canRun = (spec: ProviderSpec) =>
|
|
|
- shouldRecord ? spec.canRecord : HttpRecorder.hasCassetteSync(spec.cassette, { directory: FIXTURES_DIR })
|
|
|
+function isSelected(scenario: RecordedScenario) {
|
|
|
+ if (selectedScenarios.size === 0) return true
|
|
|
+ return [scenario.id, scenario.name, scenario.providerID, scenario.cassette, ...scenario.tags]
|
|
|
+ .map((item) => item.toLowerCase())
|
|
|
+ .some((item) => selectedScenarios.has(item))
|
|
|
+}
|
|
|
+
|
|
|
+const canRun = (scenario: RecordedScenario) =>
|
|
|
+ shouldRecord ? scenario.canRecord() : HttpRecorder.hasCassetteSync(scenario.cassette, { directory: FIXTURES_DIR })
|
|
|
+
|
|
|
+const recordError = (scenario: RecordedScenario) =>
|
|
|
+ scenario.id === "openai-oauth"
|
|
|
+ ? "Set OPENCODE_RECORD_OPENAI_AUTH to an OAuth auth JSON object in the recording environment."
|
|
|
+ : `Missing recording credentials for ${scenario.name}.`
|
|
|
+
|
|
|
+const redactRecordedBody = (body: string) =>
|
|
|
+ body
|
|
|
+ .replace(/wrk_[A-Z0-9]+/g, "wrk_redacted")
|
|
|
+ .replace(/"safety_identifier"\s*:\s*"user-[^"]+"/g, '"safety_identifier":"user_redacted"')
|
|
|
+ .replace(/"(access|access_token|refresh|refresh_token|accountId|account_id)"\s*:\s*"[^"]+"/g, '"$1":"redacted"')
|
|
|
+
|
|
|
+const recordingRedactor = Redactor.compose(
|
|
|
+ Redactor.defaults({
|
|
|
+ url: {
|
|
|
+ transform: (url) => url.replace(/\/proxy\/connections\/[^/]+\/v1/, "/proxy/connections/{connection}/v1"),
|
|
|
+ },
|
|
|
+ }),
|
|
|
+ {
|
|
|
+ request: (snapshot) => ({ ...snapshot, body: redactRecordedBody(snapshot.body) }),
|
|
|
+ response: (snapshot) => ({ ...snapshot, body: redactRecordedBody(snapshot.body) }),
|
|
|
+ },
|
|
|
+)
|
|
|
+
|
|
|
+function authLayer(scenario: RecordedScenario) {
|
|
|
+ const replayAuth = shouldRecord ? scenario.recordAuth?.() : scenario.replayAuth
|
|
|
+ if (!replayAuth) return Auth.defaultLayer
|
|
|
+ return Layer.mock(Auth.Service)({
|
|
|
+ get: (providerID) => Effect.succeed(providerID === scenario.providerID ? replayAuth : undefined),
|
|
|
+ all: () => Effect.succeed({ [scenario.providerID]: replayAuth }),
|
|
|
+ })
|
|
|
+}
|
|
|
|
|
|
async function loadFixture(providerID: string, modelID: string) {
|
|
|
- const data = await Filesystem.readJson<Record<string, ModelsDev.Provider>>(
|
|
|
- path.join(import.meta.dir, "../tool/fixtures/models-api.json"),
|
|
|
- )
|
|
|
+ const data = await modelsFixture
|
|
|
const provider = data[providerID]
|
|
|
if (!provider) throw new Error(`Missing provider in fixture: ${providerID}`)
|
|
|
const model = provider.models[modelID]
|
|
|
@@ -134,38 +262,44 @@ async function loadFixture(providerID: string, modelID: string) {
|
|
|
return model
|
|
|
}
|
|
|
|
|
|
-function recordedNativeLLMLayer(spec: ProviderSpec) {
|
|
|
+const modelsFixture = Filesystem.readJson<Record<string, ModelsDev.Provider>>(
|
|
|
+ path.join(import.meta.dir, "../tool/fixtures/models-api.json"),
|
|
|
+)
|
|
|
+
|
|
|
+function recordedNativeLLMLayer(scenario: RecordedScenario) {
|
|
|
+ const auth = authLayer(scenario)
|
|
|
+ const provider = Provider.layer.pipe(
|
|
|
+ Layer.provide(AppFileSystem.defaultLayer),
|
|
|
+ Layer.provide(Env.defaultLayer),
|
|
|
+ Layer.provide(Config.defaultLayer),
|
|
|
+ Layer.provide(auth),
|
|
|
+ Layer.provide(Plugin.defaultLayer),
|
|
|
+ Layer.provide(ModelsDev.defaultLayer),
|
|
|
+ Layer.provide(RuntimeFlags.defaultLayer),
|
|
|
+ )
|
|
|
// Only the HTTP client is recorded; RequestExecutor and the opencode LLM stack remain real.
|
|
|
const recordedClient = LLMClient.layer.pipe(
|
|
|
Layer.provide(Layer.mergeAll(RequestExecutor.layer, WebSocketExecutor.layer)),
|
|
|
Layer.provide(
|
|
|
- HttpRecorder.recordingLayer(spec.cassette, {
|
|
|
+ HttpRecorder.recordingLayer(scenario.cassette, {
|
|
|
mode: shouldRecord ? "record" : "replay",
|
|
|
- metadata: { provider: spec.providerID, protocol: spec.protocol, route: spec.protocol, tags: spec.tags },
|
|
|
- redactor: Redactor.compose(
|
|
|
- Redactor.defaults({
|
|
|
- url: {
|
|
|
- transform: (url) => url.replace(/\/proxy\/connections\/[^/]+\/v1/, "/proxy/connections/{connection}/v1"),
|
|
|
- },
|
|
|
- }),
|
|
|
- {
|
|
|
- response: (snapshot) => ({ ...snapshot, body: snapshot.body.replace(/wrk_[A-Z0-9]+/g, "wrk_redacted") }),
|
|
|
- },
|
|
|
- ),
|
|
|
+ metadata: {
|
|
|
+ provider: scenario.providerID,
|
|
|
+ protocol: scenario.protocol,
|
|
|
+ route: scenario.protocol,
|
|
|
+ tags: scenario.tags,
|
|
|
+ },
|
|
|
+ redactor: recordingRedactor,
|
|
|
}).pipe(Layer.provide(FetchHttpClient.layer)),
|
|
|
),
|
|
|
)
|
|
|
|
|
|
return Layer.mergeAll(
|
|
|
- Provider.defaultLayer.pipe(
|
|
|
- Layer.provide(Auth.defaultLayer),
|
|
|
- Layer.provide(Config.defaultLayer),
|
|
|
- Layer.provide(Plugin.defaultLayer),
|
|
|
- ),
|
|
|
+ provider,
|
|
|
LLM.layer.pipe(
|
|
|
- Layer.provide(Auth.defaultLayer),
|
|
|
+ Layer.provide(auth),
|
|
|
Layer.provide(Config.defaultLayer),
|
|
|
- Layer.provide(Provider.defaultLayer),
|
|
|
+ Layer.provide(provider),
|
|
|
Layer.provide(Plugin.defaultLayer),
|
|
|
Layer.provide(recordedClient),
|
|
|
Layer.provide(
|
|
|
@@ -176,11 +310,11 @@ function recordedNativeLLMLayer(spec: ProviderSpec) {
|
|
|
)
|
|
|
}
|
|
|
|
|
|
-const writeConfig = (directory: string, spec: ProviderSpec, model: ModelsDev.Provider["models"][string]) =>
|
|
|
+const writeConfig = (directory: string, scenario: RecordedScenario, model: ModelsDev.Provider["models"][string]) =>
|
|
|
Effect.promise(() =>
|
|
|
Bun.write(
|
|
|
path.join(directory, "opencode.json"),
|
|
|
- JSON.stringify({ $schema: "https://opencode.ai/config.json", ...spec.config(model) }),
|
|
|
+ JSON.stringify({ $schema: "https://opencode.ai/config.json", ...scenario.config(model) }),
|
|
|
),
|
|
|
)
|
|
|
|
|
|
@@ -214,13 +348,14 @@ const toolRoundtrip = (
|
|
|
},
|
|
|
]
|
|
|
|
|
|
-const driveToolLoop = (spec: ProviderSpec) =>
|
|
|
+const driveToolLoop = (scenario: RecordedScenario) =>
|
|
|
Effect.gen(function* () {
|
|
|
const test = yield* TestInstance
|
|
|
- const model = yield* Effect.promise(() => loadFixture(spec.providerID, spec.modelID))
|
|
|
- yield* writeConfig(test.directory, spec, model)
|
|
|
+ const model = yield* Effect.promise(() => loadFixture(scenario.providerID, scenario.modelID))
|
|
|
+ yield* writeConfig(test.directory, scenario, model)
|
|
|
|
|
|
- const sessionID = SessionID.make(`session-recorded-${spec.providerID}-loop`)
|
|
|
+ const stableID = scenario.stableID ?? scenario.providerID
|
|
|
+ const sessionID = SessionID.make(`session-recorded-${stableID}-loop`)
|
|
|
const modelID = ModelID.make(model.id)
|
|
|
const agent = {
|
|
|
name: "test",
|
|
|
@@ -231,17 +366,17 @@ const driveToolLoop = (spec: ProviderSpec) =>
|
|
|
temperature: 0,
|
|
|
} satisfies Agent.Info
|
|
|
const provider = yield* Provider.Service
|
|
|
- const resolved = yield* provider.getModel(spec.providerID, modelID)
|
|
|
+ const resolved = yield* provider.getModel(scenario.providerID, modelID)
|
|
|
|
|
|
const userMessage = { role: "user", content: WEATHER_USER } satisfies ModelMessage
|
|
|
const base = {
|
|
|
user: {
|
|
|
- id: MessageID.make(`msg_user-recorded-${spec.providerID}-loop`),
|
|
|
+ id: MessageID.make(`msg_user-recorded-${stableID}-loop`),
|
|
|
sessionID,
|
|
|
role: "user",
|
|
|
time: { created: 0 },
|
|
|
agent: agent.name,
|
|
|
- model: { providerID: spec.providerID, modelID },
|
|
|
+ model: { providerID: scenario.providerID, modelID },
|
|
|
} satisfies MessageV2.User,
|
|
|
sessionID,
|
|
|
model: resolved,
|
|
|
@@ -269,9 +404,18 @@ const driveToolLoop = (spec: ProviderSpec) =>
|
|
|
})
|
|
|
|
|
|
describe("session.llm native recorded", () => {
|
|
|
- for (const [name, spec] of Object.entries(PROVIDERS)) {
|
|
|
- const it = testEffect(recordedNativeLLMLayer(spec))
|
|
|
- const instance = canRun(spec) ? it.instance : it.instance.skip
|
|
|
- instance(`${name}: drives a tool loop to a final text answer`, () => driveToolLoop(spec))
|
|
|
+ for (const scenario of RECORDED_SCENARIOS.filter(isSelected)) {
|
|
|
+ if (!canRun(scenario)) {
|
|
|
+ if (shouldRecord && scenario.recordAuth && selectedScenarios.size > 0) {
|
|
|
+ test(`${scenario.name}: drives a tool loop to a final text answer`, () => {
|
|
|
+ throw new Error(recordError(scenario))
|
|
|
+ })
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ test.skip(`${scenario.name}: drives a tool loop to a final text answer`, () => {})
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ const it = testEffect(recordedNativeLLMLayer(scenario))
|
|
|
+ it.instance(`${scenario.name}: drives a tool loop to a final text answer`, () => driveToolLoop(scenario))
|
|
|
}
|
|
|
})
|