|
|
@@ -1,14 +1,16 @@
|
|
|
import { describe, expect, test } from "bun:test"
|
|
|
import {
|
|
|
+ APIError,
|
|
|
+ BadRequest,
|
|
|
+ ConnectionError,
|
|
|
+ ContextOverflow,
|
|
|
LLMClient,
|
|
|
- LLMError,
|
|
|
LLMEvent,
|
|
|
Model,
|
|
|
+ RateLimit,
|
|
|
ToolFailure,
|
|
|
- TransportReason,
|
|
|
- InvalidRequestReason,
|
|
|
- RateLimitReason,
|
|
|
type LLMClientShape,
|
|
|
+ type LLMError,
|
|
|
type LLMRequest,
|
|
|
} from "@opencode-ai/llm"
|
|
|
import * as OpenAIChat from "@opencode-ai/llm/protocols/openai-chat"
|
|
|
@@ -68,8 +70,9 @@ import { asc, eq } from "drizzle-orm"
|
|
|
import { testEffect } from "./lib/effect"
|
|
|
|
|
|
const requests: LLMRequest[] = []
|
|
|
+type ScriptedResponse = LLMEvent[] | Stream.Stream<LLMEvent, LLMError>
|
|
|
let response: LLMEvent[] = []
|
|
|
-let responses: LLMEvent[][] | undefined
|
|
|
+let responses: ScriptedResponse[] | undefined
|
|
|
let responseStream: Stream.Stream<LLMEvent, LLMError> | undefined
|
|
|
let responseStreams: Stream.Stream<LLMEvent, LLMError>[] | undefined
|
|
|
let streamGate: Deferred.Deferred<void> | undefined
|
|
|
@@ -92,9 +95,12 @@ const client = Layer.succeed(
|
|
|
responseStream = undefined
|
|
|
return stream
|
|
|
}
|
|
|
+ const scripted = responses === undefined ? response : (responses.shift() ?? [])
|
|
|
const events = streamFailure
|
|
|
? Stream.fail(streamFailure)
|
|
|
- : Stream.fromIterable(responses === undefined ? response : (responses.shift() ?? []))
|
|
|
+ : Array.isArray(scripted)
|
|
|
+ ? Stream.fromIterable(scripted)
|
|
|
+ : scripted
|
|
|
if (!streamGate) return events
|
|
|
return Stream.unwrap(
|
|
|
(streamStarted ? Deferred.succeed(streamStarted, undefined) : Effect.void).pipe(
|
|
|
@@ -480,26 +486,16 @@ const setup = Effect.gen(function* () {
|
|
|
return yield* SessionV2.Service
|
|
|
})
|
|
|
|
|
|
-const providerUnavailable = () =>
|
|
|
- new LLMError({
|
|
|
- module: "test",
|
|
|
- method: "stream",
|
|
|
- reason: new TransportReason({ message: "Provider unavailable" }),
|
|
|
- })
|
|
|
+const providerUnavailable = () => new ConnectionError({ message: "Provider unavailable" })
|
|
|
|
|
|
-const invalidRequest = () =>
|
|
|
- new LLMError({
|
|
|
- module: "test",
|
|
|
- method: "stream",
|
|
|
- reason: new InvalidRequestReason({ message: "Invalid request" }),
|
|
|
- })
|
|
|
+const contextOverflow = () => new ContextOverflow({ message: "prompt too long" })
|
|
|
|
|
|
-const rateLimited = (retryAfterMs?: number) =>
|
|
|
- new LLMError({
|
|
|
- module: "test",
|
|
|
- method: "stream",
|
|
|
- reason: new RateLimitReason({ message: "Rate limited", retryAfterMs }),
|
|
|
- })
|
|
|
+const failingResponse = (events: LLMEvent[], failure: LLMError): Stream.Stream<LLMEvent, LLMError> =>
|
|
|
+ Stream.fromIterable(events).pipe(Stream.concat(Stream.fail(failure)))
|
|
|
+
|
|
|
+const invalidRequest = () => new BadRequest({ message: "Invalid request" })
|
|
|
+
|
|
|
+const rateLimited = (retryAfterMs?: number) => new RateLimit({ message: "Rate limited", retryAfterMs })
|
|
|
|
|
|
const setupOverflowRecovery = Effect.gen(function* () {
|
|
|
const session = yield* setup
|
|
|
@@ -1646,14 +1642,14 @@ describe("SessionRunnerLLM", () => {
|
|
|
yield* admit(session, "Earlier question")
|
|
|
yield* session.resume(sessionID)
|
|
|
|
|
|
- response = [LLMEvent.providerError({ message: "summary unavailable" })]
|
|
|
+ responseStream = Stream.fail(new APIError({ message: "summary unavailable" }))
|
|
|
const compaction = yield* session.compact({ sessionID })
|
|
|
yield* session.resume(sessionID)
|
|
|
|
|
|
expect((yield* session.messages({ sessionID })).find((message) => message.id === compaction.id)).toMatchObject({
|
|
|
type: "compaction",
|
|
|
status: "failed",
|
|
|
- error: { type: "provider.error", message: "summary unavailable" },
|
|
|
+ error: { type: "provider.unknown", message: "summary unavailable" },
|
|
|
})
|
|
|
}),
|
|
|
)
|
|
|
@@ -1763,7 +1759,7 @@ describe("SessionRunnerLLM", () => {
|
|
|
currentModel = compactModel
|
|
|
requests.length = 0
|
|
|
responses = [
|
|
|
- [LLMEvent.providerError({ message: "Unsupported parameter: max_output_tokens" })],
|
|
|
+ Stream.fail(new BadRequest({ message: "Unsupported parameter: max_output_tokens" })),
|
|
|
reply.text("Must not run", "text-after-failed-compaction"),
|
|
|
]
|
|
|
yield* admit(session, "Recent exact request ".repeat(180))
|
|
|
@@ -1786,10 +1782,7 @@ describe("SessionRunnerLLM", () => {
|
|
|
Effect.gen(function* () {
|
|
|
const session = yield* setupOverflowRecovery
|
|
|
responses = [
|
|
|
- [
|
|
|
- LLMEvent.stepStart({ index: 0 }),
|
|
|
- LLMEvent.providerError({ message: "prompt too long", classification: "context-overflow" }),
|
|
|
- ],
|
|
|
+ failingResponse([LLMEvent.stepStart({ index: 0 })], contextOverflow()),
|
|
|
reply.text("## Objective\n- Recover overflow", "text-summary"),
|
|
|
reply.text("Recovered", "text-final"),
|
|
|
]
|
|
|
@@ -1816,7 +1809,7 @@ describe("SessionRunnerLLM", () => {
|
|
|
const session = yield* setupOverflowRecovery
|
|
|
currentModel = model
|
|
|
responses = [
|
|
|
- [LLMEvent.providerError({ message: "prompt too long", classification: "context-overflow" })],
|
|
|
+ Stream.fail(contextOverflow()),
|
|
|
reply.text("## Objective\n- Recover unknown limit", "text-summary-unknown-limit"),
|
|
|
reply.text("Recovered", "text-final-unknown-limit"),
|
|
|
]
|
|
|
@@ -1836,7 +1829,7 @@ describe("SessionRunnerLLM", () => {
|
|
|
const session = yield* setupOverflowRecovery
|
|
|
currentModel = undersizedContextModel
|
|
|
responses = [
|
|
|
- [LLMEvent.providerError({ message: "prompt too long", classification: "context-overflow" })],
|
|
|
+ Stream.fail(contextOverflow()),
|
|
|
reply.text("## Objective\n- Recover undersized limit", "text-summary-undersized-limit"),
|
|
|
reply.text("Recovered", "text-final-undersized-limit"),
|
|
|
]
|
|
|
@@ -1854,10 +1847,7 @@ describe("SessionRunnerLLM", () => {
|
|
|
it.effect("persists a second context overflow after one recovery", () =>
|
|
|
Effect.gen(function* () {
|
|
|
const session = yield* setupOverflowRecovery
|
|
|
- const overflow = () => [
|
|
|
- LLMEvent.stepStart({ index: 0 }),
|
|
|
- LLMEvent.providerError({ message: "prompt too long", classification: "context-overflow" }),
|
|
|
- ]
|
|
|
+ const overflow = () => failingResponse([LLMEvent.stepStart({ index: 0 })], contextOverflow())
|
|
|
responses = [overflow(), reply.text("## Objective\n- Recover once", "text-summary"), overflow()]
|
|
|
yield* admit(session, "Continue")
|
|
|
expect((yield* session.resume(sessionID).pipe(Effect.flip)).message).toBe("prompt too long")
|
|
|
@@ -1873,16 +1863,7 @@ describe("SessionRunnerLLM", () => {
|
|
|
it.effect("recovers once from a raw context overflow failure", () =>
|
|
|
Effect.gen(function* () {
|
|
|
const session = yield* setupOverflowRecovery
|
|
|
- responseStream = Stream.fail(
|
|
|
- new LLMError({
|
|
|
- module: "test",
|
|
|
- method: "stream",
|
|
|
- reason: new InvalidRequestReason({
|
|
|
- message: "prompt too long",
|
|
|
- classification: "context-overflow",
|
|
|
- }),
|
|
|
- }),
|
|
|
- )
|
|
|
+ responseStream = Stream.fail(contextOverflow())
|
|
|
responses = [
|
|
|
reply.text("## Objective\n- Recover raw overflow", "text-summary"),
|
|
|
reply.text("Recovered", "text-final"),
|
|
|
@@ -1901,10 +1882,7 @@ describe("SessionRunnerLLM", () => {
|
|
|
it.effect("publishes the original overflow when recovery summarization fails", () =>
|
|
|
Effect.gen(function* () {
|
|
|
const session = yield* setupOverflowRecovery
|
|
|
- responses = [
|
|
|
- [LLMEvent.providerError({ message: "prompt too long", classification: "context-overflow" })],
|
|
|
- [LLMEvent.providerError({ message: "summary unavailable" })],
|
|
|
- ]
|
|
|
+ responses = [Stream.fail(contextOverflow()), Stream.fail(new APIError({ message: "summary unavailable" }))]
|
|
|
yield* admit(session, "Continue")
|
|
|
expect((yield* session.resume(sessionID).pipe(Effect.flip)).message).toBe("prompt too long")
|
|
|
|
|
|
@@ -1915,7 +1893,7 @@ describe("SessionRunnerLLM", () => {
|
|
|
type: "compaction",
|
|
|
status: "failed",
|
|
|
reason: "auto",
|
|
|
- error: { type: "provider.error", message: "summary unavailable" },
|
|
|
+ error: { type: "provider.unknown", message: "summary unavailable" },
|
|
|
}),
|
|
|
)
|
|
|
expect(context.slice(-3)).toMatchObject([
|
|
|
@@ -1929,10 +1907,7 @@ describe("SessionRunnerLLM", () => {
|
|
|
it.effect("interrupts overflow recovery while the summary provider is running", () =>
|
|
|
Effect.gen(function* () {
|
|
|
const session = yield* setupOverflowRecovery
|
|
|
- responses = [
|
|
|
- [LLMEvent.providerError({ message: "prompt too long", classification: "context-overflow" })],
|
|
|
- reply.text("## Objective\n- Interrupted", "text-summary"),
|
|
|
- ]
|
|
|
+ responses = [Stream.fail(contextOverflow()), reply.text("## Objective\n- Interrupted", "text-summary")]
|
|
|
const firstGate = yield* Deferred.make<void>()
|
|
|
const summaryGate = yield* Deferred.make<void>()
|
|
|
streamGate = firstGate
|
|
|
@@ -3515,7 +3490,10 @@ describe("SessionRunnerLLM", () => {
|
|
|
const session = yield* setup
|
|
|
yield* admit(session, "Fail durably")
|
|
|
|
|
|
- response = [LLMEvent.stepStart({ index: 0 }), LLMEvent.providerError({ message: "Provider unavailable" })]
|
|
|
+ responseStream = failingResponse(
|
|
|
+ [LLMEvent.stepStart({ index: 0 })],
|
|
|
+ new APIError({ message: "Provider unavailable" }),
|
|
|
+ )
|
|
|
|
|
|
expect((yield* session.resume(sessionID).pipe(Effect.flip)).message).toBe("Provider unavailable")
|
|
|
|
|
|
@@ -3532,7 +3510,7 @@ describe("SessionRunnerLLM", () => {
|
|
|
const session = yield* setup
|
|
|
yield* admit(session, "Fail before step")
|
|
|
|
|
|
- response = [LLMEvent.providerError({ message: "Provider unavailable" })]
|
|
|
+ responseStream = Stream.fail(new APIError({ message: "Provider unavailable" }))
|
|
|
|
|
|
expect((yield* session.resume(sessionID).pipe(Effect.flip)).message).toBe("Provider unavailable")
|
|
|
|
|
|
@@ -3620,13 +3598,15 @@ describe("SessionRunnerLLM", () => {
|
|
|
const session = yield* setup
|
|
|
yield* admit(session, "Fail after output")
|
|
|
|
|
|
- response = [
|
|
|
- LLMEvent.stepStart({ index: 0 }),
|
|
|
- LLMEvent.textStart({ id: "text-partial" }),
|
|
|
- LLMEvent.textDelta({ id: "text-partial", text: "Partial" }),
|
|
|
- LLMEvent.textEnd({ id: "text-partial" }),
|
|
|
- LLMEvent.providerError({ message: "prompt too long", classification: "context-overflow" }),
|
|
|
- ]
|
|
|
+ responseStream = failingResponse(
|
|
|
+ [
|
|
|
+ LLMEvent.stepStart({ index: 0 }),
|
|
|
+ LLMEvent.textStart({ id: "text-partial" }),
|
|
|
+ LLMEvent.textDelta({ id: "text-partial", text: "Partial" }),
|
|
|
+ LLMEvent.textEnd({ id: "text-partial" }),
|
|
|
+ ],
|
|
|
+ contextOverflow(),
|
|
|
+ )
|
|
|
expect((yield* session.resume(sessionID).pipe(Effect.flip)).message).toBe("prompt too long")
|
|
|
|
|
|
expect(requests).toHaveLength(1)
|
|
|
@@ -3784,11 +3764,13 @@ describe("SessionRunnerLLM", () => {
|
|
|
toolExecutionGate = yield* Deferred.make<void>()
|
|
|
toolExecutionsStarted = yield* Deferred.make<void>()
|
|
|
toolExecutionsReady = 1
|
|
|
- response = [
|
|
|
- LLMEvent.stepStart({ index: 0 }),
|
|
|
- LLMEvent.toolCall({ id: "call-before-provider-error", name: "echo", input: { text: "settled" } }),
|
|
|
- LLMEvent.providerError({ message: "Provider unavailable" }),
|
|
|
- ]
|
|
|
+ responseStream = failingResponse(
|
|
|
+ [
|
|
|
+ LLMEvent.stepStart({ index: 0 }),
|
|
|
+ LLMEvent.toolCall({ id: "call-before-provider-error", name: "echo", input: { text: "settled" } }),
|
|
|
+ ],
|
|
|
+ new APIError({ message: "Provider unavailable" }),
|
|
|
+ )
|
|
|
|
|
|
const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
|
|
|
yield* Deferred.await(toolExecutionsStarted)
|
|
|
@@ -3815,11 +3797,10 @@ describe("SessionRunnerLLM", () => {
|
|
|
const session = yield* setup
|
|
|
yield* admit(session, "Fail hosted tool durably")
|
|
|
|
|
|
- response = [
|
|
|
- LLMEvent.stepStart({ index: 0 }),
|
|
|
- hostedCall("call-hosted-provider-error", "effect"),
|
|
|
- LLMEvent.providerError({ message: "Provider unavailable" }),
|
|
|
- ]
|
|
|
+ responseStream = failingResponse(
|
|
|
+ [LLMEvent.stepStart({ index: 0 }), hostedCall("call-hosted-provider-error", "effect")],
|
|
|
+ new APIError({ message: "Provider unavailable" }),
|
|
|
+ )
|
|
|
|
|
|
expect((yield* session.resume(sessionID).pipe(Effect.flip)).message).toBe("Provider unavailable")
|
|
|
|
|
|
@@ -3846,11 +3827,13 @@ describe("SessionRunnerLLM", () => {
|
|
|
Effect.gen(function* () {
|
|
|
const session = yield* setup
|
|
|
yield* admit(session, "Defect while provider fails")
|
|
|
- response = [
|
|
|
- LLMEvent.stepStart({ index: 0 }),
|
|
|
- LLMEvent.toolCall({ id: "call-defect-provider-error", name: "defect", input: {} }),
|
|
|
- LLMEvent.providerError({ message: "Provider unavailable" }),
|
|
|
- ]
|
|
|
+ responseStream = failingResponse(
|
|
|
+ [
|
|
|
+ LLMEvent.stepStart({ index: 0 }),
|
|
|
+ LLMEvent.toolCall({ id: "call-defect-provider-error", name: "defect", input: {} }),
|
|
|
+ ],
|
|
|
+ new APIError({ message: "Provider unavailable" }),
|
|
|
+ )
|
|
|
|
|
|
expect((yield* session.resume(sessionID).pipe(Effect.flip)).message).toBe("Provider unavailable")
|
|
|
|