prepare.test.ts 6.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178
  1. import { describe, expect, test } from "bun:test"
  2. import { Effect, Schema } from "effect"
  3. import { HttpClientRequest } from "effect/unstable/http"
  4. import { LLM, mergeProviderOptions } from "../src"
  5. import { AnthropicMessages, OpenAIChat } from "../src/protocols"
  6. import { Auth, LLMClient } from "../src/route"
  7. import { it } from "./lib/effect"
  8. import { dynamicResponse } from "./lib/http"
  9. import { deltaChunk } from "./lib/openai-chunks"
  10. import { sseEvents } from "./lib/sse"
  11. const TargetJson = Schema.fromJsonString(Schema.Unknown)
  12. const decodeJson = Schema.decodeUnknownSync(TargetJson)
  13. describe("request option precedence", () => {
  14. test("deep-merges provider option records and replaces arrays, primitives, and null", () => {
  15. const merged = mergeProviderOptions(
  16. {
  17. openai: {
  18. include: ["route"],
  19. metadata: { route: true, shared: "route" },
  20. nullable: "route",
  21. primitive: "route",
  22. },
  23. },
  24. {
  25. openai: {
  26. include: ["model"],
  27. metadata: { model: true, shared: "model" },
  28. nullable: null,
  29. primitive: "model",
  30. },
  31. },
  32. { openai: { metadata: { request: true }, primitive: false } },
  33. )
  34. expect(merged).toEqual({
  35. openai: {
  36. include: ["model"],
  37. metadata: { route: true, model: true, request: true, shared: "model" },
  38. nullable: null,
  39. primitive: false,
  40. },
  41. })
  42. })
  43. it.effect("prepares bodies with route defaults, model defaults, and call options in order", () =>
  44. Effect.gen(function* () {
  45. const route = OpenAIChat.route.with({
  46. endpoint: { baseURL: "https://api.openai.test/v1/" },
  47. auth: Auth.bearer("test"),
  48. generation: { maxTokens: 10, temperature: 1, stop: ["route"] },
  49. providerOptions: { openai: { store: false, reasoningEffort: "low" } },
  50. })
  51. const model = route.model({
  52. id: "gpt-4o-mini",
  53. defaults: {
  54. generation: { maxTokens: 20, temperature: 0.5, frequencyPenalty: 0.25, stop: ["model"] },
  55. providerOptions: { openai: { reasoningEffort: "medium" } },
  56. },
  57. })
  58. const prepared = yield* LLMClient.prepare<OpenAIChat.OpenAIChatBody>(
  59. LLM.request({
  60. model,
  61. prompt: "Say hello.",
  62. generation: { maxTokens: 30, topP: 0.9, stop: ["request"] },
  63. providerOptions: { openai: { store: true } },
  64. }),
  65. )
  66. expect(prepared.body).toMatchObject({
  67. model: "gpt-4o-mini",
  68. stream: true,
  69. max_tokens: 30,
  70. temperature: 0.5,
  71. top_p: 0.9,
  72. frequency_penalty: 0.25,
  73. store: true,
  74. reasoning_effort: "medium",
  75. })
  76. expect(prepared.body.stop).toEqual(["request"])
  77. }),
  78. )
  79. it.effect("applies model HTTP defaults before request HTTP overlays", () =>
  80. LLMClient.generate(
  81. LLM.request({
  82. model: OpenAIChat.route
  83. .with({
  84. endpoint: { baseURL: "https://api.openai.test/v1/" },
  85. auth: Auth.bearer("fresh-key"),
  86. http: {
  87. body: { metadata: { route: true, shared: "route" }, value: "route" },
  88. headers: { "x-route": "route", "x-shared": "route" },
  89. query: { route: "1", shared: "route" },
  90. },
  91. })
  92. .model({
  93. id: "gpt-4o-mini",
  94. defaults: {
  95. http: {
  96. body: { metadata: { model: true, shared: "model" }, value: "model" },
  97. headers: { "x-model": "model", "x-shared": "model" },
  98. query: { model: "1", shared: "model" },
  99. },
  100. },
  101. }),
  102. prompt: "Say hello.",
  103. http: {
  104. body: { metadata: { request: true }, value: null },
  105. headers: { "x-request": "request" },
  106. query: { request: "1" },
  107. },
  108. }),
  109. ).pipe(
  110. Effect.provide(
  111. dynamicResponse((input) =>
  112. Effect.gen(function* () {
  113. const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
  114. expect(web.url).toBe("https://api.openai.test/v1/chat/completions?route=1&shared=model&model=1&request=1")
  115. expect(web.headers.get("authorization")).toBe("Bearer fresh-key")
  116. expect(web.headers.get("x-route")).toBe("route")
  117. expect(web.headers.get("x-model")).toBe("model")
  118. expect(web.headers.get("x-request")).toBe("request")
  119. expect(web.headers.get("x-shared")).toBe("model")
  120. expect(decodeJson(input.text)).toMatchObject({
  121. metadata: { route: true, model: true, request: true, shared: "model" },
  122. value: null,
  123. })
  124. return input.respond(sseEvents(deltaChunk({}, "stop")), {
  125. headers: { "content-type": "text/event-stream" },
  126. })
  127. }),
  128. ),
  129. ),
  130. ),
  131. )
  132. it.effect("rejects raw body overlays for protocol-owned roots", () =>
  133. Effect.gen(function* () {
  134. const model = OpenAIChat.route
  135. .with({ endpoint: { baseURL: "https://api.openai.test/v1/" }, auth: Auth.bearer("test") })
  136. .model({ id: "gpt-4o-mini" })
  137. const error = yield* LLMClient.prepare(
  138. LLM.request({
  139. model,
  140. prompt: "Say hello.",
  141. http: { body: { model: "gpt-5", messages: [], tools: [] } },
  142. }),
  143. ).pipe(Effect.flip)
  144. expect(error.reason).toMatchObject({
  145. _tag: "InvalidRequest",
  146. message: "http.body cannot overlay protocol-owned field(s): model, messages, tools",
  147. })
  148. }),
  149. )
  150. it.effect("uses model output limits after route limits and before call maxTokens", () =>
  151. Effect.gen(function* () {
  152. const route = AnthropicMessages.route.with({
  153. endpoint: { baseURL: "https://api.anthropic.test/v1/" },
  154. auth: Auth.header("x-api-key", "test"),
  155. limits: { output: 128 },
  156. })
  157. const model = route.model({ id: "claude-sonnet-4-5", defaults: { limits: { output: 64 } } })
  158. const withoutMaxTokens = yield* LLMClient.prepare<AnthropicMessages.AnthropicMessagesBody>(
  159. LLM.request({ model, prompt: "Say hello.", cache: "none" }),
  160. )
  161. const withMaxTokens = yield* LLMClient.prepare<AnthropicMessages.AnthropicMessagesBody>(
  162. LLM.request({ model, prompt: "Say hello.", cache: "none", generation: { maxTokens: 32 } }),
  163. )
  164. expect(withoutMaxTokens.body.max_tokens).toBe(64)
  165. expect(withMaxTokens.body.max_tokens).toBe(32)
  166. }),
  167. )
  168. })