bedrock-converse.test.ts 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724
  1. import { EventStreamCodec } from "@smithy/eventstream-codec"
  2. import { fromUtf8, toUtf8 } from "@smithy/util-utf8"
  3. import { describe, expect } from "bun:test"
  4. import { Effect } from "effect"
  5. import { CacheHint, LLM, Message, ToolCallPart, ToolChoice } from "../../src"
  6. import { LLMClient } from "../../src/route"
  7. import { AmazonBedrock } from "../../src/providers"
  8. import * as BedrockConverse from "../../src/protocols/bedrock-converse"
  9. import { it } from "../lib/effect"
  10. import { fixedResponse } from "../lib/http"
  11. import {
  12. eventSummary,
  13. expectWeatherToolLoop,
  14. runWeatherToolLoop,
  15. weatherTool,
  16. weatherToolLoopRequest,
  17. weatherToolName,
  18. } from "../recorded-scenarios"
  19. import { recordedTests } from "../recorded-test"
  20. const codec = new EventStreamCodec(toUtf8, fromUtf8)
  21. const utf8Encoder = new TextEncoder()
  22. // Build a single AWS event-stream frame for a Converse stream event. Each
  23. // frame carries `:message-type=event` + `:event-type=<name>` headers and a
  24. // JSON payload body.
  25. const eventFrame = (type: string, payload: object) =>
  26. codec.encode({
  27. headers: {
  28. ":message-type": { type: "string", value: "event" },
  29. ":event-type": { type: "string", value: type },
  30. ":content-type": { type: "string", value: "application/json" },
  31. },
  32. body: utf8Encoder.encode(JSON.stringify(payload)),
  33. })
  34. const concat = (frames: ReadonlyArray<Uint8Array>) => {
  35. const total = frames.reduce((sum, frame) => sum + frame.length, 0)
  36. const out = new Uint8Array(total)
  37. let offset = 0
  38. for (const frame of frames) {
  39. out.set(frame, offset)
  40. offset += frame.length
  41. }
  42. return out
  43. }
  44. const eventStreamBody = (...payloads: ReadonlyArray<readonly [string, object]>) =>
  45. concat(payloads.map(([type, payload]) => eventFrame(type, payload)))
  46. // Override the default SSE content-type with the binary event-stream type so
  47. // the cassette layer treats the body as bytes when recording.
  48. const fixedBytes = (bytes: Uint8Array) =>
  49. fixedResponse(bytes.slice().buffer, { headers: { "content-type": "application/vnd.amazon.eventstream" } })
  50. const model = AmazonBedrock.configure({
  51. baseURL: "https://bedrock-runtime.test",
  52. apiKey: "test-bearer",
  53. }).model("anthropic.claude-3-5-sonnet-20240620-v1:0")
  54. const baseRequest = LLM.request({
  55. id: "req_1",
  56. model,
  57. system: "You are concise.",
  58. prompt: "Say hello.",
  59. // Wire-shape assertions in this file predate the `cache: "auto"` default;
  60. // pin the policy off so they only exercise the lowering path itself.
  61. cache: "none",
  62. generation: { maxTokens: 64, temperature: 0 },
  63. })
  64. describe("Bedrock Converse route", () => {
  65. it.effect("prepares Converse target with system, inference config, and messages", () =>
  66. Effect.gen(function* () {
  67. const prepared = yield* LLMClient.prepare(baseRequest)
  68. expect(prepared.body).toEqual({
  69. modelId: "anthropic.claude-3-5-sonnet-20240620-v1:0",
  70. system: [{ text: "You are concise." }],
  71. messages: [{ role: "user", content: [{ text: "Say hello." }] }],
  72. inferenceConfig: { maxTokens: 64, temperature: 0 },
  73. })
  74. }),
  75. )
  76. it.effect("lowers chronological system updates to wrapped user text in order", () =>
  77. Effect.gen(function* () {
  78. const prepared = yield* LLMClient.prepare<BedrockConverse.BedrockConverseBody>(
  79. LLM.request({
  80. model,
  81. messages: [Message.user("Before."), Message.system("Update."), Message.assistant("After.")],
  82. cache: "none",
  83. }),
  84. )
  85. expect(prepared.body.messages).toEqual([
  86. { role: "user", content: [{ text: "Before." }, { text: "<system-update>\nUpdate.\n</system-update>" }] },
  87. { role: "assistant", content: [{ text: "After." }] },
  88. ])
  89. }),
  90. )
  91. it.effect("prepares tool config with toolSpec and toolChoice", () =>
  92. Effect.gen(function* () {
  93. const prepared = yield* LLMClient.prepare(
  94. LLM.updateRequest(baseRequest, {
  95. tools: [
  96. {
  97. name: "lookup",
  98. description: "Lookup data",
  99. inputSchema: { type: "object", properties: { query: { type: "string" } }, required: ["query"] },
  100. },
  101. ],
  102. toolChoice: ToolChoice.make({ type: "required" }),
  103. }),
  104. )
  105. expect(prepared.body).toMatchObject({
  106. toolConfig: {
  107. tools: [
  108. {
  109. toolSpec: {
  110. name: "lookup",
  111. description: "Lookup data",
  112. inputSchema: {
  113. json: { type: "object", properties: { query: { type: "string" } }, required: ["query"] },
  114. },
  115. },
  116. },
  117. ],
  118. toolChoice: { any: {} },
  119. },
  120. })
  121. }),
  122. )
  123. it.effect("lowers assistant tool-call + tool-result message history", () =>
  124. Effect.gen(function* () {
  125. const prepared = yield* LLMClient.prepare(
  126. LLM.request({
  127. id: "req_history",
  128. model,
  129. messages: [
  130. Message.user("What is the weather?"),
  131. Message.assistant([ToolCallPart.make({ id: "tool_1", name: "lookup", input: { query: "weather" } })]),
  132. Message.tool({ id: "tool_1", name: "lookup", result: { forecast: "sunny" } }),
  133. ],
  134. cache: "none",
  135. }),
  136. )
  137. expect(prepared.body).toMatchObject({
  138. messages: [
  139. { role: "user", content: [{ text: "What is the weather?" }] },
  140. {
  141. role: "assistant",
  142. content: [{ toolUse: { toolUseId: "tool_1", name: "lookup", input: { query: "weather" } } }],
  143. },
  144. {
  145. role: "user",
  146. content: [
  147. {
  148. toolResult: {
  149. toolUseId: "tool_1",
  150. content: [{ json: { forecast: "sunny" } }],
  151. status: "success",
  152. },
  153. },
  154. ],
  155. },
  156. ],
  157. })
  158. }),
  159. )
  160. it.effect("lowers image content in tool-result messages", () =>
  161. Effect.gen(function* () {
  162. const prepared = yield* LLMClient.prepare(
  163. LLM.request({
  164. id: "req_tool_image",
  165. model,
  166. messages: [
  167. Message.user("Capture the screen."),
  168. Message.assistant([ToolCallPart.make({ id: "tool_1", name: "screenshot", input: {} })]),
  169. Message.tool({
  170. id: "tool_1",
  171. name: "screenshot",
  172. result: {
  173. type: "content",
  174. value: [
  175. { type: "text", text: "Screenshot captured." },
  176. { type: "file", uri: "data:image/png;base64,AAAA", mime: "image/png" },
  177. ],
  178. },
  179. }),
  180. ],
  181. cache: "none",
  182. }),
  183. )
  184. expect(prepared.body).toMatchObject({
  185. messages: [
  186. { role: "user", content: [{ text: "Capture the screen." }] },
  187. {
  188. role: "assistant",
  189. content: [{ toolUse: { toolUseId: "tool_1", name: "screenshot", input: {} } }],
  190. },
  191. {
  192. role: "user",
  193. content: [
  194. {
  195. toolResult: {
  196. toolUseId: "tool_1",
  197. content: [{ text: "Screenshot captured." }, { image: { format: "png", source: { bytes: "AAAA" } } }],
  198. status: "success",
  199. },
  200. },
  201. ],
  202. },
  203. ],
  204. })
  205. }),
  206. )
  207. it.effect("decodes text-delta + messageStop + metadata usage from binary event stream", () =>
  208. Effect.gen(function* () {
  209. const body = eventStreamBody(
  210. ["messageStart", { role: "assistant" }],
  211. ["contentBlockDelta", { contentBlockIndex: 0, delta: { text: "Hello" } }],
  212. ["contentBlockDelta", { contentBlockIndex: 0, delta: { text: "!" } }],
  213. ["contentBlockStop", { contentBlockIndex: 0 }],
  214. ["messageStop", { stopReason: "end_turn" }],
  215. ["metadata", { usage: { inputTokens: 5, outputTokens: 2, totalTokens: 7 } }],
  216. )
  217. const response = yield* LLMClient.generate(baseRequest).pipe(Effect.provide(fixedBytes(body)))
  218. expect(response.text).toBe("Hello!")
  219. const finishes = response.events.filter((event) => event.type === "finish")
  220. // Bedrock splits the finish across `messageStop` (carries reason) and
  221. // `metadata` (carries usage). We consolidate them into a single
  222. // terminal `finish` event with both.
  223. expect(finishes).toHaveLength(1)
  224. expect(finishes[0]).toMatchObject({ type: "finish", reason: "stop" })
  225. expect(response.usage).toMatchObject({
  226. inputTokens: 5,
  227. outputTokens: 2,
  228. totalTokens: 7,
  229. })
  230. }),
  231. )
  232. it.effect("assembles streamed tool call input", () =>
  233. Effect.gen(function* () {
  234. const body = eventStreamBody(
  235. ["messageStart", { role: "assistant" }],
  236. [
  237. "contentBlockStart",
  238. {
  239. contentBlockIndex: 0,
  240. start: { toolUse: { toolUseId: "tool_1", name: "lookup" } },
  241. },
  242. ],
  243. ["contentBlockDelta", { contentBlockIndex: 0, delta: { toolUse: { input: '{"query"' } } }],
  244. ["contentBlockDelta", { contentBlockIndex: 0, delta: { toolUse: { input: ':"weather"}' } } }],
  245. ["contentBlockStop", { contentBlockIndex: 0 }],
  246. ["messageStop", { stopReason: "tool_use" }],
  247. )
  248. const response = yield* LLMClient.generate(
  249. LLM.updateRequest(baseRequest, {
  250. tools: [{ name: "lookup", description: "Lookup", inputSchema: { type: "object" } }],
  251. }),
  252. ).pipe(Effect.provide(fixedBytes(body)))
  253. expect(response.toolCalls).toEqual([
  254. { type: "tool-call", id: "tool_1", name: "lookup", input: { query: "weather" } },
  255. ])
  256. const events = response.events.filter((event) => event.type === "tool-input-delta")
  257. expect(events).toEqual([
  258. { type: "tool-input-delta", id: "tool_1", name: "lookup", text: '{"query"' },
  259. { type: "tool-input-delta", id: "tool_1", name: "lookup", text: ':"weather"}' },
  260. ])
  261. expect(response.events.at(-1)).toMatchObject({ type: "finish", reason: "tool-calls" })
  262. }),
  263. )
  264. it.effect("decodes reasoning deltas", () =>
  265. Effect.gen(function* () {
  266. const body = eventStreamBody(
  267. ["messageStart", { role: "assistant" }],
  268. ["contentBlockDelta", { contentBlockIndex: 0, delta: { reasoningContent: { text: "Let me think." } } }],
  269. ["contentBlockStop", { contentBlockIndex: 0 }],
  270. ["messageStop", { stopReason: "end_turn" }],
  271. )
  272. const response = yield* LLMClient.generate(baseRequest).pipe(Effect.provide(fixedBytes(body)))
  273. expect(response.reasoning).toBe("Let me think.")
  274. }),
  275. )
  276. it.effect("preserves streamed reasoning signatures for continuation lowering", () =>
  277. Effect.gen(function* () {
  278. const body = eventStreamBody(
  279. ["messageStart", { role: "assistant" }],
  280. ["contentBlockDelta", { contentBlockIndex: 0, delta: { reasoningContent: { text: "Let me think." } } }],
  281. ["contentBlockDelta", { contentBlockIndex: 0, delta: { reasoningContent: { signature: "sig_1" } } }],
  282. ["contentBlockStop", { contentBlockIndex: 0 }],
  283. ["messageStop", { stopReason: "end_turn" }],
  284. )
  285. const response = yield* LLMClient.generate(baseRequest).pipe(Effect.provide(fixedBytes(body)))
  286. const reasoning = response.events.find((event) => event.type === "reasoning-end")
  287. expect(reasoning).toEqual({
  288. type: "reasoning-end",
  289. id: "reasoning-0",
  290. providerMetadata: { bedrock: { signature: "sig_1" } },
  291. })
  292. const prepared = yield* LLMClient.prepare<BedrockConverse.BedrockConverseBody>(
  293. LLM.request({
  294. model,
  295. messages: [
  296. Message.assistant([
  297. { type: "reasoning", text: "Let me think.", providerMetadata: reasoning?.providerMetadata },
  298. ]),
  299. ],
  300. cache: "none",
  301. }),
  302. )
  303. expect(prepared.body.messages).toEqual([
  304. {
  305. role: "assistant",
  306. content: [{ reasoningContent: { reasoningText: { text: "Let me think.", signature: "sig_1" } } }],
  307. },
  308. ])
  309. }),
  310. )
  311. it.effect("emits provider-error for throttlingException", () =>
  312. Effect.gen(function* () {
  313. const body = eventStreamBody(
  314. ["messageStart", { role: "assistant" }],
  315. ["throttlingException", { message: "Slow down" }],
  316. )
  317. const response = yield* LLMClient.generate(baseRequest).pipe(Effect.provide(fixedBytes(body)))
  318. expect(response.events.find((event) => event.type === "provider-error")).toEqual({
  319. type: "provider-error",
  320. message: "Slow down",
  321. retryable: true,
  322. })
  323. }),
  324. )
  325. it.effect("classifies input-too-long validation exceptions", () =>
  326. Effect.gen(function* () {
  327. const response = yield* LLMClient.generate(baseRequest).pipe(
  328. Effect.provide(
  329. fixedBytes(eventStreamBody(["validationException", { message: "Input is too long for requested model" }])),
  330. ),
  331. )
  332. expect(response.events.find((event) => event.type === "provider-error")).toEqual({
  333. type: "provider-error",
  334. message: "Input is too long for requested model",
  335. classification: "context-overflow",
  336. retryable: false,
  337. })
  338. }),
  339. )
  340. it.effect("rejects requests with no auth path", () =>
  341. Effect.gen(function* () {
  342. const unsignedModel = AmazonBedrock.configure({
  343. baseURL: "https://bedrock-runtime.test",
  344. }).model("anthropic.claude-3-5-sonnet-20240620-v1:0")
  345. const error = yield* LLMClient.generate(LLM.updateRequest(baseRequest, { model: unsignedModel })).pipe(
  346. Effect.provide(fixedBytes(eventStreamBody(["messageStop", { stopReason: "end_turn" }]))),
  347. Effect.flip,
  348. )
  349. expect(error.message).toContain("Bedrock Converse requires either route bearer auth or AWS credentials")
  350. }),
  351. )
  352. it.effect("signs requests with SigV4 when AWS credentials are provided (deterministic plumbing check)", () =>
  353. Effect.gen(function* () {
  354. const signed = AmazonBedrock.configure({
  355. baseURL: "https://bedrock-runtime.test",
  356. credentials: {
  357. region: "us-east-1",
  358. accessKeyId: "AKIAIOSFODNN7EXAMPLE",
  359. secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
  360. },
  361. }).model("anthropic.claude-3-5-sonnet-20240620-v1:0")
  362. const prepared = yield* LLMClient.prepare(LLM.updateRequest(baseRequest, { model: signed }))
  363. expect(prepared.route).toBe("bedrock-converse")
  364. expect(prepared.model).toBe(signed)
  365. }),
  366. )
  367. it.effect("emits cachePoint markers after system, user-text, and assistant-text with cache hints", () =>
  368. Effect.gen(function* () {
  369. const cache = new CacheHint({ type: "ephemeral" })
  370. const prepared = yield* LLMClient.prepare(
  371. LLM.request({
  372. id: "req_cache",
  373. model,
  374. system: [{ type: "text", text: "System prefix.", cache }],
  375. messages: [
  376. Message.user([{ type: "text", text: "User prefix.", cache }]),
  377. Message.assistant([{ type: "text", text: "Assistant prefix.", cache }]),
  378. ],
  379. generation: { maxTokens: 16, temperature: 0 },
  380. }),
  381. )
  382. expect(prepared.body).toMatchObject({
  383. // System: text block followed by cachePoint marker.
  384. system: [{ text: "System prefix." }, { cachePoint: { type: "default" } }],
  385. messages: [
  386. {
  387. role: "user",
  388. content: [{ text: "User prefix." }, { cachePoint: { type: "default" } }],
  389. },
  390. {
  391. role: "assistant",
  392. content: [{ text: "Assistant prefix." }, { cachePoint: { type: "default" } }],
  393. },
  394. ],
  395. })
  396. }),
  397. )
  398. it.effect("does not emit cachePoint when no cache hint is set", () =>
  399. Effect.gen(function* () {
  400. const prepared = yield* LLMClient.prepare(baseRequest)
  401. expect(prepared.body).toMatchObject({
  402. system: [{ text: "You are concise." }],
  403. messages: [{ role: "user", content: [{ text: "Say hello." }] }],
  404. })
  405. }),
  406. )
  407. it.effect("lowers image media into Bedrock image blocks", () =>
  408. Effect.gen(function* () {
  409. const prepared = yield* LLMClient.prepare(
  410. LLM.request({
  411. id: "req_image",
  412. model,
  413. messages: [
  414. Message.user([
  415. { type: "text", text: "What is in this image?" },
  416. { type: "media", mediaType: "image/png", data: "AAAA" },
  417. { type: "media", mediaType: "image/jpeg", data: "BBBB" },
  418. { type: "media", mediaType: "image/jpg", data: "CCCC" },
  419. { type: "media", mediaType: "image/webp", data: "DDDD" },
  420. ]),
  421. ],
  422. cache: "none",
  423. }),
  424. )
  425. expect(prepared.body).toMatchObject({
  426. messages: [
  427. {
  428. role: "user",
  429. content: [
  430. { text: "What is in this image?" },
  431. { image: { format: "png", source: { bytes: "AAAA" } } },
  432. { image: { format: "jpeg", source: { bytes: "BBBB" } } },
  433. // image/jpg is a non-standard alias; we map it to jpeg.
  434. { image: { format: "jpeg", source: { bytes: "CCCC" } } },
  435. { image: { format: "webp", source: { bytes: "DDDD" } } },
  436. ],
  437. },
  438. ],
  439. })
  440. }),
  441. )
  442. it.effect("base64-encodes Uint8Array image bytes", () =>
  443. Effect.gen(function* () {
  444. const prepared = yield* LLMClient.prepare(
  445. LLM.request({
  446. id: "req_image_bytes",
  447. model,
  448. messages: [Message.user([{ type: "media", mediaType: "image/png", data: new Uint8Array([1, 2, 3, 4, 5]) }])],
  449. }),
  450. )
  451. // Buffer.from([1,2,3,4,5]).toString("base64") === "AQIDBAU="
  452. expect(prepared.body).toMatchObject({
  453. messages: [
  454. {
  455. role: "user",
  456. content: [{ image: { format: "png", source: { bytes: "AQIDBAU=" } } }],
  457. },
  458. ],
  459. })
  460. }),
  461. )
  462. it.effect("lowers document media into Bedrock document blocks with format and name", () =>
  463. Effect.gen(function* () {
  464. const prepared = yield* LLMClient.prepare(
  465. LLM.request({
  466. id: "req_doc",
  467. model,
  468. messages: [
  469. Message.user([
  470. { type: "media", mediaType: "application/pdf", data: "UERGREFUQQ==", filename: "report.pdf" },
  471. { type: "media", mediaType: "text/csv", data: "Q1NWREFUQQ==" },
  472. ]),
  473. ],
  474. }),
  475. )
  476. expect(prepared.body).toMatchObject({
  477. messages: [
  478. {
  479. role: "user",
  480. content: [
  481. // Filename round-trips when supplied.
  482. { document: { format: "pdf", name: "report.pdf", source: { bytes: "UERGREFUQQ==" } } },
  483. // Falls back to a stable placeholder when filename is missing.
  484. { document: { format: "csv", name: "document.csv", source: { bytes: "Q1NWREFUQQ==" } } },
  485. ],
  486. },
  487. ],
  488. })
  489. }),
  490. )
  491. it.effect("rejects unsupported image media types", () =>
  492. Effect.gen(function* () {
  493. const error = yield* LLMClient.prepare(
  494. LLM.request({
  495. id: "req_bad_image",
  496. model,
  497. messages: [Message.user([{ type: "media", mediaType: "image/svg+xml", data: "x" }])],
  498. }),
  499. ).pipe(Effect.flip)
  500. expect(error.message).toContain("Bedrock Converse does not support image media type image/svg+xml")
  501. }),
  502. )
  503. it.effect("rejects unsupported document media types", () =>
  504. Effect.gen(function* () {
  505. const error = yield* LLMClient.prepare(
  506. LLM.request({
  507. id: "req_bad_doc",
  508. model,
  509. messages: [Message.user([{ type: "media", mediaType: "application/x-tar", data: "x", filename: "a.tar" }])],
  510. }),
  511. ).pipe(Effect.flip)
  512. expect(error.message).toContain("Bedrock Converse does not support media type application/x-tar")
  513. }),
  514. )
  515. it.effect("maps ttlSeconds >= 3600 to cachePoint ttl: '1h'", () =>
  516. Effect.gen(function* () {
  517. const cache = new CacheHint({ type: "ephemeral", ttlSeconds: 3600 })
  518. const prepared = yield* LLMClient.prepare(
  519. LLM.request({
  520. model,
  521. system: [{ type: "text", text: "system", cache }],
  522. prompt: "hi",
  523. }),
  524. )
  525. expect(prepared.body).toMatchObject({
  526. system: [{ text: "system" }, { cachePoint: { type: "default", ttl: "1h" } }],
  527. })
  528. }),
  529. )
  530. it.effect("appends cachePoint after marked tool definitions and tool-result blocks", () =>
  531. Effect.gen(function* () {
  532. const cache = new CacheHint({ type: "ephemeral" })
  533. const prepared = yield* LLMClient.prepare(
  534. LLM.request({
  535. model,
  536. tools: [{ name: "lookup", description: "lookup", inputSchema: { type: "object", properties: {} }, cache }],
  537. messages: [
  538. Message.user("What's the weather?"),
  539. Message.assistant([ToolCallPart.make({ id: "call_1", name: "lookup", input: {} })]),
  540. Message.tool({ id: "call_1", name: "lookup", result: { temp: 72 }, cache }),
  541. ],
  542. cache: "none",
  543. }),
  544. )
  545. expect(prepared.body).toMatchObject({
  546. toolConfig: {
  547. tools: [{ toolSpec: { name: "lookup" } }, { cachePoint: { type: "default" } }],
  548. },
  549. messages: [
  550. { role: "user", content: [{ text: "What's the weather?" }] },
  551. { role: "assistant", content: [{ toolUse: { toolUseId: "call_1" } }] },
  552. {
  553. role: "user",
  554. content: [{ toolResult: { toolUseId: "call_1" } }, { cachePoint: { type: "default" } }],
  555. },
  556. ],
  557. })
  558. }),
  559. )
  560. it.effect("drops cachePoint markers past the 4-per-request cap", () =>
  561. Effect.gen(function* () {
  562. const cache = new CacheHint({ type: "ephemeral" })
  563. const prepared = yield* LLMClient.prepare(
  564. LLM.request({
  565. model,
  566. system: [
  567. { type: "text", text: "a", cache },
  568. { type: "text", text: "b", cache },
  569. { type: "text", text: "c", cache },
  570. { type: "text", text: "d", cache },
  571. { type: "text", text: "e", cache },
  572. { type: "text", text: "f", cache },
  573. ],
  574. prompt: "hi",
  575. }),
  576. )
  577. const system = (prepared.body as { system: Array<{ cachePoint?: unknown }> }).system
  578. expect(system.filter((part) => "cachePoint" in part)).toHaveLength(4)
  579. }),
  580. )
  581. })
  582. // Live recorded integration tests. Run with `RECORD=true AWS_ACCESS_KEY_ID=...
  583. // AWS_SECRET_ACCESS_KEY=... [AWS_SESSION_TOKEN=...] bun run test ...` to refresh
  584. // cassettes; replay is the default and works without credentials.
  585. //
  586. // Region is pinned to us-east-1 in tests so the request URL is stable across
  587. // machines on replay. If you need to record from a different region (e.g. your
  588. // account has access elsewhere), pass `BEDROCK_RECORDING_REGION=eu-west-1` —
  589. // but then commit the resulting cassette and others should record from the
  590. // same region too.
  591. const RECORDING_REGION = process.env.BEDROCK_RECORDING_REGION ?? "us-east-1"
  592. const recordedModel = () =>
  593. AmazonBedrock.configure({
  594. // Most newer Anthropic models on Bedrock require a cross-region inference
  595. // profile (`us.` prefix). Nova does not require an Anthropic use-case form
  596. // and is on-demand-throughput accessible by default for most accounts.
  597. credentials: {
  598. region: RECORDING_REGION,
  599. accessKeyId: process.env.AWS_ACCESS_KEY_ID ?? "fixture",
  600. secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY ?? "fixture",
  601. sessionToken: process.env.AWS_SESSION_TOKEN,
  602. },
  603. }).model(process.env.BEDROCK_MODEL_ID ?? "us.amazon.nova-micro-v1:0")
  604. const recorded = recordedTests({
  605. prefix: "bedrock-converse",
  606. provider: "amazon-bedrock",
  607. protocol: "bedrock-converse",
  608. requires: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"],
  609. })
  610. describe("Bedrock Converse recorded", () => {
  611. recorded.effect("streams text", () =>
  612. Effect.gen(function* () {
  613. const llm = yield* LLMClient.Service
  614. const response = yield* llm.generate(
  615. LLM.request({
  616. id: "recorded_bedrock_text",
  617. model: recordedModel(),
  618. system: "Reply with the single word 'Hello'.",
  619. prompt: "Say hello.",
  620. cache: "none",
  621. generation: { maxTokens: 16, temperature: 0 },
  622. }),
  623. )
  624. expect(eventSummary(response.events)).toEqual([
  625. { type: "text", value: "Hello" },
  626. { type: "finish", reason: "stop", usage: { inputTokens: 12, outputTokens: 2, totalTokens: 14 } },
  627. ])
  628. }),
  629. )
  630. recorded.effect.with("streams a tool call", { tags: ["tool"] }, () =>
  631. Effect.gen(function* () {
  632. const llm = yield* LLMClient.Service
  633. const response = yield* llm.generate(
  634. LLM.request({
  635. id: "recorded_bedrock_tool_call",
  636. model: recordedModel(),
  637. system: "Call tools exactly as requested.",
  638. prompt: "Call get_weather with city exactly Paris.",
  639. tools: [weatherTool],
  640. toolChoice: ToolChoice.make(weatherTool),
  641. cache: "none",
  642. generation: { maxTokens: 80, temperature: 0 },
  643. }),
  644. )
  645. expect(eventSummary(response.events)).toEqual([
  646. { type: "tool-call", name: weatherToolName, input: { city: "Paris" } },
  647. { type: "finish", reason: "tool-calls", usage: { inputTokens: 419, outputTokens: 16, totalTokens: 435 } },
  648. ])
  649. }),
  650. )
  651. recorded.effect.with("drives a tool loop", { tags: ["tool", "tool-loop", "golden"] }, () =>
  652. Effect.gen(function* () {
  653. expectWeatherToolLoop(
  654. yield* runWeatherToolLoop(
  655. weatherToolLoopRequest({
  656. id: "recorded_bedrock_tool_loop",
  657. model: recordedModel(),
  658. }),
  659. ),
  660. )
  661. }),
  662. )
  663. })