fix(ai): layer prompt cache breakpoints (#38725)

.changeset/quick-caches-roll.md

@@ -0,0 +1,5 @@
+---
+"@opencode-ai/ai": patch
+---
+
+Improve Anthropic and Bedrock prompt reuse with layered cache breakpoints that roll through long tool loops.

packages/ai/README.md

@@ -207,7 +207,9 @@ Prompt caching is **on by default**. Every `LLMRequest` resolves to `cache: "aut
 
 ### Auto placement
 
-`"auto"` places three breakpoints — last tool definition, last system part, latest user message. The last-user-message boundary is the load-bearing detail: in a tool-use loop, a single user turn expands into many assistant/tool round-trips, all sharing that prefix. Caching at that boundary lets every intra-turn API call hit.
+`"auto"` places up to four breakpoints — the last tool definition, the first system part, the last system part when distinct, and the final message boundary. These expose successively larger reusable prefixes for tools, the base agent, project instructions, and the active conversation. The rolling final-message boundary is the load-bearing detail in tool loops: it advances on every request so the previous cache entry stays within Anthropic's 20-block lookback.
+
+Tools precede every system and conversation block in the provider prefix, so tool definitions must remain byte-stable and deterministically ordered for downstream breakpoints to remain reusable.
 
 The math justifies the default: Anthropic's 5-minute cache write is 1.25× base, read is 0.1×, so a single reuse within 5 minutes already wins. One-shot completions below the per-model minimum-cacheable-token threshold silently no-op on the wire, so the worst case is harmless.
 
@@ -235,7 +237,7 @@ cache: {
 
 ### Manual hints
 
-Inline `CacheHint` on any text / system / tool / tool-result part overrides automatic placement. The auto policy preserves manual hints; it only fills gaps.
+Inline `CacheHint` on any text / system / tool / tool-result part overrides automatic placement. The auto policy preserves manual hints, counts them against Anthropic and Bedrock's four-breakpoint limit, and only fills the remaining slots.
 
 ```ts
 LLM.request({
@@ -251,8 +253,8 @@ LLM.request({
 
 | Protocol                | `cache: "auto"`                                                           |
 | ----------------------- | ------------------------------------------------------------------------- |
-| Anthropic Messages      | emits up to 3 `cache_control` markers (4-breakpoint cap enforced)         |
-| Bedrock Converse        | emits up to 3 `cachePoint` blocks (4-breakpoint cap enforced)             |
+| Anthropic Messages      | emits up to 4 `cache_control` markers (4-breakpoint cap enforced)         |
+| Bedrock Converse        | emits up to 4 `cachePoint` blocks (4-breakpoint cap enforced)             |
 | OpenAI Chat / Responses | no-op (implicit caching above 1024 tokens)                                |
 | Gemini                  | no-op (implicit caching on 2.5+; explicit `CachedContent` is out-of-band) |
 

packages/ai/src/cache-policy.ts

@@ -2,32 +2,31 @@
 // the policy designates. Runs once at compile time, before the per-protocol
 // body builder, so the existing inline-hint lowering path handles the rest.
 //
-// The default `"auto"` shape places one breakpoint at the last tool definition,
-// one at the last system part, and one at the latest user message. This
-// matches what production agent harnesses (LangChain's caching middleware,
-// kern-ai's 10x cost-reduction playbook) converge on for tool-use loops: the
-// latest user message stays put while a single turn explodes into many
-// assistant/tool round-trips, so caching at that boundary lets every
-// intra-turn API call hit the prefix.
+// The default `"auto"` shape places breakpoints at the last tool definition,
+// the first and last distinct system parts, and the conversation tail. This
+// exposes reusable tool, base-agent, project, and session prefixes while
+// advancing the tail after each tool result keeps the previous cache entry
+// within Anthropic's 20-block lookback during long agent turns.
 //
-// Manual `cache: CacheHint` placements on individual parts are preserved —
-// this function only fills gaps the caller left empty.
+// Manual `cache: CacheHint` placements on individual parts are preserved and
+// count against the four-breakpoint budget; auto only fills remaining slots.
 import { CacheHint, type CachePolicy, type CachePolicyObject } from "./schema/options"
 import { LLMRequest, Message, ToolDefinition, type ContentPart } from "./schema/messages"
 
 const AUTO: CachePolicyObject = {
   tools: true,
   system: true,
-  messages: "latest-user-message",
+  messages: { tail: 1 },
 }
 
 const NONE: CachePolicyObject = {}
+const BREAKPOINT_CAP = 4
 
 // Resolution rules:
 //   - undefined   → "auto" — caching is on by default. The math favors it:
 //                   Anthropic 5m-cache write is 1.25x base, read is 0.1x,
 //                   so a single reuse within 5 minutes already wins.
-//   - "auto"      → tools + system + latest user msg.
+//   - "auto"      → tools + first/last system + final message boundary.
 //   - "none"      → no auto placement; manual `CacheHint`s still flow.
 //   - object form → exactly what the caller asked for.
 const resolve = (policy: CachePolicy | undefined): CachePolicyObject => {
@@ -44,18 +43,32 @@ const RESPECTS_INLINE_HINTS = new Set(["anthropic-messages", "bedrock-converse"]
 const makeHint = (ttlSeconds: number | undefined): CacheHint =>
   ttlSeconds !== undefined ? new CacheHint({ type: "ephemeral", ttlSeconds }) : new CacheHint({ type: "ephemeral" })
 
-const markLastTool = (tools: ReadonlyArray<ToolDefinition>, hint: CacheHint): ReadonlyArray<ToolDefinition> => {
+interface Budget {
+  remaining: number
+}
+
+const markLastTool = (
+  tools: ReadonlyArray<ToolDefinition>,
+  hint: CacheHint,
+  budget: Budget,
+): ReadonlyArray<ToolDefinition> => {
   if (tools.length === 0) return tools
   const last = tools.length - 1
-  if (tools[last]!.cache) return tools
+  if (tools[last]!.cache || budget.remaining === 0) return tools
+  budget.remaining -= 1
   return tools.map((tool, i) => (i === last ? new ToolDefinition({ ...tool, cache: hint }) : tool))
 }
 
-const markLastSystem = (system: LLMRequest["system"], hint: CacheHint): LLMRequest["system"] => {
+const markSystemBoundaries = (system: LLMRequest["system"], hint: CacheHint, budget: Budget): LLMRequest["system"] => {
   if (system.length === 0) return system
-  const last = system.length - 1
-  if (system[last]!.cache) return system
-  return system.map((part, i) => (i === last ? { ...part, cache: hint } : part))
+  let changed = false
+  const next = system.map((part, index) => {
+    if ((index !== 0 && index !== system.length - 1) || part.cache || budget.remaining === 0) return part
+    budget.remaining -= 1
+    changed = true
+    return { ...part, cache: hint }
+  })
+  return changed ? next : system
 }
 
 const lastIndexOfRole = (messages: ReadonlyArray<Message>, role: Message["role"]): number =>
@@ -64,14 +77,20 @@ const lastIndexOfRole = (messages: ReadonlyArray<Message>, role: Message["role"]
 // Mark the last text part of `messages[index]`. If no text part exists, mark
 // the last content part regardless of type — that's the breakpoint position
 // in tool-result-only messages too.
-const markMessageAt = (messages: ReadonlyArray<Message>, index: number, hint: CacheHint): ReadonlyArray<Message> => {
+const markMessageAt = (
+  messages: ReadonlyArray<Message>,
+  index: number,
+  hint: CacheHint,
+  budget: Budget,
+): ReadonlyArray<Message> => {
   if (index < 0 || index >= messages.length) return messages
   const target = messages[index]!
   if (target.content.length === 0) return messages
   const lastTextIndex = target.content.findLastIndex((part) => part.type === "text")
   const markAt = lastTextIndex >= 0 ? lastTextIndex : target.content.length - 1
   const existing = target.content[markAt]!
-  if ("cache" in existing && existing.cache) return messages
+  if (("cache" in existing && existing.cache) || budget.remaining === 0) return messages
+  budget.remaining -= 1
   const nextContent = target.content.map((part, i) => (i === markAt ? ({ ...part, cache: hint } as ContentPart) : part))
   const next = new Message({ ...target, content: nextContent })
   // Single pass over `messages`, substituting the one updated entry. Long
@@ -86,25 +105,42 @@ const markMessages = (
   messages: ReadonlyArray<Message>,
   strategy: NonNullable<CachePolicyObject["messages"]>,
   hint: CacheHint,
+  budget: Budget,
 ): ReadonlyArray<Message> => {
   if (messages.length === 0) return messages
-  if (strategy === "latest-user-message") return markMessageAt(messages, lastIndexOfRole(messages, "user"), hint)
-  if (strategy === "latest-assistant") return markMessageAt(messages, lastIndexOfRole(messages, "assistant"), hint)
+  if (strategy === "latest-user-message")
+    return markMessageAt(messages, lastIndexOfRole(messages, "user"), hint, budget)
+  if (strategy === "latest-assistant")
+    return markMessageAt(messages, lastIndexOfRole(messages, "assistant"), hint, budget)
   const start = Math.max(0, messages.length - strategy.tail)
   let next = messages
-  for (let i = start; i < messages.length; i++) next = markMessageAt(next, i, hint)
+  for (let i = start; i < messages.length; i++) next = markMessageAt(next, i, hint, budget)
   return next
 }
 
+const countHints = (request: LLMRequest) =>
+  request.tools.reduce((count, tool) => count + (tool.cache === undefined ? 0 : 1), 0) +
+  request.system.reduce((count, part) => count + (part.cache === undefined ? 0 : 1), 0) +
+  request.messages.reduce(
+    (count, message) =>
+      count +
+      message.content.reduce(
+        (contentCount, part) => contentCount + ("cache" in part && part.cache !== undefined ? 1 : 0),
+        0,
+      ),
+    0,
+  )
+
 export const applyCachePolicy = (request: LLMRequest): LLMRequest => {
   if (!RESPECTS_INLINE_HINTS.has(request.model.route.id)) return request
   const policy = resolve(request.cache)
   if (!policy.tools && !policy.system && !policy.messages) return request
 
   const hint = makeHint(policy.ttlSeconds)
-  const tools = policy.tools ? markLastTool(request.tools, hint) : request.tools
-  const system = policy.system ? markLastSystem(request.system, hint) : request.system
-  const messages = policy.messages ? markMessages(request.messages, policy.messages, hint) : request.messages
+  const budget = { remaining: Math.max(0, BREAKPOINT_CAP - countHints(request)) }
+  const tools = policy.tools ? markLastTool(request.tools, hint, budget) : request.tools
+  const system = policy.system ? markSystemBoundaries(request.system, hint, budget) : request.system
+  const messages = policy.messages ? markMessages(request.messages, policy.messages, hint, budget) : request.messages
 
   if (tools === request.tools && system === request.system && messages === request.messages) return request
   return LLMRequest.update(request, { tools, system, messages })

packages/ai/src/schema/options.ts

@@ -251,11 +251,11 @@ export class CacheHint extends Schema.Class<CacheHint>("LLM.CacheHint")({
 // Auto-placement policy for prompt caching. The protocol-neutral lowering step
 // reads this and injects `CacheHint`s at the configured boundaries; the
 // per-protocol body builders then translate those hints into wire markers as
-// usual. `"auto"` is the recommended default for agent loops — it places one
-// breakpoint at the last tool definition, one at the last system part, and one
-// at the latest user message. The combination of provider invalidation
-// hierarchy (tools → system → messages) and Anthropic/Bedrock's 20-block
-// lookback means three trailing breakpoints reliably cover the static prefix.
+// usual. `"auto"` is the recommended default for agent loops — it places
+// breakpoints at the last tool definition, the first and last distinct system
+// parts, and the conversation tail. The rolling message breakpoint keeps a
+// prior cache entry within Anthropic/Bedrock's 20-block lookback during long
+// tool loops.
 //
 // Pass `"none"` to opt out entirely (the legacy behavior). Pass the granular
 // object form to override individual choices.

packages/ai/test/cache-policy.test.ts

@@ -39,8 +39,8 @@ describe("applyCachePolicy", () => {
         }),
       )
 
-      // No explicit cache field → auto policy fires → last system part + latest
-      // user message both get cache_control markers.
+      // A single system block is both the first and last boundary, so the auto
+      // policy deduplicates it and still marks the conversation tail.
       expect(prepared.body).toMatchObject({
         system: [{ type: "text", text: "You are concise.", cache_control: { type: "ephemeral" } }],
         messages: [{ role: "user", content: [{ type: "text", text: "hi", cache_control: { type: "ephemeral" } }] }],
@@ -48,12 +48,15 @@ describe("applyCachePolicy", () => {
     }),
   )
 
-  it.effect("'auto' marks the last tool, last system part, and latest user message on Anthropic", () =>
+  it.effect("'auto' marks the last tool, first and last system parts, and final message boundary on Anthropic", () =>
     Effect.gen(function* () {
       const prepared = yield* LLMClient.prepare(
         LLM.request({
           model: anthropicModel,
-          system: "Sys A",
+          system: [
+            { type: "text", text: "Base agent" },
+            { type: "text", text: "Project instructions" },
+          ],
           tools: [{ name: "t1", description: "t1", inputSchema: { type: "object", properties: {} } }],
           messages: [
             Message.user("first user"),
@@ -66,7 +69,10 @@ describe("applyCachePolicy", () => {
 
       expect(prepared.body).toMatchObject({
         tools: [{ name: "t1", cache_control: { type: "ephemeral" } }],
-        system: [{ type: "text", text: "Sys A", cache_control: { type: "ephemeral" } }],
+        system: [
+          { type: "text", text: "Base agent", cache_control: { type: "ephemeral" } },
+          { type: "text", text: "Project instructions", cache_control: { type: "ephemeral" } },
+        ],
         messages: [
           { role: "user", content: [{ type: "text", text: "first user" }] },
           { role: "assistant", content: [{ type: "text", text: "assistant reply" }] },
@@ -120,7 +126,10 @@ describe("applyCachePolicy", () => {
       const prepared = yield* LLMClient.prepare(
         LLM.request({
           model: bedrockModel,
-          system: "Sys",
+          system: [
+            { type: "text", text: "Base agent" },
+            { type: "text", text: "Project instructions" },
+          ],
           tools: [{ name: "t1", description: "t1", inputSchema: { type: "object", properties: {} } }],
           messages: [Message.user("first user"), Message.assistant("reply"), Message.user("latest user")],
           cache: "auto",
@@ -131,7 +140,12 @@ describe("applyCachePolicy", () => {
         toolConfig: {
           tools: [{ toolSpec: { name: "t1" } }, { cachePoint: { type: "default" } }],
         },
-        system: [{ text: "Sys" }, { cachePoint: { type: "default" } }],
+        system: [
+          { text: "Base agent" },
+          { cachePoint: { type: "default" } },
+          { text: "Project instructions" },
+          { cachePoint: { type: "default" } },
+        ],
         messages: [
           { role: "user", content: [{ text: "first user" }] },
           { role: "assistant", content: [{ text: "reply" }] },
@@ -193,9 +207,55 @@ describe("applyCachePolicy", () => {
         }),
       )
 
-      const body = prepared.body as { system: Array<{ text: string; cache_control?: unknown }> }
+      const body = prepared.body as {
+        system: Array<{ text: string; cache_control?: unknown }>
+        messages: Array<{ content: Array<{ cache_control?: unknown }> }>
+      }
       expect(body.system[0]?.cache_control).toEqual({ type: "ephemeral", ttl: "1h" })
       expect(body.system[1]?.cache_control).toEqual({ type: "ephemeral" })
+      expect(body.messages[0]?.content[0]?.cache_control).toEqual({ type: "ephemeral" })
+    }),
+  )
+
+  it.effect("auto policy stays within the four-breakpoint cap when preserving manual hints", () =>
+    Effect.gen(function* () {
+      const request = LLM.request({
+        model: anthropicModel,
+        system: [
+          { type: "text", text: "Base agent" },
+          {
+            type: "text",
+            text: "Manual context",
+            cache: new CacheHint({ type: "ephemeral", ttlSeconds: 3600 }),
+          },
+          { type: "text", text: "Project instructions" },
+        ],
+        tools: [{ name: "t1", description: "t1", inputSchema: { type: "object", properties: {} } }],
+        prompt: "hi",
+        cache: "auto",
+      })
+      const applied = applyCachePolicy(request)
+      expect(applied.tools[0]?.cache).toBeDefined()
+      expect(applied.system.map((part) => part.cache !== undefined)).toEqual([true, true, true])
+      const tail = applied.messages[0]!.content[0]!
+      expect("cache" in tail ? tail.cache : undefined).toBeUndefined()
+      expect(applyCachePolicy(applied)).toBe(applied)
+
+      const prepared = yield* LLMClient.prepare(request)
+
+      const body = prepared.body as {
+        tools: Array<{ cache_control?: unknown }>
+        system: Array<{ cache_control?: unknown }>
+        messages: Array<{ content: Array<{ cache_control?: unknown }> }>
+      }
+      const marked = [
+        ...body.tools.map((tool) => tool.cache_control),
+        ...body.system.map((part) => part.cache_control),
+        ...body.messages.flatMap((message) => message.content.map((part) => part.cache_control)),
+      ].filter((cache) => cache !== undefined)
+      expect(marked).toHaveLength(4)
+      expect(body.system[1]?.cache_control).toEqual({ type: "ephemeral", ttl: "1h" })
+      expect(body.messages[0]?.content[0]?.cache_control).toBeUndefined()
     }),
   )
 

packages/ai/test/provider/anthropic-messages-cache.recorded.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect } from "bun:test"
 import { Effect } from "effect"
-import { CacheHint, LLM } from "../../src"
+import { CacheHint, LLM, LLMRequest, Message, ToolCallPart, ToolDefinition } from "../../src"
 import { LLMClient } from "../../src/route"
 import * as Anthropic from "../../src/providers/anthropic"
 import { LARGE_CACHEABLE_SYSTEM } from "../recorded-scenarios"
@@ -24,6 +24,39 @@ const cacheRequest = LLM.request({
   generation: { maxTokens: 16, temperature: 0 },
 })
 
+const lookup = ToolDefinition.make({
+  name: "lookup",
+  description: "Look up a fixture value.",
+  inputSchema: {
+    type: "object",
+    properties: { index: { type: "number" } },
+    required: ["index"],
+    additionalProperties: false,
+  },
+})
+const longToolTurn = [
+  Message.user("Run the fixture lookups."),
+  ...Array.from({ length: 11 }, (_, index) => {
+    const id = `lookup_${index}`
+    return [
+      Message.assistant(ToolCallPart.make({ id, name: lookup.name, input: { index } })),
+      Message.tool({
+        id,
+        name: lookup.name,
+        result: `Fixture result ${index}. `.repeat(80),
+      }),
+    ]
+  }).flat(),
+]
+const longToolTurnRequest = LLM.request({
+  id: "recorded_anthropic_cache_long_tool_turn",
+  model,
+  system: LARGE_CACHEABLE_SYSTEM,
+  messages: longToolTurn,
+  tools: [lookup],
+  generation: { maxTokens: 16, temperature: 0 },
+})
+
 const recorded = recordedTests({
   prefix: "anthropic-messages-cache",
   provider: "anthropic",
@@ -50,4 +83,28 @@ describe("Anthropic Messages cache recorded", () => {
       expect(second.usage?.cacheReadInputTokens ?? 0).toBeGreaterThan(0)
     }),
   )
+
+  recorded.effect.with("keeps a long tool turn inside the cache lookback", { tags: ["cache", "tool"] }, () =>
+    Effect.gen(function* () {
+      const first = yield* LLMClient.generate(longToolTurnRequest)
+      const firstRead = first.usage?.cacheReadInputTokens ?? 0
+      const firstWrite = first.usage?.cacheWriteInputTokens ?? 0
+      const firstCached = firstRead + firstWrite
+      // The prefix may already be warm when recording, so either a read or a
+      // write establishes that Anthropic recognized the cache boundary.
+      expect(firstCached).toBeGreaterThan(0)
+
+      const second = yield* LLMClient.generate(
+        LLMRequest.update(longToolTurnRequest, {
+          messages: [
+            ...longToolTurn,
+            Message.assistant("The fixture lookups are complete."),
+            Message.user("Reply exactly: OK"),
+          ],
+        }),
+      )
+      expect(second.usage?.cacheReadInputTokens ?? 0).toBeGreaterThanOrEqual(firstCached)
+      expect(second.usage?.cacheWriteInputTokens ?? 0).toBeLessThan(firstCached)
+    }),
+  )
 })

packages/ai/test/provider/bedrock-converse.test.ts

@@ -939,6 +939,7 @@ describe("Bedrock Converse route", () => {
       const prepared = yield* LLMClient.prepare<BedrockConverse.BedrockConverseBody>(
         LLM.request({
           model,
+          cache: "none",
           messages: [
             Message.assistant([ToolCallPart.make({ id: "call_1", name: "read", input: { path: "report.pdf" } })]),
             Message.tool({