feat(llm): cache hint TTL, breakpoint cap, and tool placement (#26779)

2026-05-10 21:17:38 -04:00
parent fed716ada5
commit 77e6c0d329
12 changed files with 555 additions and 39 deletions
@@ -0,0 +1,48 @@
+import { Redactor } from "@opencode-ai/http-recorder"
+import { describe, expect } from "bun:test"
+import { Effect } from "effect"
+import { CacheHint, LLM } from "../../src"
+import { LLMClient } from "../../src/route"
+import * as AnthropicMessages from "../../src/protocols/anthropic-messages"
+import { LARGE_CACHEABLE_SYSTEM } from "../recorded-scenarios"
+import { recordedTests } from "../recorded-test"
+
+const model = AnthropicMessages.model({
+  id: "claude-haiku-4-5-20251001",
+  apiKey: process.env.ANTHROPIC_API_KEY ?? "fixture",
+})
+
+// Two identical generations in a row. The first call writes the prefix into
+// Anthropic's cache; the second should report a cache read against the same
+// prefix. Cassette captures both interactions in order.
+const cacheRequest = LLM.request({
+  id: "recorded_anthropic_cache",
+  model,
+  system: [{ type: "text", text: LARGE_CACHEABLE_SYSTEM, cache: new CacheHint({ type: "ephemeral" }) }],
+  prompt: "Say hi.",
+  generation: { maxTokens: 16, temperature: 0 },
+})
+
+const recorded = recordedTests({
+  prefix: "anthropic-messages-cache",
+  provider: "anthropic",
+  protocol: "anthropic-messages",
+  requires: ["ANTHROPIC_API_KEY"],
+  options: { redactor: Redactor.defaults({ requestHeaders: { allow: ["content-type", "anthropic-version"] } }) },
+})
+
+describe("Anthropic Messages cache recorded", () => {
+  recorded.effect.with("writes then reads cache_control on identical second call", { tags: ["cache"] }, () =>
+    Effect.gen(function* () {
+      const first = yield* LLMClient.generate(cacheRequest)
+      // The first call may write the cache (cacheWriteInputTokens > 0) or it
+      // may be a fresh miss (both fields 0) depending on whether the prefix is
+      // already warm on Anthropic's side. The assertion that matters is that
+      // the SECOND call reports a non-zero cache read.
+      expect(first.usage?.cacheReadInputTokens ?? 0).toBeGreaterThanOrEqual(0)
+
+      const second = yield* LLMClient.generate(cacheRequest)
+      expect(second.usage?.cacheReadInputTokens ?? 0).toBeGreaterThan(0)
+    }),
+  )
+})
@@ -374,4 +374,134 @@ describe("Anthropic Messages route", () => {
      expect(error.message).toContain("Anthropic Messages user messages only support text content for now")
    }),
  )
+
+  it.effect("maps ttlSeconds >= 3600 to cache_control ttl: '1h'", () =>
+    Effect.gen(function* () {
+      const prepared = yield* LLMClient.prepare(
+        LLM.request({
+          model,
+          system: { type: "text", text: "system", cache: new CacheHint({ type: "ephemeral", ttlSeconds: 3600 }) },
+          prompt: "hi",
+        }),
+      )
+
+      expect(prepared.body).toMatchObject({
+        system: [{ type: "text", text: "system", cache_control: { type: "ephemeral", ttl: "1h" } }],
+      })
+    }),
+  )
+
+  it.effect("emits cache_control on tool definitions and tool-result blocks", () =>
+    Effect.gen(function* () {
+      const prepared = yield* LLMClient.prepare(
+        LLM.request({
+          model,
+          tools: [
+            {
+              name: "lookup",
+              description: "lookup tool",
+              inputSchema: { type: "object", properties: {} },
+              cache: new CacheHint({ type: "ephemeral" }),
+            },
+          ],
+          messages: [
+            LLM.user("What's the weather?"),
+            LLM.assistant([LLM.toolCall({ id: "call_1", name: "lookup", input: {} })]),
+            LLM.toolMessage({
+              id: "call_1",
+              name: "lookup",
+              result: { temp: 72 },
+              cache: new CacheHint({ type: "ephemeral" }),
+            }),
+          ],
+        }),
+      )
+
+      expect(prepared.body).toMatchObject({
+        tools: [{ name: "lookup", cache_control: { type: "ephemeral" } }],
+        messages: [
+          { role: "user", content: [{ type: "text", text: "What's the weather?" }] },
+          { role: "assistant", content: [{ type: "tool_use", id: "call_1", name: "lookup" }] },
+          {
+            role: "user",
+            content: [{ type: "tool_result", tool_use_id: "call_1", cache_control: { type: "ephemeral" } }],
+          },
+        ],
+      })
+    }),
+  )
+
+  it.effect("drops cache_control breakpoints past the 4-per-request cap", () =>
+    Effect.gen(function* () {
+      const hint = new CacheHint({ type: "ephemeral" })
+      const prepared = yield* LLMClient.prepare(
+        LLM.request({
+          model,
+          system: [
+            { type: "text", text: "a", cache: hint },
+            { type: "text", text: "b", cache: hint },
+            { type: "text", text: "c", cache: hint },
+            { type: "text", text: "d", cache: hint },
+            { type: "text", text: "e", cache: hint },
+            { type: "text", text: "f", cache: hint },
+          ],
+          prompt: "hi",
+        }),
+      )
+
+      const system = (prepared.body as { system: Array<{ cache_control?: unknown }> }).system
+      const marked = system.filter((part) => part.cache_control !== undefined)
+      expect(marked).toHaveLength(4)
+      expect(system[4]?.cache_control).toBeUndefined()
+      expect(system[5]?.cache_control).toBeUndefined()
+    }),
+  )
+
+  it.effect("spends breakpoint budget on tools before system before messages", () =>
+    Effect.gen(function* () {
+      const hint = new CacheHint({ type: "ephemeral" })
+      const prepared = yield* LLMClient.prepare(
+        LLM.request({
+          model,
+          tools: [
+            {
+              name: "t1",
+              description: "t1",
+              inputSchema: { type: "object", properties: {} },
+              cache: hint,
+            },
+            {
+              name: "t2",
+              description: "t2",
+              inputSchema: { type: "object", properties: {} },
+              cache: hint,
+            },
+            {
+              name: "t3",
+              description: "t3",
+              inputSchema: { type: "object", properties: {} },
+              cache: hint,
+            },
+            {
+              name: "t4",
+              description: "t4",
+              inputSchema: { type: "object", properties: {} },
+              cache: hint,
+            },
+          ],
+          system: [{ type: "text", text: "system-tail", cache: hint }],
+          messages: [LLM.user([{ type: "text", text: "message-tail", cache: hint }])],
+        }),
+      )
+
+      const body = prepared.body as {
+        tools: Array<{ cache_control?: unknown }>
+        system: Array<{ cache_control?: unknown }>
+        messages: Array<{ content: Array<{ cache_control?: unknown }> }>
+      }
+      expect(body.tools.every((t) => t.cache_control !== undefined)).toBe(true)
+      expect(body.system[0]?.cache_control).toBeUndefined()
+      expect(body.messages[0]?.content[0]?.cache_control).toBeUndefined()
+    }),
+  )
 })
@@ -0,0 +1,50 @@
+import { describe, expect } from "bun:test"
+import { Effect } from "effect"
+import { CacheHint, LLM } from "../../src"
+import { LLMClient } from "../../src/route"
+import * as BedrockConverse from "../../src/protocols/bedrock-converse"
+import { LARGE_CACHEABLE_SYSTEM } from "../recorded-scenarios"
+import { recordedTests } from "../recorded-test"
+
+const RECORDING_REGION = process.env.BEDROCK_RECORDING_REGION ?? "us-east-1"
+
+// Use a Claude model on Bedrock — Nova has automatic prefix caching that
+// doesn't reliably surface `cacheRead`/`cacheWrite` in usage, so the second
+// call wouldn't deterministically prove cache mapping works. Override with
+// BEDROCK_CACHE_MODEL_ID if your account has access elsewhere.
+const model = BedrockConverse.model({
+  id: process.env.BEDROCK_CACHE_MODEL_ID ?? "us.anthropic.claude-haiku-4-5-20251001-v1:0",
+  credentials: {
+    region: RECORDING_REGION,
+    accessKeyId: process.env.AWS_ACCESS_KEY_ID ?? "fixture",
+    secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY ?? "fixture",
+    sessionToken: process.env.AWS_SESSION_TOKEN,
+  },
+})
+
+const cacheRequest = LLM.request({
+  id: "recorded_bedrock_cache",
+  model,
+  system: [{ type: "text", text: LARGE_CACHEABLE_SYSTEM, cache: new CacheHint({ type: "ephemeral" }) }],
+  prompt: "Say hi.",
+  generation: { maxTokens: 16, temperature: 0 },
+})
+
+const recorded = recordedTests({
+  prefix: "bedrock-converse-cache",
+  provider: "amazon-bedrock",
+  protocol: "bedrock-converse",
+  requires: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"],
+})
+
+describe("Bedrock Converse cache recorded", () => {
+  recorded.effect.with("writes then reads cachePoint on identical second call", { tags: ["cache"] }, () =>
+    Effect.gen(function* () {
+      const first = yield* LLMClient.generate(cacheRequest)
+      expect(first.usage?.cacheReadInputTokens ?? 0).toBeGreaterThanOrEqual(0)
+
+      const second = yield* LLMClient.generate(cacheRequest)
+      expect(second.usage?.cacheReadInputTokens ?? 0).toBeGreaterThan(0)
+    }),
+  )
+})
@@ -440,6 +440,79 @@ describe("Bedrock Converse route", () => {
      expect(error.message).toContain("Bedrock Converse does not support media type application/x-tar")
    }),
  )
+
+  it.effect("maps ttlSeconds >= 3600 to cachePoint ttl: '1h'", () =>
+    Effect.gen(function* () {
+      const cache = new CacheHint({ type: "ephemeral", ttlSeconds: 3600 })
+      const prepared = yield* LLMClient.prepare(
+        LLM.request({
+          model,
+          system: [{ type: "text", text: "system", cache }],
+          prompt: "hi",
+        }),
+      )
+
+      expect(prepared.body).toMatchObject({
+        system: [{ text: "system" }, { cachePoint: { type: "default", ttl: "1h" } }],
+      })
+    }),
+  )
+
+  it.effect("appends cachePoint after marked tool definitions and tool-result blocks", () =>
+    Effect.gen(function* () {
+      const cache = new CacheHint({ type: "ephemeral" })
+      const prepared = yield* LLMClient.prepare(
+        LLM.request({
+          model,
+          tools: [
+            { name: "lookup", description: "lookup", inputSchema: { type: "object", properties: {} }, cache },
+          ],
+          messages: [
+            LLM.user("What's the weather?"),
+            LLM.assistant([LLM.toolCall({ id: "call_1", name: "lookup", input: {} })]),
+            LLM.toolMessage({ id: "call_1", name: "lookup", result: { temp: 72 }, cache }),
+          ],
+        }),
+      )
+
+      expect(prepared.body).toMatchObject({
+        toolConfig: {
+          tools: [{ toolSpec: { name: "lookup" } }, { cachePoint: { type: "default" } }],
+        },
+        messages: [
+          { role: "user", content: [{ text: "What's the weather?" }] },
+          { role: "assistant", content: [{ toolUse: { toolUseId: "call_1" } }] },
+          {
+            role: "user",
+            content: [{ toolResult: { toolUseId: "call_1" } }, { cachePoint: { type: "default" } }],
+          },
+        ],
+      })
+    }),
+  )
+
+  it.effect("drops cachePoint markers past the 4-per-request cap", () =>
+    Effect.gen(function* () {
+      const cache = new CacheHint({ type: "ephemeral" })
+      const prepared = yield* LLMClient.prepare(
+        LLM.request({
+          model,
+          system: [
+            { type: "text", text: "a", cache },
+            { type: "text", text: "b", cache },
+            { type: "text", text: "c", cache },
+            { type: "text", text: "d", cache },
+            { type: "text", text: "e", cache },
+            { type: "text", text: "f", cache },
+          ],
+          prompt: "hi",
+        }),
+      )
+
+      const system = (prepared.body as { system: Array<{ cachePoint?: unknown }> }).system
+      expect(system.filter((part) => "cachePoint" in part)).toHaveLength(4)
+    }),
+  )
 })

 // Live recorded integration tests. Run with `RECORD=true AWS_ACCESS_KEY_ID=...
@@ -0,0 +1,47 @@
+import { describe, expect } from "bun:test"
+import { Effect } from "effect"
+import { LLM } from "../../src"
+import { LLMClient } from "../../src/route"
+import * as Gemini from "../../src/protocols/gemini"
+import { LARGE_CACHEABLE_SYSTEM } from "../recorded-scenarios"
+import { recordedTests } from "../recorded-test"
+
+const model = Gemini.model({
+  id: "gemini-2.5-flash",
+  apiKey: process.env.GEMINI_API_KEY ?? "fixture",
+})
+
+// Gemini does implicit prefix caching on 2.5+ models above ~1024 tokens. The
+// `CacheHint` is currently a no-op for Gemini (the explicit `CachedContent`
+// API is out-of-band and intentionally not wired up). This test exists to
+// pin the usage-parsing path: `cachedContentTokenCount` should surface as
+// `cacheReadInputTokens` on the second identical call.
+const cacheRequest = LLM.request({
+  id: "recorded_gemini_cache",
+  model,
+  system: LARGE_CACHEABLE_SYSTEM,
+  prompt: "Say hi.",
+  generation: { maxTokens: 16, temperature: 0 },
+})
+
+const recorded = recordedTests({
+  prefix: "gemini-cache",
+  provider: "google",
+  protocol: "gemini",
+  requires: ["GEMINI_API_KEY"],
+})
+
+describe("Gemini cache recorded", () => {
+  recorded.effect.with("reports cachedContentTokenCount on identical second call", { tags: ["cache"] }, () =>
+    Effect.gen(function* () {
+      const first = yield* LLMClient.generate(cacheRequest)
+      expect(first.usage?.cacheReadInputTokens ?? 0).toBeGreaterThanOrEqual(0)
+
+      const second = yield* LLMClient.generate(cacheRequest)
+      // Implicit caching is best-effort on Gemini's side; we assert the field
+      // is at least populated and non-negative. When re-recording, verify the
+      // cassette shows > 0 in the second response's usage.
+      expect(second.usage?.cacheReadInputTokens ?? 0).toBeGreaterThanOrEqual(0)
+    }),
+  )
+})
@@ -0,0 +1,44 @@
+import { describe, expect } from "bun:test"
+import { Effect } from "effect"
+import { LLM } from "../../src"
+import { LLMClient } from "../../src/route"
+import * as OpenAIResponses from "../../src/protocols/openai-responses"
+import { LARGE_CACHEABLE_SYSTEM } from "../recorded-scenarios"
+import { recordedTests } from "../recorded-test"
+
+const model = OpenAIResponses.model({
+  id: "gpt-4.1-mini",
+  apiKey: process.env.OPENAI_API_KEY ?? "fixture",
+})
+
+// OpenAI caches prefixes automatically once they cross the 1024-token threshold;
+// `CacheHint` is a no-op for the wire body. The stable signal is the
+// `prompt_cache_key` routing hint, which keeps repeated calls on the same shard
+// so cache hits are observable.
+const cacheRequest = LLM.request({
+  id: "recorded_openai_responses_cache",
+  model,
+  system: LARGE_CACHEABLE_SYSTEM,
+  prompt: "Say hi.",
+  generation: { maxTokens: 16, temperature: 0 },
+  providerOptions: { openai: { promptCacheKey: "recorded-cache-test" } },
+})
+
+const recorded = recordedTests({
+  prefix: "openai-responses-cache",
+  provider: "openai",
+  protocol: "openai-responses",
+  requires: ["OPENAI_API_KEY"],
+})
+
+describe("OpenAI Responses cache recorded", () => {
+  recorded.effect.with("reports cached_tokens on identical second call", { tags: ["cache"] }, () =>
+    Effect.gen(function* () {
+      const first = yield* LLMClient.generate(cacheRequest)
+      expect(first.usage?.cacheReadInputTokens ?? 0).toBeGreaterThanOrEqual(0)
+
+      const second = yield* LLMClient.generate(cacheRequest)
+      expect(second.usage?.cacheReadInputTokens ?? 0).toBeGreaterThan(0)
+    }),
+  )
+})
@@ -6,6 +6,19 @@ import { tool } from "../src/tool"

 export const weatherToolName = "get_weather"

+// A deterministic system prompt long enough to clear every supported provider's
+// minimum cacheable-prefix threshold (Anthropic Haiku 3.5: 2048 tokens; Anthropic
+// Opus/Haiku 4.5: 4096 tokens; OpenAI/Gemini/Bedrock: lower). Built by repeating
+// a fixed sentence — the cassette replays bit-for-bit, so the exact text matters
+// only when re-recording with `RECORD=true`.
+export const LARGE_CACHEABLE_SYSTEM = (() => {
+  const sentence =
+    "You are a concise, factual assistant. Answer precisely and avoid filler. Cite numbers when known. "
+  // ~100 chars per sentence × 250 repeats ≈ 25,000 chars ≈ 5k+ tokens, safely
+  // above every provider's threshold.
+  return sentence.repeat(250)
+})()
+
 export const weatherTool = LLM.toolDefinition({
  name: weatherToolName,
  description: "Get current weather for a city.",