chore(llm): make cache: 'auto' the default (#26798)

packages/llm/test/cache-policy.test.ts

@@ -33,7 +33,7 @@ const geminiModel = Gemini.model({
 })
 
 describe("applyCachePolicy", () => {
-  it.effect("undefined cache leaves the request untouched (opt-in default)", () =>
+  it.effect("undefined cache resolves to 'auto' (the recommended default)", () =>
     Effect.gen(function* () {
       const prepared = yield* LLMClient.prepare(
         LLM.request({
@@ -43,8 +43,11 @@ describe("applyCachePolicy", () => {
         }),
       )
 
+      // No explicit cache field → auto policy fires → last system part + latest
+      // user message both get cache_control markers.
       expect(prepared.body).toMatchObject({
-        system: [{ type: "text", text: "You are concise.", cache_control: undefined }],
+        system: [{ type: "text", text: "You are concise.", cache_control: { type: "ephemeral" } }],
+        messages: [{ role: "user", content: [{ type: "text", text: "hi", cache_control: { type: "ephemeral" } }] }],
       })
     }),
   )
@@ -252,6 +255,7 @@ describe("applyCachePolicy", () => {
     const request = LLM.request({
       model: anthropicModel,
       prompt: "hi",
+      cache: "none",
     })
     expect(applyCachePolicy(request)).toBe(request)
   })

packages/llm/test/provider/anthropic-messages-cache.recorded.test.ts

@@ -20,6 +20,9 @@ const cacheRequest = LLM.request({
   model,
   system: [{ type: "text", text: LARGE_CACHEABLE_SYSTEM, cache: new CacheHint({ type: "ephemeral" }) }],
   prompt: "Say hi.",
+  // Manual hint on the system part is the only marker we want here — skip the
+  // auto-policy's latest-user-message breakpoint so the cassette body matches.
+  cache: "none",
   generation: { maxTokens: 16, temperature: 0 },
 })
 

packages/llm/test/provider/anthropic-messages.test.ts

@@ -18,6 +18,9 @@ const request = LLM.request({
   model,
   system: { type: "text", text: "You are concise.", cache: new CacheHint({ type: "ephemeral" }) },
   prompt: "Say hello.",
+  // This fixture predates the `cache: "auto"` default; pin the policy off so
+  // existing wire-shape assertions only see the manual hint on the system part.
+  cache: "none",
   generation: { maxTokens: 20, temperature: 0 },
 })
 
@@ -48,6 +51,7 @@ describe("Anthropic Messages route", () => {
             LLM.assistant([LLM.toolCall({ id: "call_1", name: "lookup", input: { query: "weather" } })]),
             LLM.toolMessage({ id: "call_1", name: "lookup", result: { forecast: "sunny" } }),
           ],
+          cache: "none",
         }),
       )
 

packages/llm/test/provider/bedrock-converse-cache.recorded.test.ts

@@ -27,6 +27,9 @@ const cacheRequest = LLM.request({
   model,
   system: [{ type: "text", text: LARGE_CACHEABLE_SYSTEM, cache: new CacheHint({ type: "ephemeral" }) }],
   prompt: "Say hi.",
+  // Manual hint on the system part is the only marker we want here — skip the
+  // auto-policy's latest-user-message breakpoint so the cassette body matches.
+  cache: "none",
   generation: { maxTokens: 16, temperature: 0 },
 })
 

packages/llm/test/provider/bedrock-converse.test.ts

@@ -63,6 +63,9 @@ const baseRequest = LLM.request({
   model,
   system: "You are concise.",
   prompt: "Say hello.",
+  // Wire-shape assertions in this file predate the `cache: "auto"` default;
+  // pin the policy off so they only exercise the lowering path itself.
+  cache: "none",
   generation: { maxTokens: 64, temperature: 0 },
 })
 
@@ -125,6 +128,7 @@ describe("Bedrock Converse route", () => {
             LLM.assistant([LLM.toolCall({ id: "tool_1", name: "lookup", input: { query: "weather" } })]),
             LLM.toolMessage({ id: "tool_1", name: "lookup", result: { forecast: "sunny" } }),
           ],
+          cache: "none",
         }),
       )
 
@@ -339,6 +343,7 @@ describe("Bedrock Converse route", () => {
               { type: "media", mediaType: "image/webp", data: "DDDD" },
             ]),
           ],
+          cache: "none",
         }),
       )
 
@@ -470,6 +475,7 @@ describe("Bedrock Converse route", () => {
             LLM.assistant([LLM.toolCall({ id: "call_1", name: "lookup", input: {} })]),
             LLM.toolMessage({ id: "call_1", name: "lookup", result: { temp: 72 }, cache }),
           ],
+          cache: "none",
         }),
       )
 
@@ -555,6 +561,7 @@ describe("Bedrock Converse recorded", () => {
           model: recordedModel(),
           system: "Reply with the single word 'Hello'.",
           prompt: "Say hello.",
+          cache: "none",
           generation: { maxTokens: 16, temperature: 0 },
         }),
       )
@@ -577,6 +584,7 @@ describe("Bedrock Converse recorded", () => {
           prompt: "Call get_weather with city exactly Paris.",
           tools: [weatherTool],
           toolChoice: LLM.toolChoice(weatherTool),
+          cache: "none",
           generation: { maxTokens: 80, temperature: 0 },
         }),
       )

packages/llm/test/recorded-scenarios.ts

@@ -51,6 +51,7 @@ export const textRequest = (input: {
     model: input.model,
     system: "You are concise.",
     prompt: input.prompt ?? "Reply with exactly: Hello!",
+    cache: "none",
     generation:
       input.temperature === false
         ? { maxTokens: input.maxTokens ?? 20 }
@@ -70,6 +71,7 @@ export const weatherToolRequest = (input: {
     prompt: "Call get_weather with city exactly Paris.",
     tools: [weatherTool],
     toolChoice: LLM.toolChoice(weatherTool),
+    cache: "none",
     generation:
       input.temperature === false
         ? { maxTokens: input.maxTokens ?? 80 }
@@ -88,6 +90,7 @@ export const weatherToolLoopRequest = (input: {
     model: input.model,
     system: input.system ?? "Use the get_weather tool, then answer in one short sentence.",
     prompt: "What is the weather in Paris?",
+    cache: "none",
     generation:
       input.temperature === false
         ? { maxTokens: input.maxTokens ?? 80 }