lmin/zmo-cli
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity

fix(llm): split OpenAI reasoning summary blocks (#29000)

Browse Source
This commit is contained in:
Aiden Cline
2026-05-23 20:06:45 -05:00
committed by GitHub
parent 0b3a1c2fdf
commit eb84f461b8
22 changed files with 717 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
packages/llm/test/continuation-scenarios.ts
View File

@@ -97,7 +97,7 @@ export function continuationRequest(input: {
tools: features.has("tool-call") ? [continuationTool] : [],
cache: "none",
providerOptions: features.has("encrypted-reasoning")
? { openai: { store: false, includeEncryptedReasoning: true, reasoningSummary: "auto" } }
? { openai: { store: false, include: ["reasoning.encrypted_content"], reasoningSummary: "auto" } }
: undefined,
generation: { maxTokens: 80, temperature: 0 },
})

6
packages/llm/test/fixtures/recordings/openai-responses/openai-responses-gpt-5-5-image-tool-result.json vendored
View File

File diff suppressed because one or more lines are too long

8
packages/llm/test/fixtures/recordings/openai-responses/openai-responses-gpt-5-5-reasoning-continuation.json vendored
View File

File diff suppressed because one or more lines are too long

14
packages/llm/test/fixtures/recordings/openai-responses/openai-responses-gpt-5-5-reasoning.json vendored
View File

File diff suppressed because one or more lines are too long

254
packages/llm/test/provider/openai-responses.test.ts
View File

@@ -393,7 +393,7 @@ describe("OpenAI Responses route", () => {
promptCacheKey: "session_123",
reasoningEffort: "high",
reasoningSummary: "auto",
includeEncryptedReasoning: true,
include: ["reasoning.encrypted_content"],
},
},
}),
@@ -407,6 +407,108 @@ describe("OpenAI Responses route", () => {
}),
)
it.effect("accepts the full ResponseIncludable union", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
LLM.request({
model,
prompt: "hi",
providerOptions: {
openai: {
include: ["reasoning.encrypted_content", "code_interpreter_call.outputs", "web_search_call.results"],
},
},
}),
)
expect(prepared.body.include).toEqual([
"reasoning.encrypted_content",
"code_interpreter_call.outputs",
"web_search_call.results",
])
}),
)
it.effect("filters unknown includable values out of the include array", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
LLM.request({
model,
prompt: "hi",
// The user passed one invalid entry alongside a valid one. Keep the
// valid one so the request still succeeds rather than failing on a
// typo from upstream config.
providerOptions: { openai: { include: ["reasoning.encrypted_content", "bogus.thing"] } },
}),
)
expect(prepared.body.include).toEqual(["reasoning.encrypted_content"])
}),
)
it.effect("treats an explicit empty include as no include at all", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
LLM.request({ model, prompt: "hi", providerOptions: { openai: { include: [] } } }),
)
expect(prepared.body.include).toBeUndefined()
}),
)
it.effect("treats an all-invalid include as no include at all", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
LLM.request({ model, prompt: "hi", providerOptions: { openai: { include: ["bogus.thing"] } } }),
)
expect(prepared.body.include).toBeUndefined()
}),
)
it.effect("omits include when no include is set", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
LLM.request({ model, prompt: "hi", providerOptions: { openai: { store: false } } }),
)
expect(prepared.body.include).toBeUndefined()
}),
)
it.effect("requests encrypted reasoning by default for GPT-5 reasoning models", () =>
Effect.gen(function* () {
// The native OpenAI facade configures GPT-5 stateless (store: false) with
// reasoningSummary: "auto" by default. Without `include`, a follow-up
// turn cannot replay reasoning state, so the facade also opts into
// `reasoning.encrypted_content` automatically.
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
LLM.request({
model: OpenAI.configure({ baseURL: "https://api.openai.test/v1/", apiKey: "test" }).responses("gpt-5.2"),
prompt: "hi",
}),
)
expect(prepared.body.store).toBe(false)
expect(prepared.body.include).toEqual(["reasoning.encrypted_content"])
expect(prepared.body.reasoning).toEqual({ effort: "medium", summary: "auto" })
}),
)
it.effect("lets callers opt out of the GPT-5 default include", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
LLM.request({
model: OpenAI.configure({ baseURL: "https://api.openai.test/v1/", apiKey: "test" }).responses("gpt-5.2"),
prompt: "hi",
providerOptions: { openai: { include: [] } },
}),
)
expect(prepared.body.include).toBeUndefined()
}),
)
it.effect("request OpenAI provider options override route defaults", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
@@ -547,6 +649,94 @@ describe("OpenAI Responses route", () => {
}),
)
it.effect("streams each reasoning summary part as a separate block", () =>
Effect.gen(function* () {
const response = yield* LLMClient.generate(
LLM.updateRequest(request, { providerOptions: { openai: { store: false } } }),
).pipe(
Effect.provide(
fixedResponse(
sseEvents(
{
type: "response.output_item.added",
item: { type: "reasoning", id: "rs_1", encrypted_content: null },
},
{ type: "response.reasoning_summary_part.added", item_id: "rs_1", summary_index: 0 },
{ type: "response.reasoning_summary_text.delta", item_id: "rs_1", summary_index: 0, delta: "First" },
{ type: "response.reasoning_summary_part.done", item_id: "rs_1", summary_index: 0 },
{ type: "response.reasoning_summary_part.added", item_id: "rs_1", summary_index: 1 },
{ type: "response.reasoning_summary_text.delta", item_id: "rs_1", summary_index: 1, delta: "Second" },
{ type: "response.reasoning_summary_part.done", item_id: "rs_1", summary_index: 1 },
{
type: "response.output_item.done",
item: { type: "reasoning", id: "rs_1", encrypted_content: "encrypted-state" },
},
{ type: "response.completed", response: { id: "resp_1" } },
),
),
),
)
expect(response.reasoning).toBe("FirstSecond")
expect(response.events).toMatchObject([
{ type: "step-start", index: 0 },
{
type: "reasoning-start",
id: "rs_1:0",
providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: null } },
},
{ type: "reasoning-delta", id: "rs_1:0", text: "First" },
{ type: "reasoning-end", id: "rs_1:0", providerMetadata: { openai: { itemId: "rs_1" } } },
{
type: "reasoning-start",
id: "rs_1:1",
providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: null } },
},
{ type: "reasoning-delta", id: "rs_1:1", text: "Second" },
{
type: "reasoning-end",
id: "rs_1:1",
providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" } },
},
{ type: "step-finish", index: 0, reason: "stop" },
{ type: "finish", reason: "stop" },
])
}),
)
it.effect("closes reasoning summary parts when storage is not disabled", () =>
Effect.gen(function* () {
const response = yield* LLMClient.generate(request).pipe(
Effect.provide(
fixedResponse(
sseEvents(
{
type: "response.output_item.added",
item: { type: "reasoning", id: "rs_1", encrypted_content: null },
},
{ type: "response.reasoning_summary_part.added", item_id: "rs_1", summary_index: 0 },
{ type: "response.reasoning_summary_text.delta", item_id: "rs_1", summary_index: 0, delta: "First" },
{ type: "response.reasoning_summary_part.done", item_id: "rs_1", summary_index: 0 },
{ type: "response.reasoning_summary_part.added", item_id: "rs_1", summary_index: 1 },
{ type: "response.reasoning_summary_text.delta", item_id: "rs_1", summary_index: 1, delta: "Second" },
{ type: "response.reasoning_summary_part.done", item_id: "rs_1", summary_index: 1 },
{
type: "response.output_item.done",
item: { type: "reasoning", id: "rs_1", encrypted_content: null },
},
{ type: "response.completed", response: { id: "resp_1" } },
),
),
),
)
expect(response.events.filter((event) => event.type === "reasoning-end")).toEqual([
{ type: "reasoning-end", id: "rs_1:0", providerMetadata: { openai: { itemId: "rs_1" } } },
{ type: "reasoning-end", id: "rs_1:1", providerMetadata: { openai: { itemId: "rs_1" } } },
])
}),
)
it.effect("continues a stateless reasoning conversation", () =>
Effect.gen(function* () {
const response = yield* LLMClient.generate(
@@ -570,6 +760,7 @@ describe("OpenAI Responses route", () => {
]),
Message.user("Summarize it."),
],
providerOptions: { openai: { store: false } },
}),
).pipe(
Effect.provide(
@@ -627,6 +818,7 @@ describe("OpenAI Responses route", () => {
{ type: "text", text: "After." },
]),
],
providerOptions: { openai: { store: false } },
}),
)
@@ -643,6 +835,66 @@ describe("OpenAI Responses route", () => {
}),
)
it.effect("references stored reasoning items by id", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
LLM.request({
model,
messages: [
Message.assistant([
{
type: "reasoning",
text: "Checked the previous diff.",
providerMetadata: { openai: { itemId: "rs_1" } },
},
]),
],
providerOptions: { openai: { store: true } },
}),
)
expect(prepared.body.input).toEqual([{ type: "item_reference", id: "rs_1" }])
}),
)
it.effect("joins streamed summary blocks into one continuation reasoning item", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
LLM.request({
id: "req_multi_summary_continuation",
model,
messages: [
Message.assistant([
{
type: "reasoning",
text: "First",
providerMetadata: { openai: { itemId: "rs_1" } },
},
{
type: "reasoning",
text: "Second",
providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" } },
},
]),
],
providerOptions: { openai: { store: false } },
}),
)
expect(prepared.body.input).toEqual([
{
type: "reasoning",
id: "rs_1",
encrypted_content: "encrypted-state",
summary: [
{ type: "summary_text", text: "First" },
{ type: "summary_text", text: "Second" },
],
},
])
}),
)
it.effect("skips non-persisted reasoning ids without encrypted state", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare(

2
packages/llm/test/recorded-scenarios.ts
View File

@@ -158,7 +158,7 @@ const normalizeImageText = (value: string) =>
const encryptedReasoningOptions = {
openai: {
store: false,
includeEncryptedReasoning: true,
include: ["reasoning.encrypted_content"],
reasoningEffort: "low",
reasoningSummary: "auto",
},

66
packages/llm/test/tool-runtime.test.ts
View File

@@ -4,6 +4,7 @@ import { GenerationOptions, LLM, LLMEvent, LLMRequest, LLMResponse, ToolChoice }
import { Auth, LLMClient } from "../src/route"
import * as AnthropicMessages from "../src/protocols/anthropic-messages"
import * as OpenAIChat from "../src/protocols/openai-chat"
import * as OpenAIResponses from "../src/protocols/openai-responses"
import { tool, ToolFailure, type ToolExecuteContext } from "../src/tool"
import { ToolRuntime } from "../src/tool-runtime"
import { it } from "./lib/effect"
@@ -309,6 +310,71 @@ describe("LLMClient tools", () => {
}),
)
it.effect("replays encrypted OpenAI reasoning items with tool outputs", () =>
Effect.gen(function* () {
const bodies: unknown[] = []
const layer = dynamicResponse((input) =>
Effect.sync(() => {
bodies.push(decodeJson(input.text))
return input.respond(
bodies.length === 1
? sseEvents(
{ type: "response.output_item.added", item: { type: "reasoning", id: "rs_1", encrypted_content: null } },
{ type: "response.reasoning_summary_part.added", item_id: "rs_1", summary_index: 0 },
{ type: "response.reasoning_summary_part.done", item_id: "rs_1", summary_index: 0 },
{
type: "response.output_item.done",
item: { type: "reasoning", id: "rs_1", encrypted_content: "encrypted-state" },
},
{
type: "response.output_item.added",
item: { type: "function_call", id: "item_1", call_id: "call_1", name: "get_weather", arguments: "" },
},
{ type: "response.function_call_arguments.delta", item_id: "item_1", delta: '{"city":"Paris"}' },
{
type: "response.output_item.done",
item: {
type: "function_call",
id: "item_1",
call_id: "call_1",
name: "get_weather",
arguments: '{"city":"Paris"}',
},
},
{ type: "response.completed", response: {} },
)
: sseEvents(
{ type: "response.output_text.delta", item_id: "msg_1", delta: "Done." },
{ type: "response.completed", response: {} },
),
{ headers: { "content-type": "text/event-stream" } },
)
}),
)
yield* TestToolRuntime.runTools({
request: LLM.request({
model: OpenAIResponses.route
.with({ endpoint: { baseURL: "https://api.openai.test/v1/" }, auth: Auth.bearer("test") })
.model({ id: "gpt-5.5" }),
prompt: "Use the tool.",
providerOptions: { openai: { store: false, include: ["reasoning.encrypted_content"] } },
}),
tools: { get_weather },
}).pipe(Stream.runCollect, Effect.provide(layer))
expect(bodies[1]).toMatchObject({
include: ["reasoning.encrypted_content"],
input: [
{ role: "user" },
{ type: "reasoning", id: "rs_1", summary: [], encrypted_content: "encrypted-state" },
{ type: "function_call", call_id: "call_1", name: "get_weather" },
{ type: "function_call_output", call_id: "call_1" },
],
})
}),
)
it.effect("emits tool-error for unknown tools so the model can self-correct", () =>
Effect.gen(function* () {
const layer = scriptedResponses([