lmin/zmo-cli
1
0
Fork 0
Code Issues Pull Requests Actions 82 Packages Projects Releases Wiki Activity

fix(llm): split OpenAI reasoning summary blocks (#29000)

Browse Source
This commit is contained in:
Aiden Cline
2026-05-23 20:06:45 -05:00
committed by GitHub
parent 0b3a1c2fdf
commit eb84f461b8
22 changed files with 717 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/llm/test/provider/openai-responses.test.ts
+253 -1
View File
@@ -393,7 +393,7 @@ describe("OpenAI Responses route", () => {
promptCacheKey: "session_123",
reasoningEffort: "high",
reasoningSummary: "auto",
includeEncryptedReasoning: true,
include: ["reasoning.encrypted_content"],
},
},
}),
@@ -407,6 +407,108 @@ describe("OpenAI Responses route", () => {
}),
)
it.effect("accepts the full ResponseIncludable union", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
LLM.request({
model,
prompt: "hi",
providerOptions: {
openai: {
include: ["reasoning.encrypted_content", "code_interpreter_call.outputs", "web_search_call.results"],
},
},
}),
)
expect(prepared.body.include).toEqual([
"reasoning.encrypted_content",
"code_interpreter_call.outputs",
"web_search_call.results",
])
}),
)
it.effect("filters unknown includable values out of the include array", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
LLM.request({
model,
prompt: "hi",
// The user passed one invalid entry alongside a valid one. Keep the
// valid one so the request still succeeds rather than failing on a
// typo from upstream config.
providerOptions: { openai: { include: ["reasoning.encrypted_content", "bogus.thing"] } },
}),
)
expect(prepared.body.include).toEqual(["reasoning.encrypted_content"])
}),
)
it.effect("treats an explicit empty include as no include at all", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
LLM.request({ model, prompt: "hi", providerOptions: { openai: { include: [] } } }),
)
expect(prepared.body.include).toBeUndefined()
}),
)
it.effect("treats an all-invalid include as no include at all", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
LLM.request({ model, prompt: "hi", providerOptions: { openai: { include: ["bogus.thing"] } } }),
)
expect(prepared.body.include).toBeUndefined()
}),
)
it.effect("omits include when no include is set", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
LLM.request({ model, prompt: "hi", providerOptions: { openai: { store: false } } }),
)
expect(prepared.body.include).toBeUndefined()
}),
)
it.effect("requests encrypted reasoning by default for GPT-5 reasoning models", () =>
Effect.gen(function* () {
// The native OpenAI facade configures GPT-5 stateless (store: false) with
// reasoningSummary: "auto" by default. Without `include`, a follow-up
// turn cannot replay reasoning state, so the facade also opts into
// `reasoning.encrypted_content` automatically.
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
LLM.request({
model: OpenAI.configure({ baseURL: "https://api.openai.test/v1/", apiKey: "test" }).responses("gpt-5.2"),
prompt: "hi",
}),
)
expect(prepared.body.store).toBe(false)
expect(prepared.body.include).toEqual(["reasoning.encrypted_content"])
expect(prepared.body.reasoning).toEqual({ effort: "medium", summary: "auto" })
}),
)
it.effect("lets callers opt out of the GPT-5 default include", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
LLM.request({
model: OpenAI.configure({ baseURL: "https://api.openai.test/v1/", apiKey: "test" }).responses("gpt-5.2"),
prompt: "hi",
providerOptions: { openai: { include: [] } },
}),
)
expect(prepared.body.include).toBeUndefined()
}),
)
it.effect("request OpenAI provider options override route defaults", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
@@ -547,6 +649,94 @@ describe("OpenAI Responses route", () => {
}),
)
it.effect("streams each reasoning summary part as a separate block", () =>
Effect.gen(function* () {
const response = yield* LLMClient.generate(
LLM.updateRequest(request, { providerOptions: { openai: { store: false } } }),
).pipe(
Effect.provide(
fixedResponse(
sseEvents(
{
type: "response.output_item.added",
item: { type: "reasoning", id: "rs_1", encrypted_content: null },
},
{ type: "response.reasoning_summary_part.added", item_id: "rs_1", summary_index: 0 },
{ type: "response.reasoning_summary_text.delta", item_id: "rs_1", summary_index: 0, delta: "First" },
{ type: "response.reasoning_summary_part.done", item_id: "rs_1", summary_index: 0 },
{ type: "response.reasoning_summary_part.added", item_id: "rs_1", summary_index: 1 },
{ type: "response.reasoning_summary_text.delta", item_id: "rs_1", summary_index: 1, delta: "Second" },
{ type: "response.reasoning_summary_part.done", item_id: "rs_1", summary_index: 1 },
{
type: "response.output_item.done",
item: { type: "reasoning", id: "rs_1", encrypted_content: "encrypted-state" },
},
{ type: "response.completed", response: { id: "resp_1" } },
),
),
),
)
expect(response.reasoning).toBe("FirstSecond")
expect(response.events).toMatchObject([
{ type: "step-start", index: 0 },
{
type: "reasoning-start",
id: "rs_1:0",
providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: null } },
},
{ type: "reasoning-delta", id: "rs_1:0", text: "First" },
{ type: "reasoning-end", id: "rs_1:0", providerMetadata: { openai: { itemId: "rs_1" } } },
{
type: "reasoning-start",
id: "rs_1:1",
providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: null } },
},
{ type: "reasoning-delta", id: "rs_1:1", text: "Second" },
{
type: "reasoning-end",
id: "rs_1:1",
providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" } },
},
{ type: "step-finish", index: 0, reason: "stop" },
{ type: "finish", reason: "stop" },
])
}),
)
it.effect("closes reasoning summary parts when storage is not disabled", () =>
Effect.gen(function* () {
const response = yield* LLMClient.generate(request).pipe(
Effect.provide(
fixedResponse(
sseEvents(
{
type: "response.output_item.added",
item: { type: "reasoning", id: "rs_1", encrypted_content: null },
},
{ type: "response.reasoning_summary_part.added", item_id: "rs_1", summary_index: 0 },
{ type: "response.reasoning_summary_text.delta", item_id: "rs_1", summary_index: 0, delta: "First" },
{ type: "response.reasoning_summary_part.done", item_id: "rs_1", summary_index: 0 },
{ type: "response.reasoning_summary_part.added", item_id: "rs_1", summary_index: 1 },
{ type: "response.reasoning_summary_text.delta", item_id: "rs_1", summary_index: 1, delta: "Second" },
{ type: "response.reasoning_summary_part.done", item_id: "rs_1", summary_index: 1 },
{
type: "response.output_item.done",
item: { type: "reasoning", id: "rs_1", encrypted_content: null },
},
{ type: "response.completed", response: { id: "resp_1" } },
),
),
),
)
expect(response.events.filter((event) => event.type === "reasoning-end")).toEqual([
{ type: "reasoning-end", id: "rs_1:0", providerMetadata: { openai: { itemId: "rs_1" } } },
{ type: "reasoning-end", id: "rs_1:1", providerMetadata: { openai: { itemId: "rs_1" } } },
])
}),
)
it.effect("continues a stateless reasoning conversation", () =>
Effect.gen(function* () {
const response = yield* LLMClient.generate(
@@ -570,6 +760,7 @@ describe("OpenAI Responses route", () => {
]),
Message.user("Summarize it."),
],
providerOptions: { openai: { store: false } },
}),
).pipe(
Effect.provide(
@@ -627,6 +818,7 @@ describe("OpenAI Responses route", () => {
{ type: "text", text: "After." },
]),
],
providerOptions: { openai: { store: false } },
}),
)
@@ -643,6 +835,66 @@ describe("OpenAI Responses route", () => {
}),
)
it.effect("references stored reasoning items by id", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
LLM.request({
model,
messages: [
Message.assistant([
{
type: "reasoning",
text: "Checked the previous diff.",
providerMetadata: { openai: { itemId: "rs_1" } },
},
]),
],
providerOptions: { openai: { store: true } },
}),
)
expect(prepared.body.input).toEqual([{ type: "item_reference", id: "rs_1" }])
}),
)
it.effect("joins streamed summary blocks into one continuation reasoning item", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
LLM.request({
id: "req_multi_summary_continuation",
model,
messages: [
Message.assistant([
{
type: "reasoning",
text: "First",
providerMetadata: { openai: { itemId: "rs_1" } },
},
{
type: "reasoning",
text: "Second",
providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" } },
},
]),
],
providerOptions: { openai: { store: false } },
}),
)
expect(prepared.body.input).toEqual([
{
type: "reasoning",
id: "rs_1",
encrypted_content: "encrypted-state",
summary: [
{ type: "summary_text", text: "First" },
{ type: "summary_text", text: "Second" },
],
},
])
}),
)
it.effect("skips non-persisted reasoning ids without encrypted state", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare(