Add explicit LLM stream lifecycle events (#26722)

This commit is contained in:
Kit Langton
2026-05-10 12:19:13 -04:00
committed by GitHub
parent 3b8790e034
commit ce061bf661
16 changed files with 201 additions and 126 deletions
+1 -1
View File
@@ -184,7 +184,7 @@ const FakeProtocol = Protocol.make<FakeBody, string, string, void>({
stream: { stream: {
event: Schema.String, event: Schema.String,
initial: () => undefined, initial: () => undefined,
step: (_, frame) => Effect.succeed([undefined, [{ type: "text-delta", text: frame }]] as const), step: (_, frame) => Effect.succeed([undefined, [{ type: "text-delta", id: "text-0", text: frame }]] as const),
onHalt: () => [{ type: "request-finish", reason: "stop" }], onHalt: () => [{ type: "request-finish", reason: "stop" }],
}, },
}) })
@@ -5,10 +5,10 @@ import { Endpoint } from "../route/endpoint"
import { Framing } from "../route/framing" import { Framing } from "../route/framing"
import { Protocol } from "../route/protocol" import { Protocol } from "../route/protocol"
import { import {
LLMEvent,
Usage, Usage,
type CacheHint, type CacheHint,
type FinishReason, type FinishReason,
type LLMEvent,
type LLMRequest, type LLMRequest,
type ProviderMetadata, type ProviderMetadata,
type ToolCallPart, type ToolCallPart,
@@ -415,14 +415,13 @@ const serverToolResultEvent = (block: NonNullable<AnthropicEvent["content_block"
? String((block.content as Record<string, unknown>).type) ? String((block.content as Record<string, unknown>).type)
: "" : ""
const isError = errorPayload.endsWith("_tool_result_error") const isError = errorPayload.endsWith("_tool_result_error")
return { return LLMEvent.toolResult({
type: "tool-result",
id: block.tool_use_id ?? "", id: block.tool_use_id ?? "",
name: SERVER_TOOL_RESULT_NAMES[block.type], name: SERVER_TOOL_RESULT_NAMES[block.type],
result: isError ? { type: "error", value: block.content } : { type: "json", value: block.content }, result: isError ? { type: "error", value: block.content } : { type: "json", value: block.content },
providerExecuted: true, providerExecuted: true,
providerMetadata: anthropicMetadata({ blockType: block.type }), providerMetadata: anthropicMetadata({ blockType: block.type }),
} })
} }
type StepResult = readonly [ParserState, ReadonlyArray<LLMEvent>] type StepResult = readonly [ParserState, ReadonlyArray<LLMEvent>]
@@ -453,18 +452,17 @@ const onContentBlockStart = (state: ParserState, event: AnthropicEvent): StepRes
} }
if (block.type === "text" && block.text) { if (block.type === "text" && block.text) {
return [state, [{ type: "text-delta", text: block.text }]] return [state, [LLMEvent.textDelta({ id: `text-${event.index ?? 0}`, text: block.text })]]
} }
if (block.type === "thinking" && block.thinking) { if (block.type === "thinking" && block.thinking) {
return [ return [
state, state,
[ [
{ LLMEvent.reasoningDelta({
type: "reasoning-delta", id: `reasoning-${event.index ?? 0}`,
text: block.thinking, text: block.thinking,
...(block.signature ? { providerMetadata: anthropicMetadata({ signature: block.signature }) } : {}), }),
},
], ],
] ]
} }
@@ -480,17 +478,17 @@ const onContentBlockDelta = Effect.fn("AnthropicMessages.onContentBlockDelta")(f
const delta = event.delta const delta = event.delta
if (delta?.type === "text_delta" && delta.text) { if (delta?.type === "text_delta" && delta.text) {
return [state, [{ type: "text-delta", text: delta.text }]] satisfies StepResult return [state, [LLMEvent.textDelta({ id: `text-${event.index ?? 0}`, text: delta.text })]] satisfies StepResult
} }
if (delta?.type === "thinking_delta" && delta.thinking) { if (delta?.type === "thinking_delta" && delta.thinking) {
return [state, [{ type: "reasoning-delta", text: delta.thinking }]] satisfies StepResult return [state, [LLMEvent.reasoningDelta({ id: `reasoning-${event.index ?? 0}`, text: delta.thinking })]] satisfies StepResult
} }
if (delta?.type === "signature_delta" && delta.signature) { if (delta?.type === "signature_delta" && delta.signature) {
return [ return [
state, state,
[{ type: "reasoning-delta", text: "", providerMetadata: anthropicMetadata({ signature: delta.signature }) }], [LLMEvent.reasoningEnd({ id: `reasoning-${event.index ?? 0}`, providerMetadata: anthropicMetadata({ signature: delta.signature }) })],
] satisfies StepResult ] satisfies StepResult
} }
@@ -524,21 +522,20 @@ const onMessageDelta = (state: ParserState, event: AnthropicEvent): StepResult =
return [ return [
{ ...state, usage }, { ...state, usage },
[ [
{ LLMEvent.requestFinish({
type: "request-finish",
reason: mapFinishReason(event.delta?.stop_reason), reason: mapFinishReason(event.delta?.stop_reason),
usage, usage,
...(event.delta?.stop_sequence providerMetadata: event.delta?.stop_sequence
? { providerMetadata: anthropicMetadata({ stopSequence: event.delta.stop_sequence }) } ? anthropicMetadata({ stopSequence: event.delta.stop_sequence })
: {}), : undefined,
}, }),
], ],
] ]
} }
const onError = (state: ParserState, event: AnthropicEvent): StepResult => [ const onError = (state: ParserState, event: AnthropicEvent): StepResult => [
state, state,
[{ type: "provider-error", message: event.error?.message ?? "Anthropic Messages stream error" }], [LLMEvent.providerError({ message: event.error?.message ?? "Anthropic Messages stream error" })],
] ]
const step = (state: ParserState, event: AnthropicEvent) => { const step = (state: ParserState, event: AnthropicEvent) => {
+19 -6
View File
@@ -3,10 +3,10 @@ import { Route, type RouteModelInput } from "../route/client"
import { Endpoint } from "../route/endpoint" import { Endpoint } from "../route/endpoint"
import { Protocol } from "../route/protocol" import { Protocol } from "../route/protocol"
import { import {
LLMEvent,
Usage, Usage,
type CacheHint, type CacheHint,
type FinishReason, type FinishReason,
type LLMEvent,
type LLMRequest, type LLMRequest,
type ToolCallPart, type ToolCallPart,
type ToolDefinition, type ToolDefinition,
@@ -400,13 +400,26 @@ const step = (state: ParserState, event: BedrockEvent) =>
} }
if (event.contentBlockDelta?.delta?.text) { if (event.contentBlockDelta?.delta?.text) {
return [state, [{ type: "text-delta" as const, text: event.contentBlockDelta.delta.text }]] as const return [
state,
[
LLMEvent.textDelta({
id: `text-${event.contentBlockDelta.contentBlockIndex}`,
text: event.contentBlockDelta.delta.text,
}),
],
] as const
} }
if (event.contentBlockDelta?.delta?.reasoningContent?.text) { if (event.contentBlockDelta?.delta?.reasoningContent?.text) {
return [ return [
state, state,
[{ type: "reasoning-delta" as const, text: event.contentBlockDelta.delta.reasoningContent.text }], [
LLMEvent.reasoningDelta({
id: `reasoning-${event.contentBlockDelta.contentBlockIndex}`,
text: event.contentBlockDelta.delta.reasoningContent.text,
}),
],
] as const ] as const
} }
@@ -449,7 +462,7 @@ const step = (state: ParserState, event: BedrockEvent) =>
event.modelStreamErrorException?.message ?? event.modelStreamErrorException?.message ??
event.serviceUnavailableException?.message ?? event.serviceUnavailableException?.message ??
"Bedrock Converse stream error" "Bedrock Converse stream error"
return [state, [{ type: "provider-error" as const, message, retryable: true }]] as const return [state, [LLMEvent.providerError({ message, retryable: true })]] as const
} }
if (event.validationException || event.throttlingException) { if (event.validationException || event.throttlingException) {
@@ -457,7 +470,7 @@ const step = (state: ParserState, event: BedrockEvent) =>
event.validationException?.message ?? event.throttlingException?.message ?? "Bedrock Converse error" event.validationException?.message ?? event.throttlingException?.message ?? "Bedrock Converse error"
return [ return [
state, state,
[{ type: "provider-error" as const, message, retryable: event.throttlingException !== undefined }], [LLMEvent.providerError({ message, retryable: event.throttlingException !== undefined })],
] as const ] as const
} }
@@ -468,7 +481,7 @@ const framing = BedrockEventStream.framing(ADAPTER)
const onHalt = (state: ParserState): ReadonlyArray<LLMEvent> => const onHalt = (state: ParserState): ReadonlyArray<LLMEvent> =>
state.pendingFinish state.pendingFinish
? [{ type: "request-finish", reason: state.pendingFinish.reason, usage: state.pendingFinish.usage }] ? [LLMEvent.requestFinish({ reason: state.pendingFinish.reason, usage: state.pendingFinish.usage })]
: [] : []
// ============================================================================= // =============================================================================
+8 -4
View File
@@ -5,9 +5,9 @@ import { Endpoint } from "../route/endpoint"
import { Framing } from "../route/framing" import { Framing } from "../route/framing"
import { Protocol } from "../route/protocol" import { Protocol } from "../route/protocol"
import { import {
LLMEvent,
Usage, Usage,
type FinishReason, type FinishReason,
type LLMEvent,
type LLMRequest, type LLMRequest,
type MediaPart, type MediaPart,
type TextPart, type TextPart,
@@ -311,7 +311,7 @@ const mapFinishReason = (finishReason: string | undefined, hasToolCalls: boolean
const finish = (state: ParserState): ReadonlyArray<LLMEvent> => const finish = (state: ParserState): ReadonlyArray<LLMEvent> =>
state.finishReason || state.usage state.finishReason || state.usage
? [{ type: "request-finish", reason: mapFinishReason(state.finishReason, state.hasToolCalls), usage: state.usage }] ? [LLMEvent.requestFinish({ reason: mapFinishReason(state.finishReason, state.hasToolCalls), usage: state.usage })]
: [] : []
const step = (state: ParserState, event: GeminiEvent) => { const step = (state: ParserState, event: GeminiEvent) => {
@@ -332,14 +332,18 @@ const step = (state: ParserState, event: GeminiEvent) => {
for (const part of candidate.content.parts) { for (const part of candidate.content.parts) {
if ("text" in part && part.text.length > 0) { if ("text" in part && part.text.length > 0) {
events.push({ type: part.thought ? "reasoning-delta" : "text-delta", text: part.text }) events.push(
part.thought
? LLMEvent.reasoningDelta({ id: "reasoning-0", text: part.text })
: LLMEvent.textDelta({ id: "text-0", text: part.text }),
)
continue continue
} }
if ("functionCall" in part) { if ("functionCall" in part) {
const input = part.functionCall.args const input = part.functionCall.args
const id = `tool_${nextToolCallId++}` const id = `tool_${nextToolCallId++}`
events.push({ type: "tool-call", id, name: part.functionCall.name, input }) events.push(LLMEvent.toolCall({ id, name: part.functionCall.name, input }))
hasToolCalls = true hasToolCalls = true
} }
} }
+3 -3
View File
@@ -6,9 +6,9 @@ import { Framing } from "../route/framing"
import { HttpTransport } from "../route/transport" import { HttpTransport } from "../route/transport"
import { Protocol } from "../route/protocol" import { Protocol } from "../route/protocol"
import { import {
LLMEvent,
Usage, Usage,
type FinishReason, type FinishReason,
type LLMEvent,
type LLMRequest, type LLMRequest,
type TextPart, type TextPart,
type ToolCallPart, type ToolCallPart,
@@ -312,7 +312,7 @@ const step = (state: ParserState, event: OpenAIChatEvent) =>
const toolDeltas = delta?.tool_calls ?? [] const toolDeltas = delta?.tool_calls ?? []
let tools = state.tools let tools = state.tools
if (delta?.content) events.push({ type: "text-delta", text: delta.content }) if (delta?.content) events.push(LLMEvent.textDelta({ id: "text-0", text: delta.content }))
for (const tool of toolDeltas) { for (const tool of toolDeltas) {
const result = ToolStream.appendOrStart( const result = ToolStream.appendOrStart(
@@ -350,7 +350,7 @@ const finishEvents = (state: ParserState): ReadonlyArray<LLMEvent> => {
const reason = state.finishReason === "stop" && hasToolCalls ? "tool-calls" : state.finishReason const reason = state.finishReason === "stop" && hasToolCalls ? "tool-calls" : state.finishReason
return [ return [
...state.toolCallEvents, ...state.toolCallEvents,
...(reason ? ([{ type: "request-finish", reason, usage: state.usage }] satisfies ReadonlyArray<LLMEvent>) : []), ...(reason ? [LLMEvent.requestFinish({ reason, usage: state.usage })] : []),
] ]
} }
+15 -24
View File
@@ -6,9 +6,9 @@ import { Framing } from "../route/framing"
import { HttpTransport, WebSocketTransport } from "../route/transport" import { HttpTransport, WebSocketTransport } from "../route/transport"
import { Protocol } from "../route/protocol" import { Protocol } from "../route/protocol"
import { import {
LLMEvent,
Usage, Usage,
type FinishReason, type FinishReason,
type LLMEvent,
type LLMRequest, type LLMRequest,
type ProviderMetadata, type ProviderMetadata,
type TextPart, type TextPart,
@@ -348,22 +348,20 @@ const hostedToolEvents = (
const tool = HOSTED_TOOLS[item.type] const tool = HOSTED_TOOLS[item.type]
const providerMetadata = openaiMetadata({ itemId: item.id }) const providerMetadata = openaiMetadata({ itemId: item.id })
return [ return [
{ LLMEvent.toolCall({
type: "tool-call",
id: item.id, id: item.id,
name: tool.name, name: tool.name,
input: tool.input(item), input: tool.input(item),
providerExecuted: true, providerExecuted: true,
providerMetadata, providerMetadata,
}, }),
{ LLMEvent.toolResult({
type: "tool-result",
id: item.id, id: item.id,
name: tool.name, name: tool.name,
result: hostedToolResult(item), result: hostedToolResult(item),
providerExecuted: true, providerExecuted: true,
providerMetadata, providerMetadata,
}, }),
] ]
} }
@@ -382,12 +380,7 @@ const onOutputTextDelta = (state: ParserState, event: OpenAIResponsesEvent): Ste
return [ return [
state, state,
[ [
{ LLMEvent.textDelta({ id: event.item_id ?? "text-0", text: event.delta }),
type: "text-delta",
id: event.item_id,
text: event.delta,
...(event.item_id ? { providerMetadata: openaiMetadata({ itemId: event.item_id }) } : {}),
},
], ],
] ]
} }
@@ -458,30 +451,28 @@ const onOutputItemDone = Effect.fn("OpenAIResponses.onOutputItemDone")(function*
const onResponseFinish = (state: ParserState, event: OpenAIResponsesEvent): StepResult => [ const onResponseFinish = (state: ParserState, event: OpenAIResponsesEvent): StepResult => [
state, state,
[ [
{ LLMEvent.requestFinish({
type: "request-finish",
reason: mapFinishReason(event, state.hasFunctionCall), reason: mapFinishReason(event, state.hasFunctionCall),
usage: mapUsage(event.response?.usage), usage: mapUsage(event.response?.usage),
...(event.response?.id || event.response?.service_tier providerMetadata:
? { event.response?.id || event.response?.service_tier
providerMetadata: openaiMetadata({ ? openaiMetadata({
responseId: event.response.id, responseId: event.response.id,
serviceTier: event.response.service_tier, serviceTier: event.response.service_tier,
}), })
} : undefined,
: {}), }),
},
], ],
] ]
const onResponseFailed = (state: ParserState, event: OpenAIResponsesEvent): StepResult => [ const onResponseFailed = (state: ParserState, event: OpenAIResponsesEvent): StepResult => [
state, state,
[{ type: "provider-error", message: event.message ?? event.code ?? "OpenAI Responses response failed" }], [LLMEvent.providerError({ message: event.message ?? event.code ?? "OpenAI Responses response failed" })],
] ]
const onError = (state: ParserState, event: OpenAIResponsesEvent): StepResult => [ const onError = (state: ParserState, event: OpenAIResponsesEvent): StepResult => [
state, state,
[{ type: "provider-error", message: event.message ?? event.code ?? "OpenAI Responses stream error" }], [LLMEvent.providerError({ message: event.message ?? event.code ?? "OpenAI Responses stream error" })],
] ]
const step = (state: ParserState, event: OpenAIResponsesEvent) => { const step = (state: ParserState, event: OpenAIResponsesEvent) => {
+14 -24
View File
@@ -1,5 +1,5 @@
import { Effect } from "effect" import { Effect } from "effect"
import { LLMError, type ProviderMetadata, type ToolCall, type ToolInputDelta } from "../../schema" import { LLMError, LLMEvent, type ProviderMetadata, type ToolCall, type ToolInputDelta } from "../../schema"
import { eventError, parseToolInput, type ToolAccumulator } from "../shared" import { eventError, parseToolInput, type ToolAccumulator } from "../shared"
type StreamKey = string | number type StreamKey = string | number
@@ -49,34 +49,24 @@ const withoutTool = <K extends StreamKey>(tools: State<K>, key: K): State<K> =>
return next return next
} }
const inputDelta = (tool: PendingTool, text: string): ToolInputDelta => ({ const inputDelta = (tool: PendingTool, text: string): ToolInputDelta =>
type: "tool-input-delta", LLMEvent.toolInputDelta({
id: tool.id, id: tool.id,
name: tool.name, name: tool.name,
text, text,
...(tool.providerMetadata ? { providerMetadata: tool.providerMetadata } : {}), })
})
const toolCall = (route: string, tool: PendingTool, inputOverride?: string) => const toolCall = (route: string, tool: PendingTool, inputOverride?: string) =>
parseToolInput(route, tool.name, inputOverride ?? tool.input).pipe( parseToolInput(route, tool.name, inputOverride ?? tool.input).pipe(
Effect.map( Effect.map(
(input): ToolCall => (input): ToolCall =>
tool.providerExecuted LLMEvent.toolCall({
? { id: tool.id,
type: "tool-call", name: tool.name,
id: tool.id, input,
name: tool.name, providerExecuted: tool.providerExecuted ? true : undefined,
input, providerMetadata: tool.providerMetadata,
providerExecuted: true, }),
...(tool.providerMetadata ? { providerMetadata: tool.providerMetadata } : {}),
}
: {
type: "tool-call",
id: tool.id,
name: tool.name,
input,
...(tool.providerMetadata ? { providerMetadata: tool.providerMetadata } : {}),
},
), ),
) )
+90 -27
View File
@@ -1,5 +1,5 @@
import { Schema } from "effect" import { Schema } from "effect"
import { FinishReason, ProtocolID, ProviderMetadata, RouteID } from "./ids" import { ContentBlockID, FinishReason, ProtocolID, ProviderMetadata, ResponseID, RouteID, ToolCallID } from "./ids"
import { ModelRef } from "./options" import { ModelRef } from "./options"
import { ToolResultValue } from "./messages" import { ToolResultValue } from "./messages"
@@ -14,60 +14,87 @@ export class Usage extends Schema.Class<Usage>("LLM.Usage")({
}) {} }) {}
export const RequestStart = Schema.Struct({ export const RequestStart = Schema.Struct({
type: Schema.Literal("request-start"), type: Schema.tag("request-start"),
id: Schema.String, id: ResponseID,
model: ModelRef, model: ModelRef,
}).annotate({ identifier: "LLM.Event.RequestStart" }) }).annotate({ identifier: "LLM.Event.RequestStart" })
export type RequestStart = Schema.Schema.Type<typeof RequestStart> export type RequestStart = Schema.Schema.Type<typeof RequestStart>
export const StepStart = Schema.Struct({ export const StepStart = Schema.Struct({
type: Schema.Literal("step-start"), type: Schema.tag("step-start"),
index: Schema.Number, index: Schema.Number,
}).annotate({ identifier: "LLM.Event.StepStart" }) }).annotate({ identifier: "LLM.Event.StepStart" })
export type StepStart = Schema.Schema.Type<typeof StepStart> export type StepStart = Schema.Schema.Type<typeof StepStart>
export const TextStart = Schema.Struct({ export const TextStart = Schema.Struct({
type: Schema.Literal("text-start"), type: Schema.tag("text-start"),
id: Schema.String, id: ContentBlockID,
providerMetadata: Schema.optional(ProviderMetadata), providerMetadata: Schema.optional(ProviderMetadata),
}).annotate({ identifier: "LLM.Event.TextStart" }) }).annotate({ identifier: "LLM.Event.TextStart" })
export type TextStart = Schema.Schema.Type<typeof TextStart> export type TextStart = Schema.Schema.Type<typeof TextStart>
export const TextDelta = Schema.Struct({ export const TextDelta = Schema.Struct({
type: Schema.Literal("text-delta"), type: Schema.tag("text-delta"),
id: Schema.optional(Schema.String), id: ContentBlockID,
text: Schema.String, text: Schema.String,
providerMetadata: Schema.optional(ProviderMetadata),
}).annotate({ identifier: "LLM.Event.TextDelta" }) }).annotate({ identifier: "LLM.Event.TextDelta" })
export type TextDelta = Schema.Schema.Type<typeof TextDelta> export type TextDelta = Schema.Schema.Type<typeof TextDelta>
export const TextEnd = Schema.Struct({ export const TextEnd = Schema.Struct({
type: Schema.Literal("text-end"), type: Schema.tag("text-end"),
id: Schema.String, id: ContentBlockID,
providerMetadata: Schema.optional(ProviderMetadata), providerMetadata: Schema.optional(ProviderMetadata),
}).annotate({ identifier: "LLM.Event.TextEnd" }) }).annotate({ identifier: "LLM.Event.TextEnd" })
export type TextEnd = Schema.Schema.Type<typeof TextEnd> export type TextEnd = Schema.Schema.Type<typeof TextEnd>
export const ReasoningDelta = Schema.Struct({ export const ReasoningStart = Schema.Struct({
type: Schema.Literal("reasoning-delta"), type: Schema.tag("reasoning-start"),
id: Schema.optional(Schema.String), id: ContentBlockID,
text: Schema.String,
providerMetadata: Schema.optional(ProviderMetadata), providerMetadata: Schema.optional(ProviderMetadata),
}).annotate({ identifier: "LLM.Event.ReasoningStart" })
export type ReasoningStart = Schema.Schema.Type<typeof ReasoningStart>
export const ReasoningDelta = Schema.Struct({
type: Schema.tag("reasoning-delta"),
id: ContentBlockID,
text: Schema.String,
}).annotate({ identifier: "LLM.Event.ReasoningDelta" }) }).annotate({ identifier: "LLM.Event.ReasoningDelta" })
export type ReasoningDelta = Schema.Schema.Type<typeof ReasoningDelta> export type ReasoningDelta = Schema.Schema.Type<typeof ReasoningDelta>
export const ReasoningEnd = Schema.Struct({
type: Schema.tag("reasoning-end"),
id: ContentBlockID,
providerMetadata: Schema.optional(ProviderMetadata),
}).annotate({ identifier: "LLM.Event.ReasoningEnd" })
export type ReasoningEnd = Schema.Schema.Type<typeof ReasoningEnd>
export const ToolInputStart = Schema.Struct({
type: Schema.tag("tool-input-start"),
id: ToolCallID,
name: Schema.String,
providerMetadata: Schema.optional(ProviderMetadata),
}).annotate({ identifier: "LLM.Event.ToolInputStart" })
export type ToolInputStart = Schema.Schema.Type<typeof ToolInputStart>
export const ToolInputDelta = Schema.Struct({ export const ToolInputDelta = Schema.Struct({
type: Schema.Literal("tool-input-delta"), type: Schema.tag("tool-input-delta"),
id: Schema.String, id: ToolCallID,
name: Schema.String, name: Schema.String,
text: Schema.String, text: Schema.String,
providerMetadata: Schema.optional(ProviderMetadata),
}).annotate({ identifier: "LLM.Event.ToolInputDelta" }) }).annotate({ identifier: "LLM.Event.ToolInputDelta" })
export type ToolInputDelta = Schema.Schema.Type<typeof ToolInputDelta> export type ToolInputDelta = Schema.Schema.Type<typeof ToolInputDelta>
export const ToolInputEnd = Schema.Struct({
type: Schema.tag("tool-input-end"),
id: ToolCallID,
name: Schema.String,
providerMetadata: Schema.optional(ProviderMetadata),
}).annotate({ identifier: "LLM.Event.ToolInputEnd" })
export type ToolInputEnd = Schema.Schema.Type<typeof ToolInputEnd>
export const ToolCall = Schema.Struct({ export const ToolCall = Schema.Struct({
type: Schema.Literal("tool-call"), type: Schema.tag("tool-call"),
id: Schema.String, id: ToolCallID,
name: Schema.String, name: Schema.String,
input: Schema.Unknown, input: Schema.Unknown,
providerExecuted: Schema.optional(Schema.Boolean), providerExecuted: Schema.optional(Schema.Boolean),
@@ -76,8 +103,8 @@ export const ToolCall = Schema.Struct({
export type ToolCall = Schema.Schema.Type<typeof ToolCall> export type ToolCall = Schema.Schema.Type<typeof ToolCall>
export const ToolResult = Schema.Struct({ export const ToolResult = Schema.Struct({
type: Schema.Literal("tool-result"), type: Schema.tag("tool-result"),
id: Schema.String, id: ToolCallID,
name: Schema.String, name: Schema.String,
result: ToolResultValue, result: ToolResultValue,
providerExecuted: Schema.optional(Schema.Boolean), providerExecuted: Schema.optional(Schema.Boolean),
@@ -86,8 +113,8 @@ export const ToolResult = Schema.Struct({
export type ToolResult = Schema.Schema.Type<typeof ToolResult> export type ToolResult = Schema.Schema.Type<typeof ToolResult>
export const ToolError = Schema.Struct({ export const ToolError = Schema.Struct({
type: Schema.Literal("tool-error"), type: Schema.tag("tool-error"),
id: Schema.String, id: ToolCallID,
name: Schema.String, name: Schema.String,
message: Schema.String, message: Schema.String,
providerMetadata: Schema.optional(ProviderMetadata), providerMetadata: Schema.optional(ProviderMetadata),
@@ -95,7 +122,7 @@ export const ToolError = Schema.Struct({
export type ToolError = Schema.Schema.Type<typeof ToolError> export type ToolError = Schema.Schema.Type<typeof ToolError>
export const StepFinish = Schema.Struct({ export const StepFinish = Schema.Struct({
type: Schema.Literal("step-finish"), type: Schema.tag("step-finish"),
index: Schema.Number, index: Schema.Number,
reason: FinishReason, reason: FinishReason,
usage: Schema.optional(Usage), usage: Schema.optional(Usage),
@@ -104,7 +131,7 @@ export const StepFinish = Schema.Struct({
export type StepFinish = Schema.Schema.Type<typeof StepFinish> export type StepFinish = Schema.Schema.Type<typeof StepFinish>
export const RequestFinish = Schema.Struct({ export const RequestFinish = Schema.Struct({
type: Schema.Literal("request-finish"), type: Schema.tag("request-finish"),
reason: FinishReason, reason: FinishReason,
usage: Schema.optional(Usage), usage: Schema.optional(Usage),
providerMetadata: Schema.optional(ProviderMetadata), providerMetadata: Schema.optional(ProviderMetadata),
@@ -112,7 +139,7 @@ export const RequestFinish = Schema.Struct({
export type RequestFinish = Schema.Schema.Type<typeof RequestFinish> export type RequestFinish = Schema.Schema.Type<typeof RequestFinish>
export const ProviderErrorEvent = Schema.Struct({ export const ProviderErrorEvent = Schema.Struct({
type: Schema.Literal("provider-error"), type: Schema.tag("provider-error"),
message: Schema.String, message: Schema.String,
retryable: Schema.optional(Schema.Boolean), retryable: Schema.optional(Schema.Boolean),
providerMetadata: Schema.optional(ProviderMetadata), providerMetadata: Schema.optional(ProviderMetadata),
@@ -125,8 +152,12 @@ const llmEventTagged = Schema.Union([
TextStart, TextStart,
TextDelta, TextDelta,
TextEnd, TextEnd,
ReasoningStart,
ReasoningDelta, ReasoningDelta,
ReasoningEnd,
ToolInputStart,
ToolInputDelta, ToolInputDelta,
ToolInputEnd,
ToolCall, ToolCall,
ToolResult, ToolResult,
ToolError, ToolError,
@@ -135,20 +166,52 @@ const llmEventTagged = Schema.Union([
ProviderErrorEvent, ProviderErrorEvent,
]).pipe(Schema.toTaggedUnion("type")) ]).pipe(Schema.toTaggedUnion("type"))
type WithID<Event extends { readonly id: unknown }, ID> = Omit<Event, "type" | "id"> & { readonly id: ID | string }
const responseID = (value: ResponseID | string) => ResponseID.make(value)
const contentBlockID = (value: ContentBlockID | string) => ContentBlockID.make(value)
const toolCallID = (value: ToolCallID | string) => ToolCallID.make(value)
/** /**
* camelCase aliases for `LLMEvent.guards` (provided by `Schema.toTaggedUnion`). * camelCase aliases for `LLMEvent.guards` (provided by `Schema.toTaggedUnion`).
* Lets consumers write `events.filter(LLMEvent.is.toolCall)` instead of * Lets consumers write `events.filter(LLMEvent.is.toolCall)` instead of
* `events.filter(LLMEvent.guards["tool-call"])`. * `events.filter(LLMEvent.guards["tool-call"])`.
*/ */
export const LLMEvent = Object.assign(llmEventTagged, { export const LLMEvent = Object.assign(llmEventTagged, {
requestStart: (input: WithID<RequestStart, ResponseID>) => RequestStart.make({ ...input, id: responseID(input.id) }),
stepStart: StepStart.make,
textStart: (input: WithID<TextStart, ContentBlockID>) => TextStart.make({ ...input, id: contentBlockID(input.id) }),
textDelta: (input: WithID<TextDelta, ContentBlockID>) => TextDelta.make({ ...input, id: contentBlockID(input.id) }),
textEnd: (input: WithID<TextEnd, ContentBlockID>) => TextEnd.make({ ...input, id: contentBlockID(input.id) }),
reasoningStart: (input: WithID<ReasoningStart, ContentBlockID>) =>
ReasoningStart.make({ ...input, id: contentBlockID(input.id) }),
reasoningDelta: (input: WithID<ReasoningDelta, ContentBlockID>) =>
ReasoningDelta.make({ ...input, id: contentBlockID(input.id) }),
reasoningEnd: (input: WithID<ReasoningEnd, ContentBlockID>) =>
ReasoningEnd.make({ ...input, id: contentBlockID(input.id) }),
toolInputStart: (input: WithID<ToolInputStart, ToolCallID>) =>
ToolInputStart.make({ ...input, id: toolCallID(input.id) }),
toolInputDelta: (input: WithID<ToolInputDelta, ToolCallID>) =>
ToolInputDelta.make({ ...input, id: toolCallID(input.id) }),
toolInputEnd: (input: WithID<ToolInputEnd, ToolCallID>) => ToolInputEnd.make({ ...input, id: toolCallID(input.id) }),
toolCall: (input: WithID<ToolCall, ToolCallID>) => ToolCall.make({ ...input, id: toolCallID(input.id) }),
toolResult: (input: WithID<ToolResult, ToolCallID>) => ToolResult.make({ ...input, id: toolCallID(input.id) }),
toolError: (input: WithID<ToolError, ToolCallID>) => ToolError.make({ ...input, id: toolCallID(input.id) }),
stepFinish: StepFinish.make,
requestFinish: RequestFinish.make,
providerError: ProviderErrorEvent.make,
is: { is: {
requestStart: llmEventTagged.guards["request-start"], requestStart: llmEventTagged.guards["request-start"],
stepStart: llmEventTagged.guards["step-start"], stepStart: llmEventTagged.guards["step-start"],
textStart: llmEventTagged.guards["text-start"], textStart: llmEventTagged.guards["text-start"],
textDelta: llmEventTagged.guards["text-delta"], textDelta: llmEventTagged.guards["text-delta"],
textEnd: llmEventTagged.guards["text-end"], textEnd: llmEventTagged.guards["text-end"],
reasoningStart: llmEventTagged.guards["reasoning-start"],
reasoningDelta: llmEventTagged.guards["reasoning-delta"], reasoningDelta: llmEventTagged.guards["reasoning-delta"],
reasoningEnd: llmEventTagged.guards["reasoning-end"],
toolInputStart: llmEventTagged.guards["tool-input-start"],
toolInputDelta: llmEventTagged.guards["tool-input-delta"], toolInputDelta: llmEventTagged.guards["tool-input-delta"],
toolInputEnd: llmEventTagged.guards["tool-input-end"],
toolCall: llmEventTagged.guards["tool-call"], toolCall: llmEventTagged.guards["tool-call"],
toolResult: llmEventTagged.guards["tool-result"], toolResult: llmEventTagged.guards["tool-result"],
toolError: llmEventTagged.guards["tool-error"], toolError: llmEventTagged.guards["tool-error"],
+9
View File
@@ -14,6 +14,15 @@ export type ModelID = typeof ModelID.Type
export const ProviderID = Schema.String.pipe(Schema.brand("LLM.ProviderID")) export const ProviderID = Schema.String.pipe(Schema.brand("LLM.ProviderID"))
export type ProviderID = typeof ProviderID.Type export type ProviderID = typeof ProviderID.Type
export const ResponseID = Schema.String
export type ResponseID = Schema.Schema.Type<typeof ResponseID>
export const ContentBlockID = Schema.String
export type ContentBlockID = Schema.Schema.Type<typeof ContentBlockID>
export const ToolCallID = Schema.String
export type ToolCallID = Schema.Schema.Type<typeof ToolCallID>
export const ReasoningEfforts = ["none", "minimal", "low", "medium", "high", "xhigh", "max"] as const export const ReasoningEfforts = ["none", "minimal", "low", "medium", "high", "xhigh", "max"] as const
export const ReasoningEffort = Schema.Literals(ReasoningEfforts) export const ReasoningEffort = Schema.Literals(ReasoningEfforts)
export type ReasoningEffort = Schema.Schema.Type<typeof ReasoningEffort> export type ReasoningEffort = Schema.Schema.Type<typeof ReasoningEffort>
+14 -6
View File
@@ -4,7 +4,7 @@ import {
type ContentPart, type ContentPart,
type FinishReason, type FinishReason,
type LLMError, type LLMError,
type LLMEvent, LLMEvent,
LLMRequest, LLMRequest,
Message, Message,
type ProviderMetadata, type ProviderMetadata,
@@ -115,11 +115,19 @@ interface StepState {
const accumulate = (state: StepState, event: LLMEvent) => { const accumulate = (state: StepState, event: LLMEvent) => {
if (event.type === "text-delta") { if (event.type === "text-delta") {
appendStreamingText(state, "text", event.text, event.providerMetadata) appendStreamingText(state, "text", event.text, undefined)
return return
} }
if (event.type === "reasoning-delta") { if (event.type === "reasoning-delta") {
appendStreamingText(state, "reasoning", event.text, event.providerMetadata) appendStreamingText(state, "reasoning", event.text, undefined)
return
}
if (event.type === "reasoning-end") {
appendStreamingText(state, "reasoning", "", event.providerMetadata)
return
}
if (event.type === "text-end") {
appendStreamingText(state, "text", "", event.providerMetadata)
return return
} }
if (event.type === "tool-call") { if (event.type === "tool-call") {
@@ -219,10 +227,10 @@ const decodeAndExecute = (tool: AnyTool, input: unknown): Effect.Effect<ToolResu
const emitEvents = (call: ToolCallPart, result: ToolResultValue): ReadonlyArray<LLMEvent> => const emitEvents = (call: ToolCallPart, result: ToolResultValue): ReadonlyArray<LLMEvent> =>
result.type === "error" result.type === "error"
? [ ? [
{ type: "tool-error", id: call.id, name: call.name, message: String(result.value) }, LLMEvent.toolError({ id: call.id, name: call.name, message: String(result.value) }),
{ type: "tool-result", id: call.id, name: call.name, result }, LLMEvent.toolResult({ id: call.id, name: call.name, result }),
] ]
: [{ type: "tool-result", id: call.id, name: call.name, result }] : [LLMEvent.toolResult({ id: call.id, name: call.name, result })]
const followUpRequest = ( const followUpRequest = (
request: LLMRequest, request: LLMRequest,
+3 -1
View File
@@ -50,7 +50,9 @@ const request = LLM.request({
}) })
const raiseEvent = (event: FakeEvent): import("../src/schema").LLMEvent => const raiseEvent = (event: FakeEvent): import("../src/schema").LLMEvent =>
event.type === "finish" ? { type: "request-finish", reason: event.reason } : { type: "text-delta", text: event.text } event.type === "finish"
? { type: "request-finish", reason: event.reason }
: { type: "text-delta", id: "text-0", text: event.text }
const fakeProtocol = Protocol.make<FakeBody, FakeEvent, FakeEvent, void>({ const fakeProtocol = Protocol.make<FakeBody, FakeEvent, FakeEvent, void>({
id: "fake", id: "fake",
+1 -1
View File
@@ -126,7 +126,7 @@ describe("llm constructors", () => {
expect( expect(
LLMResponse.text({ LLMResponse.text({
events: [ events: [
{ type: "text-delta", text: "hi" }, { type: "text-delta", id: "text-0", text: "hi" },
{ type: "request-finish", reason: "stop" }, { type: "request-finish", reason: "stop" },
], ],
}), }),
@@ -115,7 +115,7 @@ describe("Anthropic Messages route", () => {
cacheReadInputTokens: 1, cacheReadInputTokens: 1,
totalTokens: 7, totalTokens: 7,
}) })
expect(response.events.find((event) => event.type === "reasoning-delta" && event.text === "")).toMatchObject({ expect(response.events.find((event) => event.type === "reasoning-end")).toMatchObject({
providerMetadata: { anthropic: { signature: "sig_1" } }, providerMetadata: { anthropic: { signature: "sig_1" } },
}) })
expect(response.events.at(-1)).toMatchObject({ expect(response.events.at(-1)).toMatchObject({
+3 -3
View File
@@ -204,9 +204,9 @@ describe("Gemini route", () => {
totalTokens: 7, totalTokens: 7,
}) })
expect(response.events).toEqual([ expect(response.events).toEqual([
{ type: "reasoning-delta", text: "thinking" }, { type: "reasoning-delta", id: "reasoning-0", text: "thinking" },
{ type: "text-delta", text: "Hello" }, { type: "text-delta", id: "text-0", text: "Hello" },
{ type: "text-delta", text: "!" }, { type: "text-delta", id: "text-0", text: "!" },
{ {
type: "request-finish", type: "request-finish",
reason: "stop", reason: "stop",
@@ -225,8 +225,8 @@ describe("OpenAI Chat route", () => {
expect(response.text).toBe("Hello!") expect(response.text).toBe("Hello!")
expect(response.events).toEqual([ expect(response.events).toEqual([
{ type: "text-delta", text: "Hello" }, { type: "text-delta", id: "text-0", text: "Hello" },
{ type: "text-delta", text: "!" }, { type: "text-delta", id: "text-0", text: "!" },
{ {
type: "request-finish", type: "request-finish",
reason: "stop", reason: "stop",
@@ -336,8 +336,8 @@ describe("OpenAI Responses route", () => {
expect(response.text).toBe("Hello!") expect(response.text).toBe("Hello!")
expect(response.events).toEqual([ expect(response.events).toEqual([
{ type: "text-delta", id: "msg_1", text: "Hello", providerMetadata: { openai: { itemId: "msg_1" } } }, { type: "text-delta", id: "msg_1", text: "Hello" },
{ type: "text-delta", id: "msg_1", text: "!", providerMetadata: { openai: { itemId: "msg_1" } } }, { type: "text-delta", id: "msg_1", text: "!" },
{ {
type: "request-finish", type: "request-finish",
reason: "stop", reason: "stop",
@@ -394,14 +394,12 @@ describe("OpenAI Responses route", () => {
id: "call_1", id: "call_1",
name: "lookup", name: "lookup",
text: '{"query"', text: '{"query"',
providerMetadata: { openai: { itemId: "item_1" } },
}, },
{ {
type: "tool-input-delta", type: "tool-input-delta",
id: "call_1", id: "call_1",
name: "lookup", name: "lookup",
text: ':"weather"}', text: ':"weather"}',
providerMetadata: { openai: { itemId: "item_1" } },
}, },
{ {
type: "tool-call", type: "tool-call",