Add native LLM core foundation (#24712)
This commit is contained in:
Kit Langton
@@ -0,0 +1,549 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { ConfigProvider, Effect, Layer, Stream } from "effect"
|
||||
import { Headers, HttpClientRequest } from "effect/unstable/http"
|
||||
import { LLM, LLMError } from "../../src"
|
||||
import { Auth, LLMClient, RequestExecutor, WebSocketExecutor } from "../../src/route"
|
||||
import * as Azure from "../../src/providers/azure"
|
||||
import * as OpenAI from "../../src/providers/openai"
|
||||
import * as OpenAIResponses from "../../src/protocols/openai-responses"
|
||||
import * as ProviderShared from "../../src/protocols/shared"
|
||||
import { it } from "../lib/effect"
|
||||
import { dynamicResponse, fixedResponse } from "../lib/http"
|
||||
import { sseEvents } from "../lib/sse"
|
||||
|
||||
const model = OpenAIResponses.model({
|
||||
id: "gpt-4.1-mini",
|
||||
baseURL: "https://api.openai.test/v1/",
|
||||
headers: { authorization: "Bearer test" },
|
||||
})
|
||||
|
||||
const request = LLM.request({
|
||||
id: "req_1",
|
||||
model,
|
||||
system: "You are concise.",
|
||||
prompt: "Say hello.",
|
||||
generation: { maxTokens: 20, temperature: 0 },
|
||||
})
|
||||
|
||||
const configEnv = (env: Record<string, string>) => Effect.provide(ConfigProvider.layer(ConfigProvider.fromEnv({ env })))
|
||||
|
||||
describe("OpenAI Responses route", () => {
|
||||
it.effect("prepares OpenAI Responses target", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(request)
|
||||
|
||||
expect(prepared.body).toEqual({
|
||||
model: "gpt-4.1-mini",
|
||||
input: [
|
||||
{ role: "system", content: "You are concise." },
|
||||
{ role: "user", content: [{ type: "input_text", text: "Say hello." }] },
|
||||
],
|
||||
stream: true,
|
||||
max_output_tokens: 20,
|
||||
temperature: 0,
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("prepares OpenAI Responses WebSocket target", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(
|
||||
LLM.updateRequest(request, {
|
||||
model: OpenAI.responsesWebSocket("gpt-4.1-mini", { baseURL: "https://api.openai.test/v1/", apiKey: "test" }),
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.route).toBe("openai-responses-websocket")
|
||||
expect(prepared.protocol).toBe("openai-responses")
|
||||
expect(prepared.metadata).toEqual({ transport: "websocket-json" })
|
||||
expect(prepared.body).toMatchObject({ model: "gpt-4.1-mini", stream: true })
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("streams OpenAI Responses over WebSocket", () =>
|
||||
Effect.gen(function* () {
|
||||
const sent: string[] = []
|
||||
const opened: Array<{ readonly url: string; readonly authorization: string | undefined }> = []
|
||||
let closed = false
|
||||
const deps = Layer.mergeAll(
|
||||
Layer.succeed(
|
||||
RequestExecutor.Service,
|
||||
RequestExecutor.Service.of({
|
||||
execute: () => Effect.die("unexpected HTTP request"),
|
||||
}),
|
||||
),
|
||||
Layer.succeed(
|
||||
WebSocketExecutor.Service,
|
||||
WebSocketExecutor.Service.of({
|
||||
open: (input) =>
|
||||
Effect.succeed({
|
||||
sendText: (message) =>
|
||||
Effect.sync(() => {
|
||||
opened.push({ url: input.url, authorization: input.headers.authorization })
|
||||
sent.push(message)
|
||||
}),
|
||||
messages: Stream.fromArray([
|
||||
ProviderShared.encodeJson({ type: "response.output_text.delta", item_id: "msg_1", delta: "Hi" }),
|
||||
ProviderShared.encodeJson({ type: "response.completed", response: { id: "resp_ws" } }),
|
||||
]),
|
||||
close: Effect.sync(() => {
|
||||
closed = true
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
),
|
||||
)
|
||||
const response = yield* LLMClient.generate(
|
||||
LLM.request({
|
||||
model: OpenAI.responsesWebSocket("gpt-4.1-mini", { baseURL: "https://api.openai.test/v1/", apiKey: "test" }),
|
||||
prompt: "Say hello.",
|
||||
}),
|
||||
).pipe(Effect.provide(LLMClient.layerWithWebSocket.pipe(Layer.provide(deps))))
|
||||
|
||||
expect(response.text).toBe("Hi")
|
||||
expect(opened).toEqual([{ url: "wss://api.openai.test/v1/responses", authorization: "Bearer test" }])
|
||||
expect(closed).toBe(true)
|
||||
expect(sent).toHaveLength(1)
|
||||
expect(JSON.parse(sent[0])).toEqual({
|
||||
type: "response.create",
|
||||
model: "gpt-4.1-mini",
|
||||
input: [{ role: "user", content: [{ type: "input_text", text: "Say hello." }] }],
|
||||
store: false,
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("requires WebSocket runtime for OpenAI Responses WebSocket", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* LLMClient.generate(
|
||||
LLM.request({
|
||||
model: OpenAI.responsesWebSocket("gpt-4.1-mini", { baseURL: "https://api.openai.test/v1/", apiKey: "test" }),
|
||||
prompt: "Say hello.",
|
||||
}),
|
||||
).pipe(
|
||||
Effect.provide(
|
||||
LLMClient.layer.pipe(
|
||||
Layer.provide(
|
||||
Layer.succeed(
|
||||
RequestExecutor.Service,
|
||||
RequestExecutor.Service.of({
|
||||
execute: () => Effect.die("unexpected HTTP request"),
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
Effect.flip,
|
||||
)
|
||||
|
||||
expect(error.message).toContain("requires WebSocketExecutor.Service")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("fails immediately when WebSocket is already closed", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* WebSocketExecutor.fromWebSocket(
|
||||
{ readyState: globalThis.WebSocket.CLOSED } as globalThis.WebSocket,
|
||||
{ url: "wss://api.openai.test/v1/responses", headers: Headers.empty },
|
||||
).pipe(Effect.flip)
|
||||
|
||||
expect(error.message).toContain("closed before opening")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("adds native query params to the Responses URL", () =>
|
||||
Effect.gen(function* () {
|
||||
yield* LLMClient.generate(
|
||||
LLM.updateRequest(request, {
|
||||
model: OpenAIResponses.model({ ...model, queryParams: { "api-version": "v1" } }),
|
||||
}),
|
||||
).pipe(
|
||||
Effect.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.gen(function* () {
|
||||
const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
|
||||
expect(web.url).toBe("https://api.openai.test/v1/responses?api-version=v1")
|
||||
return input.respond(sseEvents({ type: "response.completed", response: {} }), {
|
||||
headers: { "content-type": "text/event-stream" },
|
||||
})
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("uses Azure api-key header for static OpenAI Responses keys", () =>
|
||||
Effect.gen(function* () {
|
||||
yield* LLMClient.generate(
|
||||
LLM.updateRequest(request, {
|
||||
model: Azure.responses("gpt-4.1-mini", {
|
||||
baseURL: "https://opencode-test.openai.azure.com/openai/v1/",
|
||||
apiKey: "azure-key",
|
||||
headers: { authorization: "Bearer stale" },
|
||||
}),
|
||||
}),
|
||||
).pipe(
|
||||
Effect.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.gen(function* () {
|
||||
const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
|
||||
expect(web.headers.get("api-key")).toBe("azure-key")
|
||||
expect(web.headers.get("authorization")).toBeNull()
|
||||
return input.respond(sseEvents({ type: "response.completed", response: {} }), {
|
||||
headers: { "content-type": "text/event-stream" },
|
||||
})
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("loads OpenAI default auth from Effect Config", () =>
|
||||
LLMClient.generate(
|
||||
LLM.updateRequest(request, {
|
||||
model: OpenAI.responses("gpt-4.1-mini", { baseURL: "https://api.openai.test/v1/" }),
|
||||
}),
|
||||
).pipe(
|
||||
configEnv({ OPENAI_API_KEY: "env-key" }),
|
||||
Effect.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.gen(function* () {
|
||||
const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
|
||||
expect(web.headers.get("authorization")).toBe("Bearer env-key")
|
||||
return input.respond(sseEvents({ type: "response.completed", response: {} }), {
|
||||
headers: { "content-type": "text/event-stream" },
|
||||
})
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
it.effect("lets explicit auth override OpenAI default API key auth", () =>
|
||||
LLMClient.generate(
|
||||
LLM.updateRequest(request, {
|
||||
model: OpenAI.responses("gpt-4.1-mini", {
|
||||
baseURL: "https://api.openai.test/v1/",
|
||||
auth: Auth.bearer("oauth-token"),
|
||||
}),
|
||||
}),
|
||||
).pipe(
|
||||
Effect.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.gen(function* () {
|
||||
const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
|
||||
expect(web.headers.get("authorization")).toBe("Bearer oauth-token")
|
||||
return input.respond(sseEvents({ type: "response.completed", response: {} }), {
|
||||
headers: { "content-type": "text/event-stream" },
|
||||
})
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
it.effect("prepares function call and function output input items", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
id: "req_tool_result",
|
||||
model,
|
||||
messages: [
|
||||
LLM.user("What is the weather?"),
|
||||
LLM.assistant([LLM.toolCall({ id: "call_1", name: "lookup", input: { query: "weather" } })]),
|
||||
LLM.toolMessage({ id: "call_1", name: "lookup", result: { forecast: "sunny" } }),
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body).toEqual({
|
||||
model: "gpt-4.1-mini",
|
||||
input: [
|
||||
{ role: "user", content: [{ type: "input_text", text: "What is the weather?" }] },
|
||||
{ type: "function_call", call_id: "call_1", name: "lookup", arguments: '{"query":"weather"}' },
|
||||
{ type: "function_call_output", call_id: "call_1", output: '{"forecast":"sunny"}' },
|
||||
],
|
||||
stream: true,
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("maps OpenAI provider options to Responses options", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
|
||||
LLM.request({
|
||||
model: OpenAI.model("gpt-5.2", { baseURL: "https://api.openai.test/v1/" }),
|
||||
prompt: "think",
|
||||
providerOptions: {
|
||||
openai: {
|
||||
promptCacheKey: "session_123",
|
||||
reasoningEffort: "high",
|
||||
reasoningSummary: "auto",
|
||||
includeEncryptedReasoning: true,
|
||||
},
|
||||
},
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body.store).toBe(false)
|
||||
expect(prepared.body.prompt_cache_key).toBe("session_123")
|
||||
expect(prepared.body.include).toEqual(["reasoning.encrypted_content"])
|
||||
expect(prepared.body.reasoning).toEqual({ effort: "high", summary: "auto" })
|
||||
expect(prepared.body.text).toEqual({ verbosity: "low" })
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("request OpenAI provider options override model defaults", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
|
||||
LLM.request({
|
||||
model: OpenAI.model("gpt-4.1-mini", {
|
||||
baseURL: "https://api.openai.test/v1/",
|
||||
providerOptions: { openai: { promptCacheKey: "model_cache" } },
|
||||
}),
|
||||
prompt: "no cache",
|
||||
providerOptions: { openai: { promptCacheKey: "request_cache" } },
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body.prompt_cache_key).toBe("request_cache")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("parses text and usage stream fixtures", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = sseEvents(
|
||||
{ type: "response.output_text.delta", item_id: "msg_1", delta: "Hello" },
|
||||
{ type: "response.output_text.delta", item_id: "msg_1", delta: "!" },
|
||||
{
|
||||
type: "response.completed",
|
||||
response: {
|
||||
id: "resp_1",
|
||||
service_tier: "default",
|
||||
usage: {
|
||||
input_tokens: 5,
|
||||
output_tokens: 2,
|
||||
total_tokens: 7,
|
||||
input_tokens_details: { cached_tokens: 1 },
|
||||
output_tokens_details: { reasoning_tokens: 0 },
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
const response = yield* LLMClient.generate(request).pipe(Effect.provide(fixedResponse(body)))
|
||||
|
||||
expect(response.text).toBe("Hello!")
|
||||
expect(response.events).toEqual([
|
||||
{ type: "text-delta", id: "msg_1", text: "Hello", providerMetadata: { openai: { itemId: "msg_1" } } },
|
||||
{ type: "text-delta", id: "msg_1", text: "!", providerMetadata: { openai: { itemId: "msg_1" } } },
|
||||
{
|
||||
type: "request-finish",
|
||||
reason: "stop",
|
||||
providerMetadata: { openai: { responseId: "resp_1", serviceTier: "default" } },
|
||||
usage: {
|
||||
inputTokens: 5,
|
||||
outputTokens: 2,
|
||||
reasoningTokens: 0,
|
||||
cacheReadInputTokens: 1,
|
||||
totalTokens: 7,
|
||||
native: {
|
||||
input_tokens: 5,
|
||||
output_tokens: 2,
|
||||
total_tokens: 7,
|
||||
input_tokens_details: { cached_tokens: 1 },
|
||||
output_tokens_details: { reasoning_tokens: 0 },
|
||||
},
|
||||
},
|
||||
},
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("assembles streamed function call input", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = sseEvents(
|
||||
{
|
||||
type: "response.output_item.added",
|
||||
item: { type: "function_call", id: "item_1", call_id: "call_1", name: "lookup", arguments: "" },
|
||||
},
|
||||
{ type: "response.function_call_arguments.delta", item_id: "item_1", delta: '{"query"' },
|
||||
{ type: "response.function_call_arguments.delta", item_id: "item_1", delta: ':"weather"}' },
|
||||
{
|
||||
type: "response.output_item.done",
|
||||
item: {
|
||||
type: "function_call",
|
||||
id: "item_1",
|
||||
call_id: "call_1",
|
||||
name: "lookup",
|
||||
arguments: '{"query":"weather"}',
|
||||
},
|
||||
},
|
||||
{ type: "response.completed", response: { usage: { input_tokens: 5, output_tokens: 1 } } },
|
||||
)
|
||||
const response = yield* LLMClient.generate(
|
||||
LLM.updateRequest(request, {
|
||||
tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }],
|
||||
}),
|
||||
).pipe(Effect.provide(fixedResponse(body)))
|
||||
|
||||
expect(response.events).toEqual([
|
||||
{
|
||||
type: "tool-input-delta",
|
||||
id: "call_1",
|
||||
name: "lookup",
|
||||
text: '{"query"',
|
||||
providerMetadata: { openai: { itemId: "item_1" } },
|
||||
},
|
||||
{
|
||||
type: "tool-input-delta",
|
||||
id: "call_1",
|
||||
name: "lookup",
|
||||
text: ':"weather"}',
|
||||
providerMetadata: { openai: { itemId: "item_1" } },
|
||||
},
|
||||
{
|
||||
type: "tool-call",
|
||||
id: "call_1",
|
||||
name: "lookup",
|
||||
input: { query: "weather" },
|
||||
providerMetadata: { openai: { itemId: "item_1" } },
|
||||
},
|
||||
{
|
||||
type: "request-finish",
|
||||
reason: "tool-calls",
|
||||
usage: { inputTokens: 5, outputTokens: 1, totalTokens: 6, native: { input_tokens: 5, output_tokens: 1 } },
|
||||
},
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("decodes web_search_call as provider-executed tool-call + tool-result", () =>
|
||||
Effect.gen(function* () {
|
||||
const item = {
|
||||
type: "web_search_call",
|
||||
id: "ws_1",
|
||||
status: "completed",
|
||||
action: { type: "search", query: "effect 4" },
|
||||
}
|
||||
const body = sseEvents(
|
||||
{ type: "response.output_item.added", item },
|
||||
{ type: "response.output_item.done", item },
|
||||
{ type: "response.completed", response: { usage: { input_tokens: 5, output_tokens: 1 } } },
|
||||
)
|
||||
const response = yield* LLMClient.generate(request).pipe(Effect.provide(fixedResponse(body)))
|
||||
|
||||
const callsAndResults = response.events.filter(
|
||||
(event) => event.type === "tool-call" || event.type === "tool-result",
|
||||
)
|
||||
expect(callsAndResults).toEqual([
|
||||
{
|
||||
type: "tool-call",
|
||||
id: "ws_1",
|
||||
name: "web_search",
|
||||
input: { type: "search", query: "effect 4" },
|
||||
providerExecuted: true,
|
||||
providerMetadata: { openai: { itemId: "ws_1" } },
|
||||
},
|
||||
{
|
||||
type: "tool-result",
|
||||
id: "ws_1",
|
||||
name: "web_search",
|
||||
result: { type: "json", value: item },
|
||||
providerExecuted: true,
|
||||
providerMetadata: { openai: { itemId: "ws_1" } },
|
||||
},
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("decodes code_interpreter_call as provider-executed events with code input", () =>
|
||||
Effect.gen(function* () {
|
||||
const item = {
|
||||
type: "code_interpreter_call",
|
||||
id: "ci_1",
|
||||
status: "completed",
|
||||
code: "print(1+1)",
|
||||
container_id: "cnt_xyz",
|
||||
outputs: [{ type: "logs", logs: "2\n" }],
|
||||
}
|
||||
const body = sseEvents(
|
||||
{ type: "response.output_item.done", item },
|
||||
{ type: "response.completed", response: { usage: { input_tokens: 5, output_tokens: 1 } } },
|
||||
)
|
||||
const response = yield* LLMClient.generate(request).pipe(Effect.provide(fixedResponse(body)))
|
||||
|
||||
const toolCall = response.events.find((event) => event.type === "tool-call")
|
||||
expect(toolCall).toEqual({
|
||||
type: "tool-call",
|
||||
id: "ci_1",
|
||||
name: "code_interpreter",
|
||||
input: { code: "print(1+1)", container_id: "cnt_xyz" },
|
||||
providerExecuted: true,
|
||||
providerMetadata: { openai: { itemId: "ci_1" } },
|
||||
})
|
||||
const toolResult = response.events.find((event) => event.type === "tool-result")
|
||||
expect(toolResult).toEqual({
|
||||
type: "tool-result",
|
||||
id: "ci_1",
|
||||
name: "code_interpreter",
|
||||
result: { type: "json", value: item },
|
||||
providerExecuted: true,
|
||||
providerMetadata: { openai: { itemId: "ci_1" } },
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects unsupported user media content", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
id: "req_media",
|
||||
model,
|
||||
messages: [LLM.user({ type: "media", mediaType: "image/png", data: "AAECAw==" })],
|
||||
}),
|
||||
).pipe(Effect.flip)
|
||||
|
||||
expect(error.message).toContain("OpenAI Responses user messages only support text content for now")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("emits provider-error events for mid-stream provider errors", () =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* LLMClient.generate(request).pipe(
|
||||
Effect.provide(fixedResponse(sseEvents({ type: "error", code: "rate_limit_exceeded", message: "Slow down" }))),
|
||||
)
|
||||
|
||||
expect(response.events).toEqual([{ type: "provider-error", message: "Slow down" }])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("falls back to error code when no message is present", () =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* LLMClient.generate(request).pipe(
|
||||
Effect.provide(fixedResponse(sseEvents({ type: "error", code: "internal_error" }))),
|
||||
)
|
||||
|
||||
expect(response.events).toEqual([{ type: "provider-error", message: "internal_error" }])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("fails HTTP provider errors before stream parsing", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* LLMClient.generate(request).pipe(
|
||||
Effect.provide(
|
||||
fixedResponse('{"error":{"type":"invalid_request_error","message":"Bad request"}}', {
|
||||
status: 400,
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
),
|
||||
Effect.flip,
|
||||
)
|
||||
|
||||
expect(error).toBeInstanceOf(LLMError)
|
||||
expect(error.reason).toMatchObject({ _tag: "InvalidRequest" })
|
||||
expect(error.message).toContain("HTTP 400")
|
||||
}),
|
||||
)
|
||||
})
|
||||
Reference in New Issue
Block a user