lmin/zmo-cli
1
0
Fork 0
Code Issues Pull Requests Actions 82 Packages Projects Releases Wiki Activity

Refactor LLM route-first provider API (#28523)

Browse Source
This commit is contained in:
Kit Langton
2026-05-20 20:15:52 -04:00
committed by GitHub
parent 5381795844
commit 41f6daf96a
87 changed files with 2450 additions and 1520 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/llm/test/provider/anthropic-messages-cache.recorded.test.ts
+3 -4
View File
@@ -3,14 +3,13 @@ import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { CacheHint, LLM } from "../../src"
import { LLMClient } from "../../src/route"
import * as AnthropicMessages from "../../src/protocols/anthropic-messages"
import * as Anthropic from "../../src/providers/anthropic"
import { LARGE_CACHEABLE_SYSTEM } from "../recorded-scenarios"
import { recordedTests } from "../recorded-test"
const model = AnthropicMessages.model({
id: "claude-haiku-4-5-20251001",
const model = Anthropic.configure({
apiKey: process.env.ANTHROPIC_API_KEY ?? "fixture",
})
}).model("claude-haiku-4-5-20251001")
// Two identical generations in a row. The first call writes the prefix into
// Anthropic's cache; the second should report a cache read against the same
packages/llm/test/provider/anthropic-messages.recorded.test.ts
+3 -4
View File
@@ -3,14 +3,13 @@ import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { LLM, LLMError, Message, ToolCallPart } from "../../src"
import { LLMClient } from "../../src/route"
import * as AnthropicMessages from "../../src/protocols/anthropic-messages"
import * as Anthropic from "../../src/providers/anthropic"
import { weatherToolName } from "../recorded-scenarios"
import { recordedTests } from "../recorded-test"
const model = AnthropicMessages.model({
id: "claude-haiku-4-5-20251001",
const model = Anthropic.configure({
apiKey: process.env.ANTHROPIC_API_KEY ?? "fixture",
})
}).model("claude-haiku-4-5-20251001")
const malformedToolOrderRequest = LLM.request({
id: "recorded_anthropic_malformed_tool_order",
packages/llm/test/provider/anthropic-messages.test.ts
+4 -6
View File
@@ -1,17 +1,15 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { CacheHint, LLM, LLMError, Message, ToolCallPart, Usage } from "../../src"
import { LLMClient } from "../../src/route"
import { Auth, LLMClient } from "../../src/route"
import * as AnthropicMessages from "../../src/protocols/anthropic-messages"
import { it } from "../lib/effect"
import { fixedResponse } from "../lib/http"
import { sseEvents } from "../lib/sse"
const model = AnthropicMessages.model({
id: "claude-sonnet-4-5",
baseURL: "https://api.anthropic.test/v1/",
headers: { "x-api-key": "test" },
})
const model = AnthropicMessages.route
.with({ endpoint: { baseURL: "https://api.anthropic.test/v1/" }, auth: Auth.header("x-api-key", "test") })
.model({ id: "claude-sonnet-4-5" })
const request = LLM.request({
id: "req_1",
packages/llm/test/provider/bedrock-converse-cache.recorded.test.ts
+3 -4
View File
@@ -2,7 +2,7 @@ import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { CacheHint, LLM } from "../../src"
import { LLMClient } from "../../src/route"
import * as BedrockConverse from "../../src/protocols/bedrock-converse"
import { AmazonBedrock } from "../../src/providers"
import { LARGE_CACHEABLE_SYSTEM } from "../recorded-scenarios"
import { recordedTests } from "../recorded-test"
@@ -12,15 +12,14 @@ const RECORDING_REGION = process.env.BEDROCK_RECORDING_REGION ?? "us-east-1"
// doesn't reliably surface `cacheRead`/`cacheWrite` in usage, so the second
// call wouldn't deterministically prove cache mapping works. Override with
// BEDROCK_CACHE_MODEL_ID if your account has access elsewhere.
const model = BedrockConverse.model({
id: process.env.BEDROCK_CACHE_MODEL_ID ?? "us.anthropic.claude-haiku-4-5-20251001-v1:0",
const model = AmazonBedrock.configure({
credentials: {
region: RECORDING_REGION,
accessKeyId: process.env.AWS_ACCESS_KEY_ID ?? "fixture",
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY ?? "fixture",
sessionToken: process.env.AWS_SESSION_TOKEN,
},
})
}).model(process.env.BEDROCK_CACHE_MODEL_ID ?? "us.anthropic.claude-haiku-4-5-20251001-v1:0")
const cacheRequest = LLM.request({
id: "recorded_bedrock_cache",
packages/llm/test/provider/bedrock-converse.test.ts
+60 -20
View File
@@ -4,6 +4,7 @@ import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { CacheHint, LLM, Message, ToolCallPart, ToolChoice } from "../../src"
import { LLMClient } from "../../src/route"
import { AmazonBedrock } from "../../src/providers"
import * as BedrockConverse from "../../src/protocols/bedrock-converse"
import { it } from "../lib/effect"
import { fixedResponse } from "../lib/http"
@@ -52,11 +53,10 @@ const eventStreamBody = (...payloads: ReadonlyArray<readonly [string, object]>)
const fixedBytes = (bytes: Uint8Array) =>
fixedResponse(bytes.slice().buffer, { headers: { "content-type": "application/vnd.amazon.eventstream" } })
const model = BedrockConverse.model({
id: "anthropic.claude-3-5-sonnet-20240620-v1:0",
const model = AmazonBedrock.configure({
baseURL: "https://bedrock-runtime.test",
apiKey: "test-bearer",
})
}).model("anthropic.claude-3-5-sonnet-20240620-v1:0")
const baseRequest = LLM.request({
id: "req_1",
@@ -156,6 +156,55 @@ describe("Bedrock Converse route", () => {
}),
)
it.effect("lowers image content in tool-result messages", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare(
LLM.request({
id: "req_tool_image",
model,
messages: [
Message.user("Capture the screen."),
Message.assistant([ToolCallPart.make({ id: "tool_1", name: "screenshot", input: {} })]),
Message.tool({
id: "tool_1",
name: "screenshot",
result: {
type: "content",
value: [
{ type: "text", text: "Screenshot captured." },
{ type: "media", mediaType: "image/png", data: "AAAA" },
],
},
}),
],
cache: "none",
}),
)
expect(prepared.body).toMatchObject({
messages: [
{ role: "user", content: [{ text: "Capture the screen." }] },
{
role: "assistant",
content: [{ toolUse: { toolUseId: "tool_1", name: "screenshot", input: {} } }],
},
{
role: "user",
content: [
{
toolResult: {
toolUseId: "tool_1",
content: [{ text: "Screenshot captured." }, { image: { format: "png", source: { bytes: "AAAA" } } }],
status: "success",
},
},
],
},
],
})
}),
)
it.effect("decodes text-delta + messageStop + metadata usage from binary event stream", () =>
Effect.gen(function* () {
const body = eventStreamBody(
@@ -249,39 +298,32 @@ describe("Bedrock Converse route", () => {
it.effect("rejects requests with no auth path", () =>
Effect.gen(function* () {
const unsignedModel = BedrockConverse.model({
id: "anthropic.claude-3-5-sonnet-20240620-v1:0",
const unsignedModel = AmazonBedrock.configure({
baseURL: "https://bedrock-runtime.test",
})
}).model("anthropic.claude-3-5-sonnet-20240620-v1:0")
const error = yield* LLMClient.generate(LLM.updateRequest(baseRequest, { model: unsignedModel })).pipe(
Effect.provide(fixedBytes(eventStreamBody(["messageStop", { stopReason: "end_turn" }]))),
Effect.flip,
)
expect(error.message).toContain("Bedrock Converse requires either model.apiKey")
expect(error.message).toContain("Bedrock Converse requires either route bearer auth or AWS credentials")
}),
)
it.effect("signs requests with SigV4 when AWS credentials are provided (deterministic plumbing check)", () =>
Effect.gen(function* () {
const signed = BedrockConverse.model({
id: "anthropic.claude-3-5-sonnet-20240620-v1:0",
const signed = AmazonBedrock.configure({
baseURL: "https://bedrock-runtime.test",
credentials: {
region: "us-east-1",
accessKeyId: "AKIAIOSFODNN7EXAMPLE",
secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
},
})
}).model("anthropic.claude-3-5-sonnet-20240620-v1:0")
const prepared = yield* LLMClient.prepare(LLM.updateRequest(baseRequest, { model: signed }))
expect(prepared.route).toBe("bedrock-converse")
// The prepare phase doesn't sign — toHttp does. We assert the credential
// is plumbed onto the model native field for the signer to find.
expect(prepared.model.native).toMatchObject({
aws_credentials: { region: "us-east-1", accessKeyId: "AKIAIOSFODNN7EXAMPLE" },
aws_region: "us-east-1",
})
expect(prepared.model).toBe(signed)
}),
)
@@ -531,18 +573,17 @@ describe("Bedrock Converse route", () => {
const RECORDING_REGION = process.env.BEDROCK_RECORDING_REGION ?? "us-east-1"
const recordedModel = () =>
BedrockConverse.model({
AmazonBedrock.configure({
// Most newer Anthropic models on Bedrock require a cross-region inference
// profile (`us.` prefix). Nova does not require an Anthropic use-case form
// and is on-demand-throughput accessible by default for most accounts.
id: process.env.BEDROCK_MODEL_ID ?? "us.amazon.nova-micro-v1:0",
credentials: {
region: RECORDING_REGION,
accessKeyId: process.env.AWS_ACCESS_KEY_ID ?? "fixture",
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY ?? "fixture",
sessionToken: process.env.AWS_SESSION_TOKEN,
},
})
}).model(process.env.BEDROCK_MODEL_ID ?? "us.amazon.nova-micro-v1:0")
const recorded = recordedTests({
prefix: "bedrock-converse",
@@ -598,7 +639,6 @@ describe("Bedrock Converse recorded", () => {
recorded.effect.with("drives a tool loop", { tags: ["tool", "tool-loop", "golden"] }, () =>
Effect.gen(function* () {
const llm = yield* LLMClient.Service
expectWeatherToolLoop(
yield* runWeatherToolLoop(
weatherToolLoopRequest({
packages/llm/test/provider/cloudflare.test.ts
+22 -22
View File
@@ -2,7 +2,7 @@ import { describe, expect } from "bun:test"
import { ConfigProvider, Effect, Schema } from "effect"
import { HttpClientRequest } from "effect/unstable/http"
import { LLM } from "../../src"
import * as Cloudflare from "../../src/providers/cloudflare"
import { CloudflareAIGateway, CloudflareWorkersAI } from "../../src/providers/cloudflare"
import { LLMClient } from "../../src/route"
import { it } from "../lib/effect"
import { dynamicResponse } from "../lib/http"
@@ -21,18 +21,18 @@ const deltaChunk = (delta: object, finishReason: string | null = null) => ({
describe("Cloudflare", () => {
it.effect("prepares AI Gateway models through the OpenAI-compatible Chat protocol", () =>
Effect.gen(function* () {
const model = Cloudflare.aiGateway("workers-ai/@cf/meta/llama-3.3-70b-instruct", {
const model = CloudflareAIGateway.configure({
accountId: "test-account",
gatewayId: "test-gateway",
apiKey: "test-token",
})
}).model("workers-ai/@cf/meta/llama-3.3-70b-instruct")
expect(model).toMatchObject({
id: "workers-ai/@cf/meta/llama-3.3-70b-instruct",
provider: "cloudflare-ai-gateway",
route: "cloudflare-ai-gateway",
baseURL: "https://gateway.ai.cloudflare.com/v1/test-account/test-gateway/compat",
route: { id: "cloudflare-ai-gateway" },
})
expect(model.route.endpoint.baseURL).toBe("https://gateway.ai.cloudflare.com/v1/test-account/test-gateway/compat")
const prepared = yield* LLMClient.prepare(LLM.request({ model, prompt: "Say hello." }))
@@ -49,11 +49,11 @@ describe("Cloudflare", () => {
Effect.gen(function* () {
const response = yield* LLM.generate(
LLM.request({
model: Cloudflare.aiGateway("openai/gpt-4o-mini", {
model: CloudflareAIGateway.configure({
accountId: "test-account",
gatewayId: "test-gateway",
apiKey: "test-token",
}),
}).model("openai/gpt-4o-mini"),
prompt: "Say hello.",
}),
).pipe(
@@ -86,11 +86,11 @@ describe("Cloudflare", () => {
it.effect("defaults AI Gateway id to default when omitted or blank", () =>
Effect.gen(function* () {
expect(
Cloudflare.aiGateway("workers-ai/@cf/meta/llama-3.3-70b-instruct", {
CloudflareAIGateway.configure({
accountId: "test-account",
gatewayId: "",
gatewayApiKey: "test-token",
}).baseURL,
}).model("workers-ai/@cf/meta/llama-3.3-70b-instruct").route.endpoint.baseURL,
).toBe("https://gateway.ai.cloudflare.com/v1/test-account/default/compat")
}),
)
@@ -99,11 +99,11 @@ describe("Cloudflare", () => {
Effect.gen(function* () {
yield* LLM.generate(
LLM.request({
model: Cloudflare.aiGateway("openai/gpt-4o-mini", {
model: CloudflareAIGateway.configure({
accountId: "test-account",
gatewayApiKey: "gateway-token",
apiKey: "provider-token",
}),
}).model("openai/gpt-4o-mini"),
prompt: "Say hello.",
}),
).pipe(
@@ -129,31 +129,31 @@ describe("Cloudflare", () => {
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare(
LLM.request({
model: Cloudflare.aiGateway("openai/gpt-4o-mini", {
model: CloudflareAIGateway.configure({
baseURL: "https://gateway.proxy.test/v1/custom/compat",
apiKey: "test-token",
}),
}).model("openai/gpt-4o-mini"),
prompt: "Say hello.",
}),
)
expect(prepared.model.baseURL).toBe("https://gateway.proxy.test/v1/custom/compat")
expect(prepared.model.route.endpoint.baseURL).toBe("https://gateway.proxy.test/v1/custom/compat")
}),
)
it.effect("prepares direct Workers AI models through the OpenAI-compatible Chat protocol", () =>
Effect.gen(function* () {
const model = Cloudflare.workersAI("@cf/meta/llama-3.1-8b-instruct", {
const model = CloudflareWorkersAI.configure({
accountId: "test-account",
apiKey: "test-token",
})
}).model("@cf/meta/llama-3.1-8b-instruct")
expect(model).toMatchObject({
id: "@cf/meta/llama-3.1-8b-instruct",
provider: "cloudflare-workers-ai",
route: "cloudflare-workers-ai",
baseURL: "https://api.cloudflare.com/client/v4/accounts/test-account/ai/v1",
route: { id: "cloudflare-workers-ai" },
})
expect(model.route.endpoint.baseURL).toBe("https://api.cloudflare.com/client/v4/accounts/test-account/ai/v1")
const prepared = yield* LLMClient.prepare(LLM.request({ model, prompt: "Say hello." }))
@@ -170,10 +170,10 @@ describe("Cloudflare", () => {
Effect.gen(function* () {
const response = yield* LLM.generate(
LLM.request({
model: Cloudflare.workersAI("@cf/meta/llama-3.1-8b-instruct", {
model: CloudflareWorkersAI.configure({
accountId: "test-account",
apiKey: "test-token",
}),
}).model("@cf/meta/llama-3.1-8b-instruct"),
prompt: "Say hello.",
}),
).pipe(
@@ -205,9 +205,9 @@ describe("Cloudflare", () => {
Effect.gen(function* () {
yield* LLM.generate(
LLM.request({
model: Cloudflare.workersAI("@cf/meta/llama-3.1-8b-instruct", {
model: CloudflareWorkersAI.configure({
accountId: "test-account",
}),
}).model("@cf/meta/llama-3.1-8b-instruct"),
prompt: "Say hello.",
}),
).pipe(
packages/llm/test/provider/gemini-cache.recorded.test.ts
+3 -4
View File
@@ -2,14 +2,13 @@ import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { LLM } from "../../src"
import { LLMClient } from "../../src/route"
import * as Gemini from "../../src/protocols/gemini"
import * as Google from "../../src/providers/google"
import { LARGE_CACHEABLE_SYSTEM } from "../recorded-scenarios"
import { recordedTests } from "../recorded-test"
const model = Gemini.model({
id: "gemini-2.5-flash",
const model = Google.configure({
apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY ?? process.env.GEMINI_API_KEY ?? "fixture",
})
}).model("gemini-2.5-flash")
// Gemini does implicit prefix caching on 2.5+ models above ~1024 tokens. The
// `CacheHint` is currently a no-op for Gemini (the explicit `CachedContent`
packages/llm/test/provider/gemini.test.ts
+7 -6
View File
@@ -1,17 +1,18 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { LLM, LLMError, Message, ToolCallPart, Usage } from "../../src"
import { LLMClient } from "../../src/route"
import { Auth, LLMClient } from "../../src/route"
import * as Gemini from "../../src/protocols/gemini"
import { it } from "../lib/effect"
import { fixedResponse } from "../lib/http"
import { sseEvents, sseRaw } from "../lib/sse"
const model = Gemini.model({
id: "gemini-2.5-flash",
baseURL: "https://generativelanguage.test/v1beta/",
headers: { "x-goog-api-key": "test" },
})
const model = Gemini.route
.with({
endpoint: { baseURL: "https://generativelanguage.test/v1beta/" },
auth: Auth.header("x-goog-api-key", "test"),
})
.model({ id: "gemini-2.5-flash" })
const request = LLM.request({
id: "req_1",
packages/llm/test/provider/golden.recorded.test.ts
+38 -41
View File
@@ -1,32 +1,30 @@
import { Redactor } from "@opencode-ai/http-recorder"
import * as AnthropicMessages from "../../src/protocols/anthropic-messages"
import * as Gemini from "../../src/protocols/gemini"
import * as OpenAIChat from "../../src/protocols/openai-chat"
import * as OpenAIResponses from "../../src/protocols/openai-responses"
import * as Cloudflare from "../../src/providers/cloudflare"
import * as Anthropic from "../../src/providers/anthropic"
import { CloudflareAIGateway, CloudflareWorkersAI } from "../../src/providers/cloudflare"
import * as Google from "../../src/providers/google"
import * as OpenAI from "../../src/providers/openai"
import * as OpenAICompatible from "../../src/providers/openai-compatible"
import * as OpenRouter from "../../src/providers/openrouter"
import * as XAI from "../../src/providers/xai"
import { describeRecordedGoldenScenarios } from "../recorded-golden"
const openAIChat = OpenAIChat.model({ id: "gpt-4o-mini", apiKey: process.env.OPENAI_API_KEY ?? "fixture" })
const openAIResponses = OpenAIResponses.model({ id: "gpt-5.5", apiKey: process.env.OPENAI_API_KEY ?? "fixture" })
const openAIResponsesWebSocket = OpenAI.responsesWebSocket("gpt-4.1-mini", {
const openAI = OpenAI.configure({
apiKey: process.env.OPENAI_API_KEY ?? "fixture",
})
const anthropicHaiku = AnthropicMessages.model({
id: "claude-haiku-4-5-20251001",
const openAIChat = openAI.chat("gpt-4o-mini")
const openAIResponses = openAI.responses("gpt-5.5")
const openAIResponsesWebSocket = openAI.responsesWebSocket("gpt-4.1-mini")
const anthropic = Anthropic.configure({
apiKey: process.env.ANTHROPIC_API_KEY ?? "fixture",
})
const anthropicOpus = AnthropicMessages.model({
id: "claude-opus-4-7",
apiKey: process.env.ANTHROPIC_API_KEY ?? "fixture",
})
const gemini = Gemini.model({ id: "gemini-2.5-flash", apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY ?? "fixture" })
const xaiBasic = XAI.model("grok-3-mini", { apiKey: process.env.XAI_API_KEY ?? "fixture" })
const xaiFlagship = XAI.model("grok-4.3", { apiKey: process.env.XAI_API_KEY ?? "fixture" })
const cloudflareAIGatewayWorkers = Cloudflare.aiGateway("workers-ai/@cf/meta/llama-3.1-8b-instruct", {
const anthropicHaiku = anthropic.model("claude-haiku-4-5-20251001")
const anthropicOpus = anthropic.model("claude-opus-4-7")
const google = Google.configure({ apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY ?? "fixture" })
const gemini = google.model("gemini-2.5-flash")
const xai = XAI.configure({ apiKey: process.env.XAI_API_KEY ?? "fixture" })
const xaiBasic = xai.model("grok-3-mini")
const xaiFlagship = xai.model("grok-4.3")
const cloudflareAIGateway = CloudflareAIGateway.configure({
accountId: process.env.CLOUDFLARE_ACCOUNT_ID ?? "fixture-account",
gatewayId:
process.env.CLOUDFLARE_GATEWAY_ID && process.env.CLOUDFLARE_GATEWAY_ID !== process.env.CLOUDFLARE_ACCOUNT_ID
@@ -34,32 +32,31 @@ const cloudflareAIGatewayWorkers = Cloudflare.aiGateway("workers-ai/@cf/meta/lla
: undefined,
gatewayApiKey: process.env.CLOUDFLARE_API_TOKEN ?? "fixture",
})
const cloudflareAIGatewayWorkersTools = Cloudflare.aiGateway("workers-ai/@cf/openai/gpt-oss-20b", {
accountId: process.env.CLOUDFLARE_ACCOUNT_ID ?? "fixture-account",
gatewayId:
process.env.CLOUDFLARE_GATEWAY_ID && process.env.CLOUDFLARE_GATEWAY_ID !== process.env.CLOUDFLARE_ACCOUNT_ID
? process.env.CLOUDFLARE_GATEWAY_ID
: undefined,
gatewayApiKey: process.env.CLOUDFLARE_API_TOKEN ?? "fixture",
})
const cloudflareWorkersAI = Cloudflare.workersAI("@cf/meta/llama-3.1-8b-instruct", {
const cloudflareWorkers = CloudflareWorkersAI.configure({
accountId: process.env.CLOUDFLARE_ACCOUNT_ID ?? "fixture-account",
apiKey: process.env.CLOUDFLARE_API_KEY ?? "fixture",
})
const cloudflareWorkersAITools = Cloudflare.workersAI("@cf/openai/gpt-oss-20b", {
accountId: process.env.CLOUDFLARE_ACCOUNT_ID ?? "fixture-account",
apiKey: process.env.CLOUDFLARE_API_KEY ?? "fixture",
})
const deepseek = OpenAICompatible.deepseek.model("deepseek-chat", { apiKey: process.env.DEEPSEEK_API_KEY ?? "fixture" })
const together = OpenAICompatible.togetherai.model("meta-llama/Llama-3.3-70B-Instruct-Turbo", {
apiKey: process.env.TOGETHER_AI_API_KEY ?? "fixture",
})
const groq = OpenAICompatible.groq.model("llama-3.3-70b-versatile", { apiKey: process.env.GROQ_API_KEY ?? "fixture" })
const openrouter = OpenRouter.model("openai/gpt-4o-mini", { apiKey: process.env.OPENROUTER_API_KEY ?? "fixture" })
const openrouterGpt55 = OpenRouter.model("openai/gpt-5.5", { apiKey: process.env.OPENROUTER_API_KEY ?? "fixture" })
const openrouterOpus = OpenRouter.model("anthropic/claude-opus-4.7", {
const cloudflareAIGatewayWorkers = cloudflareAIGateway.model("workers-ai/@cf/meta/llama-3.1-8b-instruct")
const cloudflareAIGatewayWorkersTools = cloudflareAIGateway.model("workers-ai/@cf/openai/gpt-oss-20b")
const cloudflareWorkersAI = cloudflareWorkers.model("@cf/meta/llama-3.1-8b-instruct")
const cloudflareWorkersAITools = cloudflareWorkers.model("@cf/openai/gpt-oss-20b")
const deepseek = OpenAICompatible.deepseek
.configure({ apiKey: process.env.DEEPSEEK_API_KEY ?? "fixture" })
.model("deepseek-chat")
const together = OpenAICompatible.togetherai
.configure({
apiKey: process.env.TOGETHER_AI_API_KEY ?? "fixture",
})
.model("meta-llama/Llama-3.3-70B-Instruct-Turbo")
const groq = OpenAICompatible.groq
.configure({ apiKey: process.env.GROQ_API_KEY ?? "fixture" })
.model("llama-3.3-70b-versatile")
const openRouter = OpenRouter.configure({ apiKey: process.env.OPENROUTER_API_KEY ?? "fixture" })
const openrouter = openRouter.model("openai/gpt-4o-mini")
const openrouterGpt55 = openRouter.model("openai/gpt-5.5")
const openrouterOpus = OpenRouter.configure({
apiKey: process.env.OPENROUTER_API_KEY ?? "fixture",
})
}).model("anthropic/claude-opus-4.7")
const redactCloudflareURL = (url: string) =>
url
@@ -120,7 +117,7 @@ describeRecordedGoldenScenarios([
prefix: "gemini",
model: gemini,
requires: ["GOOGLE_GENERATIVE_AI_API_KEY"],
scenarios: [{ id: "text", maxTokens: 80 }, "tool-call"],
scenarios: [{ id: "text", maxTokens: 80 }, "tool-call", { id: "image", maxTokens: 160 }],
},
{
name: "xAI Grok 3 Mini",
packages/llm/test/provider/openai-chat.test.ts
+15 -12
View File
@@ -1,11 +1,11 @@
import { describe, expect } from "bun:test"
import { Effect, Schema, Stream } from "effect"
import { HttpClientRequest } from "effect/unstable/http"
import { LLM, LLMError, Message, ToolCallPart, Usage } from "../../src"
import { LLM, LLMError, Message, Model, ToolCallPart, Usage } from "../../src"
import * as Azure from "../../src/providers/azure"
import * as OpenAI from "../../src/providers/openai"
import * as OpenAIChat from "../../src/protocols/openai-chat"
import { LLMClient } from "../../src/route"
import { Auth, LLMClient } from "../../src/route"
import { it } from "../lib/effect"
import { dynamicResponse, fixedResponse, truncatedStream } from "../lib/http"
import { deltaChunk, usageChunk } from "../lib/openai-chunks"
@@ -15,11 +15,9 @@ const TargetJson = Schema.fromJsonString(Schema.Unknown)
const encodeJson = Schema.encodeSync(TargetJson)
const decodeJson = Schema.decodeUnknownSync(TargetJson)
const model = OpenAIChat.model({
id: "gpt-4o-mini",
baseURL: "https://api.openai.test/v1/",
headers: { authorization: "Bearer test" },
})
const model = OpenAIChat.route
.with({ endpoint: { baseURL: "https://api.openai.test/v1/" }, auth: Auth.bearer("test") })
.model({ id: "gpt-4o-mini" })
const request = LLM.request({
id: "req_1",
@@ -56,7 +54,7 @@ describe("OpenAI Chat route", () => {
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIChat.OpenAIChatBody>(
LLM.request({
model: OpenAI.chat("gpt-4o-mini", { baseURL: "https://api.openai.test/v1/" }),
model: OpenAI.configure({ baseURL: "https://api.openai.test/v1/", apiKey: "test" }).chat("gpt-4o-mini"),
prompt: "think",
providerOptions: { openai: { reasoningEffort: "low" } },
}),
@@ -69,7 +67,9 @@ describe("OpenAI Chat route", () => {
it.effect("adds native query params to the Chat Completions URL", () =>
LLMClient.generate(
LLM.updateRequest(request, { model: OpenAIChat.model({ ...model, queryParams: { "api-version": "v1" } }) }),
LLM.updateRequest(request, {
model: Model.update(model, { route: model.route.with({ endpoint: { query: { "api-version": "v1" } } }) }),
}),
).pipe(
Effect.provide(
dynamicResponse((input) =>
@@ -88,17 +88,18 @@ describe("OpenAI Chat route", () => {
it.effect("uses Azure api-key header for static OpenAI Chat keys", () =>
LLMClient.generate(
LLM.updateRequest(request, {
model: Azure.chat("gpt-4o-mini", {
model: Azure.configure({
baseURL: "https://opencode-test.openai.azure.com/openai/v1/",
apiKey: "azure-key",
headers: { authorization: "Bearer stale" },
}),
}).chat("gpt-4o-mini"),
}),
).pipe(
Effect.provide(
dynamicResponse((input) =>
Effect.gen(function* () {
const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
expect(web.url).toBe("https://opencode-test.openai.azure.com/openai/v1/chat/completions?api-version=v1")
expect(web.headers.get("api-key")).toBe("azure-key")
expect(web.headers.get("authorization")).toBeNull()
return input.respond(sseEvents(deltaChunk({}, "stop")), {
@@ -113,7 +114,9 @@ describe("OpenAI Chat route", () => {
it.effect("applies serializable HTTP overlays after payload lowering", () =>
LLMClient.generate(
LLM.updateRequest(request, {
model: OpenAIChat.model({ ...model, apiKey: "fresh-key", headers: { authorization: "Bearer stale" } }),
model: model.route
.with({ auth: Auth.bearer("fresh-key"), headers: { authorization: "Bearer stale" } })
.model({ id: model.id }),
http: {
body: { metadata: { source: "test" } },
headers: { authorization: "Bearer request", "x-custom": "yes" },
packages/llm/test/provider/openai-compatible-chat.test.ts
+23 -22
View File
@@ -2,7 +2,7 @@ import { describe, expect } from "bun:test"
import { Effect, Schema } from "effect"
import { HttpClientRequest } from "effect/unstable/http"
import { LLM, Message, ToolCallPart } from "../../src"
import { LLMClient } from "../../src/route"
import { Auth, LLMClient } from "../../src/route"
import * as OpenAICompatible from "../../src/providers/openai-compatible"
import * as OpenAICompatibleChat from "../../src/protocols/openai-compatible-chat"
import { it } from "../lib/effect"
@@ -12,13 +12,13 @@ import { sseEvents } from "../lib/sse"
const Json = Schema.fromJsonString(Schema.Unknown)
const decodeJson = Schema.decodeUnknownSync(Json)
const model = OpenAICompatibleChat.model({
id: "deepseek-chat",
provider: "deepseek",
baseURL: "https://api.deepseek.test/v1/",
apiKey: "test-key",
queryParams: { "api-version": "2026-01-01" },
})
const model = OpenAICompatibleChat.route
.with({
provider: "deepseek",
endpoint: { baseURL: "https://api.deepseek.test/v1/", query: { "api-version": "2026-01-01" } },
auth: Auth.bearer("test-key"),
})
.model({ id: "deepseek-chat" })
const request = LLM.request({
id: "req_1",
@@ -63,10 +63,11 @@ describe("OpenAI-compatible Chat route", () => {
expect(prepared.model).toMatchObject({
id: "deepseek-chat",
provider: "deepseek",
route: "openai-compatible-chat",
route: { id: "openai-compatible-chat" },
})
expect(prepared.model.route.endpoint).toMatchObject({
baseURL: "https://api.deepseek.test/v1/",
apiKey: "test-key",
queryParams: { "api-version": "2026-01-01" },
query: { "api-version": "2026-01-01" },
})
expect(prepared.body).toEqual({
model: "deepseek-chat",
@@ -93,13 +94,12 @@ describe("OpenAI-compatible Chat route", () => {
Effect.gen(function* () {
expect(
providerFamilies.map(([provider, family]) => {
const model = family.model(`${provider}-model`, { apiKey: "test-key" })
const model = family.configure({ apiKey: "test-key" }).model(`${provider}-model`)
return {
id: String(model.id),
provider: String(model.provider),
route: model.route,
baseURL: model.baseURL,
apiKey: model.apiKey,
route: model.route.id,
baseURL: model.route.endpoint.baseURL,
}
}),
).toEqual(
@@ -108,19 +108,20 @@ describe("OpenAI-compatible Chat route", () => {
provider,
route: "openai-compatible-chat",
baseURL,
apiKey: "test-key",
})),
)
const custom = OpenAICompatible.deepseek.model("deepseek-chat", {
apiKey: "test-key",
baseURL: "https://custom.deepseek.test/v1",
})
const custom = OpenAICompatible.deepseek
.configure({
apiKey: "test-key",
baseURL: "https://custom.deepseek.test/v1",
})
.model("deepseek-chat")
expect(custom).toMatchObject({
provider: "deepseek",
route: "openai-compatible-chat",
baseURL: "https://custom.deepseek.test/v1",
route: { id: "openai-compatible-chat" },
})
expect(custom.route.endpoint.baseURL).toBe("https://custom.deepseek.test/v1")
}),
)
packages/llm/test/provider/openai-responses-cache.recorded.test.ts
+3 -4
View File
@@ -2,14 +2,13 @@ import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { LLM } from "../../src"
import { LLMClient } from "../../src/route"
import * as OpenAIResponses from "../../src/protocols/openai-responses"
import * as OpenAI from "../../src/providers/openai"
import { LARGE_CACHEABLE_SYSTEM } from "../recorded-scenarios"
import { recordedTests } from "../recorded-test"
const model = OpenAIResponses.model({
id: "gpt-4.1-mini",
const model = OpenAI.configure({
apiKey: process.env.OPENAI_API_KEY ?? "fixture",
})
}).responses("gpt-4.1-mini")
// OpenAI caches prefixes automatically once they cross the 1024-token threshold;
// `CacheHint` is a no-op for the wire body. The stable signal is the
packages/llm/test/provider/openai-responses.test.ts
+44 -48
View File
@@ -1,7 +1,7 @@
import { describe, expect } from "bun:test"
import { ConfigProvider, Effect, Layer, Stream } from "effect"
import { Headers, HttpClientRequest } from "effect/unstable/http"
import { LLM, LLMError, Message, ToolCallPart, Usage } from "../../src"
import { LLM, LLMError, Message, Model, ToolCallPart, Usage } from "../../src"
import { Auth, LLMClient, RequestExecutor, WebSocketExecutor } from "../../src/route"
import * as Azure from "../../src/providers/azure"
import * as OpenAI from "../../src/providers/openai"
@@ -11,11 +11,9 @@ import { it } from "../lib/effect"
import { dynamicResponse, fixedResponse } from "../lib/http"
import { sseEvents } from "../lib/sse"
const model = OpenAIResponses.model({
id: "gpt-4.1-mini",
baseURL: "https://api.openai.test/v1/",
headers: { authorization: "Bearer test" },
})
const model = OpenAIResponses.route
.with({ endpoint: { baseURL: "https://api.openai.test/v1/" }, auth: Auth.bearer("test") })
.model({ id: "gpt-4.1-mini" })
const request = LLM.request({
id: "req_1",
@@ -49,7 +47,9 @@ describe("OpenAI Responses route", () => {
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare(
LLM.updateRequest(request, {
model: OpenAI.responsesWebSocket("gpt-4.1-mini", { baseURL: "https://api.openai.test/v1/", apiKey: "test" }),
model: OpenAI.configure({ baseURL: "https://api.openai.test/v1/", apiKey: "test" }).responsesWebSocket(
"gpt-4.1-mini",
),
}),
)
@@ -95,10 +95,12 @@ describe("OpenAI Responses route", () => {
)
const response = yield* LLMClient.generate(
LLM.request({
model: OpenAI.responsesWebSocket("gpt-4.1-mini", { baseURL: "https://api.openai.test/v1/", apiKey: "test" }),
model: OpenAI.configure({ baseURL: "https://api.openai.test/v1/", apiKey: "test" }).responsesWebSocket(
"gpt-4.1-mini",
),
prompt: "Say hello.",
}),
).pipe(Effect.provide(LLMClient.layerWithWebSocket.pipe(Layer.provide(deps))))
).pipe(Effect.provide(LLMClient.layer.pipe(Layer.provide(deps))))
expect(response.text).toBe("Hi")
expect(opened).toEqual([{ url: "wss://api.openai.test/v1/responses", authorization: "Bearer test" }])
@@ -113,33 +115,6 @@ describe("OpenAI Responses route", () => {
}),
)
it.effect("requires WebSocket runtime for OpenAI Responses WebSocket", () =>
Effect.gen(function* () {
const error = yield* LLMClient.generate(
LLM.request({
model: OpenAI.responsesWebSocket("gpt-4.1-mini", { baseURL: "https://api.openai.test/v1/", apiKey: "test" }),
prompt: "Say hello.",
}),
).pipe(
Effect.provide(
LLMClient.layer.pipe(
Layer.provide(
Layer.succeed(
RequestExecutor.Service,
RequestExecutor.Service.of({
execute: () => Effect.die("unexpected HTTP request"),
}),
),
),
),
),
Effect.flip,
)
expect(error.message).toContain("requires WebSocketExecutor.Service")
}),
)
it.effect("fails immediately when WebSocket is already closed", () =>
Effect.gen(function* () {
const error = yield* WebSocketExecutor.fromWebSocket(
@@ -155,7 +130,7 @@ describe("OpenAI Responses route", () => {
Effect.gen(function* () {
yield* LLMClient.generate(
LLM.updateRequest(request, {
model: OpenAIResponses.model({ ...model, queryParams: { "api-version": "v1" } }),
model: Model.update(model, { route: model.route.with({ endpoint: { query: { "api-version": "v1" } } }) }),
}),
).pipe(
Effect.provide(
@@ -177,17 +152,18 @@ describe("OpenAI Responses route", () => {
Effect.gen(function* () {
yield* LLMClient.generate(
LLM.updateRequest(request, {
model: Azure.responses("gpt-4.1-mini", {
model: Azure.configure({
baseURL: "https://opencode-test.openai.azure.com/openai/v1/",
apiKey: "azure-key",
headers: { authorization: "Bearer stale" },
}),
}).responses("gpt-4.1-mini"),
}),
).pipe(
Effect.provide(
dynamicResponse((input) =>
Effect.gen(function* () {
const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
expect(web.url).toBe("https://opencode-test.openai.azure.com/openai/v1/responses?api-version=v1")
expect(web.headers.get("api-key")).toBe("azure-key")
expect(web.headers.get("authorization")).toBeNull()
return input.respond(sseEvents({ type: "response.completed", response: {} }), {
@@ -203,7 +179,7 @@ describe("OpenAI Responses route", () => {
it.effect("loads OpenAI default auth from Effect Config", () =>
LLMClient.generate(
LLM.updateRequest(request, {
model: OpenAI.responses("gpt-4.1-mini", { baseURL: "https://api.openai.test/v1/" }),
model: OpenAI.configure({ baseURL: "https://api.openai.test/v1/" }).responses("gpt-4.1-mini"),
}),
).pipe(
configEnv({ OPENAI_API_KEY: "env-key" }),
@@ -224,10 +200,10 @@ describe("OpenAI Responses route", () => {
it.effect("lets explicit auth override OpenAI default API key auth", () =>
LLMClient.generate(
LLM.updateRequest(request, {
model: OpenAI.responses("gpt-4.1-mini", {
model: OpenAI.configure({
baseURL: "https://api.openai.test/v1/",
auth: Auth.bearer("oauth-token"),
}),
}).responses("gpt-4.1-mini"),
}),
).pipe(
Effect.provide(
@@ -274,7 +250,7 @@ describe("OpenAI Responses route", () => {
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
LLM.request({
model: OpenAI.model("gpt-5.2", { baseURL: "https://api.openai.test/v1/" }),
model: OpenAI.configure({ baseURL: "https://api.openai.test/v1/", apiKey: "test" }).model("gpt-5.2"),
prompt: "think",
providerOptions: {
openai: {
@@ -295,14 +271,15 @@ describe("OpenAI Responses route", () => {
}),
)
it.effect("request OpenAI provider options override model defaults", () =>
it.effect("request OpenAI provider options override route defaults", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
LLM.request({
model: OpenAI.model("gpt-4.1-mini", {
model: OpenAI.configure({
baseURL: "https://api.openai.test/v1/",
apiKey: "test",
providerOptions: { openai: { promptCacheKey: "model_cache" } },
}),
}).model("gpt-4.1-mini"),
prompt: "no cache",
providerOptions: { openai: { promptCacheKey: "request_cache" } },
}),
@@ -532,17 +509,36 @@ describe("OpenAI Responses route", () => {
}),
)
it.effect("lowers user image content", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
LLM.request({
id: "req_media",
model,
messages: [Message.user({ type: "media", mediaType: "image/png", data: "AAECAw==" })],
}),
)
expect(prepared.body.input).toEqual([
{
role: "user",
content: [{ type: "input_image", image_url: "data:image/png;base64,AAECAw==" }],
},
])
}),
)
it.effect("rejects unsupported user media content", () =>
Effect.gen(function* () {
const error = yield* LLMClient.prepare(
LLM.request({
id: "req_media",
model,
messages: [Message.user({ type: "media", mediaType: "image/png", data: "AAECAw==" })],
messages: [Message.user({ type: "media", mediaType: "application/pdf", data: "AAECAw==" })],
}),
).pipe(Effect.flip)
expect(error.message).toContain("OpenAI Responses user messages only support text content for now")
expect(error.message).toContain("OpenAI Responses user media content only supports images")
}),
)
packages/llm/test/provider/openrouter.test.ts
+6 -6
View File
@@ -8,15 +8,14 @@ import { it } from "../lib/effect"
describe("OpenRouter", () => {
it.effect("prepares OpenRouter models through the OpenAI-compatible Chat route", () =>
Effect.gen(function* () {
const model = OpenRouter.model("openai/gpt-4o-mini", { apiKey: "test-key" })
const model = OpenRouter.configure({ apiKey: "test-key" }).model("openai/gpt-4o-mini")
expect(model).toMatchObject({
id: "openai/gpt-4o-mini",
provider: "openrouter",
route: "openrouter",
baseURL: "https://openrouter.ai/api/v1",
apiKey: "test-key",
route: { id: "openrouter" },
})
expect(model.route.endpoint.baseURL).toBe("https://openrouter.ai/api/v1")
const prepared = yield* LLMClient.prepare(LLM.request({ model, prompt: "Say hello." }))
@@ -33,7 +32,8 @@ describe("OpenRouter", () => {
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare(
LLM.request({
model: OpenRouter.model("anthropic/claude-3.7-sonnet:thinking", {
model: OpenRouter.configure({
apiKey: "test-key",
providerOptions: {
openrouter: {
usage: true,
@@ -41,7 +41,7 @@ describe("OpenRouter", () => {
promptCacheKey: "session_123",
},
},
}),
}).model("anthropic/claude-3.7-sonnet:thinking"),
prompt: "Think briefly.",
}),
)