feat(native-llm): route Anthropic API-key models through native runtime (#28271)

This commit is contained in:
Kit Langton
2026-05-19 08:20:27 -04:00
committed by GitHub
parent cb15b3ad84
commit 6618e2bce2
7 changed files with 349 additions and 291 deletions
@@ -37,13 +37,16 @@ type StreamInput = {
} }
export function status(input: Pick<StreamInput, "model" | "provider" | "auth">): RuntimeStatus { export function status(input: Pick<StreamInput, "model" | "provider" | "auth">): RuntimeStatus {
if (input.model.providerID !== "openai" && !input.model.providerID.startsWith("opencode")) const providerID = input.model.providerID
return { type: "unsupported", reason: "provider is not openai or opencode" } if (providerID !== "openai" && providerID !== "anthropic" && !providerID.startsWith("opencode"))
if (input.model.api.npm !== "@ai-sdk/openai") return { type: "unsupported", reason: "provider package is not OpenAI" } return { type: "unsupported", reason: "provider is not openai, opencode, or anthropic" }
const npm = input.model.api.npm
if (npm !== "@ai-sdk/openai" && npm !== "@ai-sdk/anthropic")
return { type: "unsupported", reason: "provider package is not OpenAI or Anthropic" }
if (input.auth?.type === "oauth") return { type: "unsupported", reason: "OAuth auth is not supported" } if (input.auth?.type === "oauth") return { type: "unsupported", reason: "OAuth auth is not supported" }
const apiKey = typeof input.provider.options.apiKey === "string" ? input.provider.options.apiKey : input.provider.key const apiKey = typeof input.provider.options.apiKey === "string" ? input.provider.options.apiKey : input.provider.key
if (!apiKey) return { type: "unsupported", reason: "OpenAI API key is not configured" } if (!apiKey) return { type: "unsupported", reason: "API key is not configured" }
return { return {
type: "supported", type: "supported",
@@ -0,0 +1,53 @@
{
"version": 1,
"metadata": {
"name": "session/native-anthropic-tool-loop",
"recordedAt": "2026-05-19T01:40:12.788Z",
"provider": "anthropic",
"protocol": "anthropic-messages",
"route": "anthropic-messages",
"tags": [
"opencode",
"native",
"tool-loop"
]
},
"interactions": [
{
"transport": "http",
"request": {
"method": "POST",
"url": "https://api.anthropic.com/v1/messages",
"headers": {
"content-type": "application/json"
},
"body": "{\"model\":\"claude-haiku-4-5-20251001\",\"system\":[{\"type\":\"text\",\"text\":\"Answer using tools when appropriate.\\nUse the get_weather tool exactly once to look up Paris, then reply with exactly: Paris is sunny.\",\"cache_control\":{\"type\":\"ephemeral\"}}],\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What is the weather in Paris?\",\"cache_control\":{\"type\":\"ephemeral\"}}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get the current weather for a city.\",\"input_schema\":{\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"cache_control\":{\"type\":\"ephemeral\"}}],\"stream\":true,\"max_tokens\":32000,\"temperature\":0}"
},
"response": {
"status": 200,
"headers": {
"content-type": "text/event-stream; charset=utf-8"
},
"body": "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_01KSRzhxWxF38x5yYVYvktbc\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":622,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":54,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}} }\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_01A8pEqifk2HVQfq1ZDNP6iY\",\"name\":\"get_weather\",\"input\":{},\"caller\":{\"type\":\"direct\"}} }\n\nevent: ping\ndata: {\"type\": \"ping\"}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"city\\\": \\\"P\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"aris\\\"}\"} }\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0 }\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":622,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":54} }\n\nevent: message_stop\ndata: {\"type\":\"message_stop\" }\n\n"
}
},
{
"transport": "http",
"request": {
"method": "POST",
"url": "https://api.anthropic.com/v1/messages",
"headers": {
"content-type": "application/json"
},
"body": "{\"model\":\"claude-haiku-4-5-20251001\",\"system\":[{\"type\":\"text\",\"text\":\"Answer using tools when appropriate.\\nUse the get_weather tool exactly once to look up Paris, then reply with exactly: Paris is sunny.\",\"cache_control\":{\"type\":\"ephemeral\"}}],\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What is the weather in Paris?\",\"cache_control\":{\"type\":\"ephemeral\"}}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_01A8pEqifk2HVQfq1ZDNP6iY\",\"name\":\"get_weather\",\"input\":{\"city\":{}}}]},{\"role\":\"user\",\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_01A8pEqifk2HVQfq1ZDNP6iY\",\"content\":\"{\\\"temperature\\\":22,\\\"condition\\\":\\\"sunny\\\"}\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get the current weather for a city.\",\"input_schema\":{\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"cache_control\":{\"type\":\"ephemeral\"}}],\"stream\":true,\"max_tokens\":32000,\"temperature\":0}"
},
"response": {
"status": 200,
"headers": {
"content-type": "text/event-stream; charset=utf-8"
},
"body": "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_01UyghbuSVecMVozDny14vCD\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":697,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":1,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}} }\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"} }\n\nevent: ping\ndata: {\"type\": \"ping\"}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"Paris\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\" is sunny.\"} }\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0 }\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":697,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":7} }\n\nevent: message_stop\ndata: {\"type\":\"message_stop\" }\n\n"
}
}
]
}
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -1,7 +1,7 @@
import { NodeFileSystem } from "@effect/platform-node" import { NodeFileSystem } from "@effect/platform-node"
import { HttpRecorder, Redactor } from "@opencode-ai/http-recorder" import { HttpRecorder, Redactor } from "@opencode-ai/http-recorder"
import { describe, expect } from "bun:test" import { describe, expect } from "bun:test"
import { tool } from "ai" import { tool, type ModelMessage, type JSONValue } from "ai"
import { Effect, Layer, Stream } from "effect" import { Effect, Layer, Stream } from "effect"
import { FetchHttpClient } from "effect/unstable/http" import { FetchHttpClient } from "effect/unstable/http"
import path from "node:path" import path from "node:path"
@@ -12,6 +12,7 @@ import { Plugin } from "@/plugin"
import { Provider } from "@/provider/provider" import { Provider } from "@/provider/provider"
import { ModelID, ProviderID } from "@/provider/schema" import { ModelID, ProviderID } from "@/provider/schema"
import { Filesystem } from "@/util/filesystem" import { Filesystem } from "@/util/filesystem"
import { LLMEvent, LLMResponse } from "@opencode-ai/llm"
import { LLMClient, RequestExecutor } from "@opencode-ai/llm/route" import { LLMClient, RequestExecutor } from "@opencode-ai/llm/route"
import { RuntimeFlags } from "@/effect/runtime-flags" import { RuntimeFlags } from "@/effect/runtime-flags"
import type { Agent } from "../../src/agent/agent" import type { Agent } from "../../src/agent/agent"
@@ -22,22 +23,105 @@ import type { ModelsDev } from "@opencode-ai/core/models-dev"
import { TestInstance } from "../fixture/fixture" import { TestInstance } from "../fixture/fixture"
import { testEffect } from "../lib/effect" import { testEffect } from "../lib/effect"
const OPENAI_CASSETTE = "session/native-openai-tool-call"
const ZEN_CASSETTE = "session/native-zen-tool-call"
const FIXTURES_DIR = path.join(import.meta.dir, "../fixtures/recordings") const FIXTURES_DIR = path.join(import.meta.dir, "../fixtures/recordings")
const OPENAI_API_KEY = process.env.OPENCODE_RECORD_OPENAI_API_KEY ?? process.env.OPENAI_API_KEY
const CONSOLE_TOKEN = process.env.OPENCODE_RECORD_CONSOLE_TOKEN const zenURL = (connection: string) => `https://console.opencode.ai/proxy/connections/${connection}/v1`
const ZEN_ORG_ID = process.env.OPENCODE_RECORD_ZEN_ORG_ID
const ZEN_API_URL = type ProviderSpec = {
process.env.OPENCODE_RECORD_ZEN_API_URL ?? "https://console.opencode.ai/proxy/connections/fixture/v1" readonly providerID: ProviderID
readonly modelID: string
readonly cassette: string
readonly protocol: string
readonly tags: ReadonlyArray<string>
readonly canRecord: boolean
readonly config: (model: ModelsDev.Provider["models"][string]) => Partial<Config.Info>
}
const cloneModel = (model: ModelsDev.Provider["models"][string]) =>
structuredClone(model) as NonNullable<NonNullable<Config.Info["provider"]>[string]["models"]>[string]
const PROVIDERS = {
openai: {
providerID: ProviderID.openai,
modelID: "gpt-4.1-mini",
cassette: "session/native-openai-tool-loop",
protocol: "openai-responses",
tags: ["opencode", "native", "tool-loop"],
canRecord: Boolean(process.env.OPENCODE_RECORD_OPENAI_API_KEY ?? process.env.OPENAI_API_KEY),
config: (model) => ({
enabled_providers: ["openai"],
provider: {
openai: {
name: "OpenAI",
env: ["OPENAI_API_KEY"],
npm: "@ai-sdk/openai",
api: "https://api.openai.com/v1",
models: { [model.id]: cloneModel(model) },
options: {
apiKey: process.env.OPENCODE_RECORD_OPENAI_API_KEY ?? process.env.OPENAI_API_KEY ?? "fixture-openai-key",
baseURL: "https://api.openai.com/v1",
},
},
},
}),
},
opencode: {
providerID: ProviderID.opencode,
modelID: "gpt-5.2-codex",
cassette: "session/native-zen-tool-loop",
protocol: "openai-responses",
tags: ["opencode", "zen", "native", "tool-loop"],
canRecord: Boolean(process.env.OPENCODE_RECORD_CONSOLE_TOKEN && process.env.OPENCODE_RECORD_ZEN_ORG_ID),
config: (model) => ({
enabled_providers: ["opencode"],
provider: {
opencode: {
name: "OpenCode Zen",
env: ["OPENCODE_CONSOLE_TOKEN"],
npm: "@ai-sdk/openai-compatible",
// The connection slug is account-specific; the cassette redactor
// normalizes it to {connection} for replay. Set during recording.
api: zenURL(process.env.OPENCODE_RECORD_ZEN_CONNECTION ?? "fixture"),
models: { [model.id]: cloneModel(model) },
options: {
apiKey: process.env.OPENCODE_RECORD_CONSOLE_TOKEN ?? "fixture-console-token",
headers: { "x-org-id": process.env.OPENCODE_RECORD_ZEN_ORG_ID ?? "fixture-org" },
},
},
},
}),
},
anthropic: {
providerID: ProviderID.anthropic,
modelID: "claude-haiku-4-5-20251001",
cassette: "session/native-anthropic-tool-loop",
protocol: "anthropic-messages",
tags: ["opencode", "native", "tool-loop"],
canRecord: Boolean(process.env.OPENCODE_RECORD_ANTHROPIC_API_KEY ?? process.env.ANTHROPIC_API_KEY),
config: (model) => ({
enabled_providers: ["anthropic"],
provider: {
anthropic: {
name: "Anthropic",
env: ["ANTHROPIC_API_KEY"],
npm: "@ai-sdk/anthropic",
api: "https://api.anthropic.com/v1",
models: { [model.id]: cloneModel(model) },
options: {
apiKey:
process.env.OPENCODE_RECORD_ANTHROPIC_API_KEY ?? process.env.ANTHROPIC_API_KEY ?? "fixture-anthropic-key",
baseURL: "https://api.anthropic.com/v1",
},
},
},
}),
},
} satisfies Record<string, ProviderSpec>
const shouldRecord = process.env.RECORD === "true" const shouldRecord = process.env.RECORD === "true"
const canRunOpenAI = shouldRecord
? Boolean(OPENAI_API_KEY) const canRun = (spec: ProviderSpec) =>
: HttpRecorder.hasCassetteSync(OPENAI_CASSETTE, { directory: FIXTURES_DIR }) shouldRecord ? spec.canRecord : HttpRecorder.hasCassetteSync(spec.cassette, { directory: FIXTURES_DIR })
const canRunZen = shouldRecord
? Boolean(CONSOLE_TOKEN && ZEN_ORG_ID)
: HttpRecorder.hasCassetteSync(ZEN_CASSETTE, { directory: FIXTURES_DIR })
async function loadFixture(providerID: string, modelID: string) { async function loadFixture(providerID: string, modelID: string) {
const data = await Filesystem.readJson<Record<string, ModelsDev.Provider>>( const data = await Filesystem.readJson<Record<string, ModelsDev.Provider>>(
@@ -50,234 +134,140 @@ async function loadFixture(providerID: string, modelID: string) {
return model return model
} }
const openAIConfig = (model: ModelsDev.Provider["models"][string]): Partial<Config.Info> => ({ function recordedNativeLLMLayer(spec: ProviderSpec) {
enabled_providers: ["openai"],
provider: {
openai: {
name: "OpenAI",
env: ["OPENAI_API_KEY"],
npm: "@ai-sdk/openai",
api: "https://api.openai.com/v1",
models: {
[model.id]: JSON.parse(JSON.stringify(model)) as NonNullable<
NonNullable<Config.Info["provider"]>[string]["models"]
>[string],
},
options: {
apiKey: OPENAI_API_KEY ?? "fixture-openai-key",
baseURL: "https://api.openai.com/v1",
},
},
},
})
const zenConfig = (model: ModelsDev.Provider["models"][string]): Partial<Config.Info> => ({
enabled_providers: ["opencode"],
provider: {
opencode: {
name: "OpenCode Zen",
env: ["OPENCODE_CONSOLE_TOKEN"],
npm: "@ai-sdk/openai-compatible",
api: ZEN_API_URL,
models: {
[model.id]: JSON.parse(JSON.stringify(model)) as NonNullable<
NonNullable<Config.Info["provider"]>[string]["models"]
>[string],
},
options: {
apiKey: CONSOLE_TOKEN ?? "fixture-console-token",
headers: {
"x-org-id": ZEN_ORG_ID ?? "fixture-org",
},
},
},
},
})
function recordedNativeLLMLayer(cassette: string, metadata: Record<string, unknown>) {
const cassetteService = HttpRecorder.Cassette.fileSystem({ directory: FIXTURES_DIR }).pipe(
Layer.provide(NodeFileSystem.layer),
)
// Only the HTTP client is recorded; RequestExecutor and the opencode LLM stack remain real. // Only the HTTP client is recorded; RequestExecutor and the opencode LLM stack remain real.
const recorder = HttpRecorder.recordingLayer(cassette, { const recordedClient = LLMClient.layer.pipe(
mode: shouldRecord ? "record" : "replay", Layer.provide(RequestExecutor.layer),
metadata, Layer.provide(
redactor: Redactor.compose( HttpRecorder.recordingLayer(spec.cassette, {
Redactor.defaults({ mode: shouldRecord ? "record" : "replay",
url: { metadata: { provider: spec.providerID, protocol: spec.protocol, route: spec.protocol, tags: spec.tags },
transform: (url) => url.replace(/\/proxy\/connections\/[^/]+\/v1/, "/proxy/connections/{connection}/v1"), redactor: Redactor.compose(
}, Redactor.defaults({
}), url: {
{ transform: (url) => url.replace(/\/proxy\/connections\/[^/]+\/v1/, "/proxy/connections/{connection}/v1"),
response: (snapshot) => ({ ...snapshot, body: snapshot.body.replace(/wrk_[A-Z0-9]+/g, "wrk_redacted") }), },
}, }),
{
response: (snapshot) => ({ ...snapshot, body: snapshot.body.replace(/wrk_[A-Z0-9]+/g, "wrk_redacted") }),
},
),
}).pipe(Layer.provide(FetchHttpClient.layer)),
), ),
}).pipe(Layer.provide(FetchHttpClient.layer))
const executor = RequestExecutor.layer.pipe(Layer.provide(recorder))
const client = LLMClient.layer.pipe(Layer.provide(executor))
const providerLayer = Provider.defaultLayer.pipe(
Layer.provide(Auth.defaultLayer),
Layer.provide(Config.defaultLayer),
Layer.provide(Plugin.defaultLayer),
)
const llmLayer = LLM.layer.pipe(
Layer.provide(Auth.defaultLayer),
Layer.provide(Config.defaultLayer),
Layer.provide(Provider.defaultLayer),
Layer.provide(Plugin.defaultLayer),
Layer.provide(client),
Layer.provide(cassetteService),
Layer.provide(RuntimeFlags.layer({ experimentalNativeLlm: true })),
) )
return Layer.mergeAll(providerLayer, llmLayer) return Layer.mergeAll(
Provider.defaultLayer.pipe(
Layer.provide(Auth.defaultLayer),
Layer.provide(Config.defaultLayer),
Layer.provide(Plugin.defaultLayer),
),
LLM.layer.pipe(
Layer.provide(Auth.defaultLayer),
Layer.provide(Config.defaultLayer),
Layer.provide(Provider.defaultLayer),
Layer.provide(Plugin.defaultLayer),
Layer.provide(recordedClient),
Layer.provide(HttpRecorder.Cassette.fileSystem({ directory: FIXTURES_DIR }).pipe(Layer.provide(NodeFileSystem.layer))),
Layer.provide(RuntimeFlags.layer({ experimentalNativeLlm: true })),
),
)
} }
const openAIIt = testEffect( const writeConfig = (directory: string, spec: ProviderSpec, model: ModelsDev.Provider["models"][string]) =>
recordedNativeLLMLayer(OPENAI_CASSETTE, {
provider: "openai",
protocol: "openai-responses",
route: "openai-responses",
tags: ["opencode", "native", "tool-call"],
}),
)
const zenIt = testEffect(
recordedNativeLLMLayer(ZEN_CASSETTE, {
provider: "opencode",
protocol: "openai-responses",
route: "openai-responses",
tags: ["opencode", "zen", "native", "tool-call"],
}),
)
const recordedOpenAIInstance = canRunOpenAI ? openAIIt.instance : openAIIt.instance.skip
const recordedZenInstance = canRunZen ? zenIt.instance : zenIt.instance.skip
const writeConfig = (
directory: string,
model: ModelsDev.Provider["models"][string],
config: (model: ModelsDev.Provider["models"][string]) => Partial<Config.Info> = openAIConfig,
) =>
Effect.promise(() => Effect.promise(() =>
Bun.write( Bun.write(
path.join(directory, "opencode.json"), path.join(directory, "opencode.json"),
JSON.stringify({ $schema: "https://opencode.ai/config.json", ...config(model) }), JSON.stringify({ $schema: "https://opencode.ai/config.json", ...spec.config(model) }),
), ),
) )
const getModel = (providerID: ProviderID, modelID: ModelID) =>
Effect.gen(function* () {
const provider = yield* Provider.Service
return yield* provider.getModel(providerID, modelID)
})
const collect = (input: LLM.StreamInput) => const collect = (input: LLM.StreamInput) =>
Effect.gen(function* () { Effect.gen(function* () {
const llm = yield* LLM.Service const llm = yield* LLM.Service
return Array.from(yield* llm.stream(input).pipe(Stream.runCollect)) return Array.from(yield* llm.stream(input).pipe(Stream.runCollect))
}) })
describe("session.llm native recorded", () => { const WEATHER_RESULT = { temperature: 22, condition: "sunny" } as const
recordedOpenAIInstance("uses real RequestExecutor with HTTP recorder for native OpenAI tools", () => const WEATHER_SYSTEM =
Effect.gen(function* () { "Use the get_weather tool exactly once to look up Paris, then reply with exactly: Paris is sunny."
const test = yield* TestInstance const WEATHER_USER = "What is the weather in Paris?"
const model = yield* Effect.promise(() => loadFixture("openai", "gpt-4.1-mini"))
yield* writeConfig(test.directory, model)
const sessionID = SessionID.make("session-recorded-native-tool") const weatherTool = tool({
const agent = { description: "Get the current weather for a city.",
name: "test", inputSchema: z.object({ city: z.string() }),
mode: "primary", execute: async () => WEATHER_RESULT,
prompt: "Call tools exactly as instructed.", })
options: {},
permission: [{ permission: "*", pattern: "*", action: "allow" }],
temperature: 0,
} satisfies Agent.Info
const resolved = yield* getModel(ProviderID.openai, ModelID.make(model.id))
let executed: unknown
const events = yield* collect({ const toolRoundtrip = (
user: { call: { readonly id: string; readonly name: string; readonly input: unknown },
id: MessageID.make("msg_user-recorded-native-tool"), result: JSONValue,
sessionID, ): ModelMessage[] => [
role: "user", { role: "assistant", content: [{ type: "tool-call", toolCallId: call.id, toolName: call.name, input: call.input }] },
time: { created: 0 }, {
agent: agent.name, role: "tool",
model: { providerID: ProviderID.make("openai"), modelID: ModelID.make(model.id) }, content: [{ type: "tool-result", toolCallId: call.id, toolName: call.name, output: { type: "json", value: result } }],
} satisfies MessageV2.User, },
]
const driveToolLoop = (spec: ProviderSpec) =>
Effect.gen(function* () {
const test = yield* TestInstance
const model = yield* Effect.promise(() => loadFixture(spec.providerID, spec.modelID))
yield* writeConfig(test.directory, spec, model)
const sessionID = SessionID.make(`session-recorded-${spec.providerID}-loop`)
const modelID = ModelID.make(model.id)
const agent = {
name: "test",
mode: "primary",
prompt: "Answer using tools when appropriate.",
options: {},
permission: [{ permission: "*", pattern: "*", action: "allow" }],
temperature: 0,
} satisfies Agent.Info
const provider = yield* Provider.Service
const resolved = yield* provider.getModel(spec.providerID, modelID)
const userMessage = { role: "user", content: WEATHER_USER } satisfies ModelMessage
const base = {
user: {
id: MessageID.make(`msg_user-recorded-${spec.providerID}-loop`),
sessionID, sessionID,
model: resolved, role: "user",
agent, time: { created: 0 },
system: ["You must call the lookup tool exactly once with query weather. Do not answer in text."], agent: agent.name,
messages: [{ role: "user", content: "Use lookup." }], model: { providerID: spec.providerID, modelID },
toolChoice: "required", } satisfies MessageV2.User,
tools: { sessionID,
lookup: tool({ model: resolved,
description: "Lookup data.", agent,
inputSchema: z.object({ query: z.string() }), system: [WEATHER_SYSTEM],
execute: async (args, options) => { tools: { get_weather: weatherTool },
executed = { args, toolCallId: options.toolCallId } }
return { output: "looked up" }
},
}),
},
})
expect(events.filter((event) => event.type === "step-finish")).toHaveLength(1) const turn1 = yield* collect({ ...base, messages: [userMessage] })
expect(events.filter((event) => event.type === "finish")).toHaveLength(1) const toolCall = turn1.find(LLMEvent.is.toolCall)
expect(events.some((event) => event.type === "tool-result")).toBe(true) expect(toolCall).toBeDefined()
expect(executed).toMatchObject({ args: { query: "weather" }, toolCallId: expect.any(String) }) expect(turn1.find(LLMEvent.is.toolResult)).toBeDefined()
}), expect(toolCall!.name).toBe("get_weather")
) expect(toolCall!.input).toMatchObject({ city: expect.stringMatching(/Paris/i) })
expect(turn1.filter(LLMEvent.is.stepFinish)).toHaveLength(1)
recordedZenInstance("uses console-managed Zen config with native OpenAI-compatible tools", () => const turn2 = yield* collect({
Effect.gen(function* () { ...base,
const test = yield* TestInstance messages: [userMessage, ...toolRoundtrip(toolCall!, WEATHER_RESULT)],
const model = yield* Effect.promise(() => loadFixture("opencode", "gpt-5.2-codex")) })
yield* writeConfig(test.directory, model, zenConfig)
const sessionID = SessionID.make("session-recorded-native-zen-tool") expect(LLMResponse.text({ events: turn2 })).toMatch(/Paris is sunny/i)
const agent = { expect(turn2.filter(LLMEvent.is.finish)).toHaveLength(1)
name: "test", expect(turn2.filter(LLMEvent.is.toolCall)).toHaveLength(0)
mode: "primary", })
prompt: "Call tools exactly as instructed.",
options: {},
permission: [{ permission: "*", pattern: "*", action: "allow" }],
} satisfies Agent.Info
const resolved = yield* getModel(ProviderID.opencode, ModelID.make(model.id))
let executed: unknown
const events = yield* collect({
user: {
id: MessageID.make("msg_user-recorded-native-zen-tool"),
sessionID,
role: "user",
time: { created: 0 },
agent: agent.name,
model: { providerID: ProviderID.opencode, modelID: ModelID.make(model.id) },
} satisfies MessageV2.User,
sessionID,
model: resolved,
agent,
system: ["You must call the lookup tool exactly once with query weather. Do not answer in text."],
messages: [{ role: "user", content: "Use lookup." }],
toolChoice: "required",
tools: {
lookup: tool({
description: "Lookup data.",
inputSchema: z.object({ query: z.string() }),
execute: async (args, options) => {
executed = { args, toolCallId: options.toolCallId }
return { output: "looked up" }
},
}),
},
})
expect(events.filter((event) => event.type === "step-finish")).toHaveLength(1) describe("session.llm native recorded", () => {
expect(events.filter((event) => event.type === "finish")).toHaveLength(1) for (const [name, spec] of Object.entries(PROVIDERS)) {
expect(events.some((event) => event.type === "tool-result")).toBe(true) const it = testEffect(recordedNativeLLMLayer(spec))
expect(executed).toMatchObject({ args: { query: "weather" }, toolCallId: expect.any(String) }) const instance = canRun(spec) ? it.instance : it.instance.skip
}), instance(`${name}: drives a tool loop to a final text answer`, () => driveToolLoop(spec))
) }
}) })
@@ -262,11 +262,11 @@ describe("session.llm-native.request", () => {
}) })
expect( expect(
LLMNativeRuntime.status({ LLMNativeRuntime.status({
model: { ...baseModel, providerID: ProviderID.make("anthropic") }, model: { ...baseModel, providerID: ProviderID.make("google") },
provider: { ...providerInfo, id: ProviderID.make("anthropic") }, provider: { ...providerInfo, id: ProviderID.make("google") },
auth: undefined, auth: undefined,
}), }),
).toEqual({ type: "unsupported", reason: "provider is not openai or opencode" }) ).toEqual({ type: "unsupported", reason: "provider is not openai, opencode, or anthropic" })
expect( expect(
LLMNativeRuntime.status({ LLMNativeRuntime.status({
model: baseModel, model: baseModel,
@@ -277,11 +277,11 @@ describe("session.llm-native.request", () => {
expect( expect(
LLMNativeRuntime.status({ LLMNativeRuntime.status({
model: { ...baseModel, api: { ...baseModel.api, npm: "@ai-sdk/anthropic" } }, model: { ...baseModel, api: { ...baseModel.api, npm: "@ai-sdk/google" } },
provider: providerInfo, provider: providerInfo,
auth: undefined, auth: undefined,
}), }),
).toEqual({ type: "unsupported", reason: "provider package is not OpenAI" }) ).toEqual({ type: "unsupported", reason: "provider package is not OpenAI or Anthropic" })
expect( expect(
LLMNativeRuntime.status({ LLMNativeRuntime.status({
@@ -289,7 +289,27 @@ describe("session.llm-native.request", () => {
provider: { ...providerInfo, options: {} }, provider: { ...providerInfo, options: {} },
auth: undefined, auth: undefined,
}), }),
).toEqual({ type: "unsupported", reason: "OpenAI API key is not configured" }) ).toEqual({ type: "unsupported", reason: "API key is not configured" })
})
test("enables native runtime for Anthropic API-key models", () => {
expect(
LLMNativeRuntime.status({
model: {
...baseModel,
providerID: ProviderID.make("anthropic"),
api: { ...baseModel.api, npm: "@ai-sdk/anthropic", url: "https://api.anthropic.com/v1" },
},
provider: {
...providerInfo,
id: ProviderID.make("anthropic"),
name: "Anthropic",
env: ["ANTHROPIC_API_KEY"],
options: { apiKey: "test-anthropic-key" },
},
auth: undefined,
}),
).toMatchObject({ type: "supported", apiKey: "test-anthropic-key" })
}) })
test("prefers console provider api key over stored opencode auth", () => { test("prefers console provider api key over stored opencode auth", () => {