Add native LLM core foundation (#24712)

packages/llm/test/lib/effect.ts

@@ -0,0 +1,50 @@
+import { test, type TestOptions } from "bun:test"
+import { Cause, Effect, Exit, Layer } from "effect"
+import type * as Scope from "effect/Scope"
+import * as TestClock from "effect/testing/TestClock"
+import * as TestConsole from "effect/testing/TestConsole"
+
+type Body<A, E, R> = Effect.Effect<A, E, R> | (() => Effect.Effect<A, E, R>)
+
+const body = <A, E, R>(value: Body<A, E, R>) => Effect.suspend(() => (typeof value === "function" ? value() : value))
+
+const run = <A, E, R, E2>(value: Body<A, E, R | Scope.Scope>, layer: Layer.Layer<R, E2>) =>
+  Effect.gen(function* () {
+    const exit = yield* body(value).pipe(Effect.scoped, Effect.provide(layer), Effect.exit)
+    if (Exit.isFailure(exit)) {
+      for (const err of Cause.prettyErrors(exit.cause)) {
+        yield* Effect.logError(err)
+      }
+    }
+    return yield* exit
+  }).pipe(Effect.runPromise)
+
+const make = <R, E>(testLayer: Layer.Layer<R, E>, liveLayer: Layer.Layer<R, E>) => {
+  const effect = <A, E2>(name: string, value: Body<A, E2, R | Scope.Scope>, opts?: number | TestOptions) =>
+    test(name, () => run(value, testLayer), opts)
+
+  effect.only = <A, E2>(name: string, value: Body<A, E2, R | Scope.Scope>, opts?: number | TestOptions) =>
+    test.only(name, () => run(value, testLayer), opts)
+
+  effect.skip = <A, E2>(name: string, value: Body<A, E2, R | Scope.Scope>, opts?: number | TestOptions) =>
+    test.skip(name, () => run(value, testLayer), opts)
+
+  const live = <A, E2>(name: string, value: Body<A, E2, R | Scope.Scope>, opts?: number | TestOptions) =>
+    test(name, () => run(value, liveLayer), opts)
+
+  live.only = <A, E2>(name: string, value: Body<A, E2, R | Scope.Scope>, opts?: number | TestOptions) =>
+    test.only(name, () => run(value, liveLayer), opts)
+
+  live.skip = <A, E2>(name: string, value: Body<A, E2, R | Scope.Scope>, opts?: number | TestOptions) =>
+    test.skip(name, () => run(value, liveLayer), opts)
+
+  return { effect, live }
+}
+
+const testEnv = Layer.mergeAll(TestConsole.layer, TestClock.layer())
+const liveEnv = TestConsole.layer
+
+export const it = make(testEnv, liveEnv)
+
+export const testEffect = <R, E>(layer: Layer.Layer<R, E>) =>
+  make(Layer.provideMerge(layer, testEnv), Layer.provideMerge(layer, liveEnv))

packages/llm/test/lib/http.ts

@@ -0,0 +1,96 @@
+import { Effect, Layer, Ref } from "effect"
+import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
+import { LLMClient, RequestExecutor } from "../../src/route"
+import type { Service as LLMClientService } from "../../src/route/client"
+import type { Service as RequestExecutorService } from "../../src/route/executor"
+
+export type HandlerInput = {
+  readonly request: HttpClientRequest.HttpClientRequest
+  readonly text: string
+  readonly respond: (
+    body: ConstructorParameters<typeof Response>[0],
+    init?: ResponseInit,
+  ) => HttpClientResponse.HttpClientResponse
+}
+
+export type Handler = (input: HandlerInput) => Effect.Effect<HttpClientResponse.HttpClientResponse>
+
+const handlerLayer = (handler: Handler): Layer.Layer<HttpClient.HttpClient> =>
+  Layer.succeed(
+    HttpClient.HttpClient,
+    HttpClient.make((request) =>
+      Effect.gen(function* () {
+        const web = yield* HttpClientRequest.toWeb(request).pipe(Effect.orDie)
+        const text = yield* Effect.promise(() => web.text())
+        return yield* handler({
+          request,
+          text,
+          respond: (body, init) => HttpClientResponse.fromWeb(request, new Response(body, init)),
+        })
+      }),
+    ),
+  )
+
+export type RuntimeEnv = RequestExecutorService | LLMClientService
+
+export const runtimeLayer = (layer: Layer.Layer<HttpClient.HttpClient>): Layer.Layer<RuntimeEnv> => {
+  const requestExecutorLayer = RequestExecutor.layer.pipe(Layer.provide(layer))
+  const llmClientLayer = LLMClient.layer.pipe(Layer.provide(requestExecutorLayer))
+  return Layer.mergeAll(requestExecutorLayer, llmClientLayer)
+}
+
+const SSE_HEADERS = { "content-type": "text/event-stream" } as const
+
+/**
+ * Layer that returns a single fixed response body. Use for stream-parser
+ * fixture tests where the request shape is irrelevant. The body type widens
+ * to whatever `Response` accepts so binary fixtures (`Uint8Array`,
+ * `ReadableStream`, etc.) flow through without casts.
+ */
+export const fixedResponse = (
+  body: ConstructorParameters<typeof Response>[0],
+  init: ResponseInit = { headers: SSE_HEADERS },
+) => runtimeLayer(handlerLayer((input) => Effect.succeed(input.respond(body, init))))
+
+/**
+ * Layer that builds a response per request. Useful for echo servers.
+ */
+export const dynamicResponse = (handler: Handler) => runtimeLayer(handlerLayer(handler))
+
+/**
+ * Layer that emits the supplied SSE chunks and then aborts mid-stream. Used to
+ * exercise transport errors that surface during parsing.
+ */
+export const truncatedStream = (chunks: ReadonlyArray<string>) =>
+  dynamicResponse((input) =>
+    Effect.sync(() => {
+      const encoder = new TextEncoder()
+      const stream = new ReadableStream({
+        start(controller) {
+          for (const chunk of chunks) controller.enqueue(encoder.encode(chunk))
+          controller.error(new Error("connection reset"))
+        },
+      })
+      return input.respond(stream, { headers: SSE_HEADERS })
+    }),
+  )
+
+/**
+ * Layer that returns successive bodies on each request. Useful for scripting
+ * multi-step model exchanges (e.g. tool-call loops). The last body in the
+ * array is reused if the test makes more requests than scripted.
+ */
+export const scriptedResponses = (bodies: ReadonlyArray<string>, init: ResponseInit = { headers: SSE_HEADERS }) => {
+  if (bodies.length === 0) throw new Error("scriptedResponses requires at least one body")
+  return Layer.unwrap(
+    Effect.gen(function* () {
+      const cursor = yield* Ref.make(0)
+      return dynamicResponse((input) =>
+        Effect.gen(function* () {
+          const index = yield* Ref.getAndUpdate(cursor, (n) => n + 1)
+          return input.respond(bodies[index] ?? bodies[bodies.length - 1], init)
+        }),
+      )
+    }),
+  )
+}

packages/llm/test/lib/openai-chunks.ts

@@ -0,0 +1,27 @@
+/**
+ * Shared chunk shapes for OpenAI Chat / OpenAI-compatible Chat fixture tests.
+ * Multiple test files build the same `{ id, choices: [{ delta, finish_reason }], usage }`
+ * envelope; consolidating here keeps tool-call event shapes consistent.
+ */
+
+const FIXTURE_ID = "chatcmpl_fixture"
+
+export const deltaChunk = (delta: object, finishReason: string | null = null) => ({
+  id: FIXTURE_ID,
+  choices: [{ delta, finish_reason: finishReason }],
+  usage: null,
+})
+
+export const usageChunk = (usage: object) => ({
+  id: FIXTURE_ID,
+  choices: [],
+  usage,
+})
+
+export const finishChunk = (reason: string) => deltaChunk({}, reason)
+
+export const toolCallChunk = (id: string, name: string, args: string, index = 0) =>
+  deltaChunk({
+    role: "assistant",
+    tool_calls: [{ index, id, function: { name, arguments: args } }],
+  })

packages/llm/test/lib/sse.ts

@@ -0,0 +1,17 @@
+/**
+ * Helpers for building deterministic SSE bodies in tests.
+ *
+ * Inline template-literal SSE strings are hard to write and review when chunks
+ * contain JSON; this helper accepts plain values and serializes them, so test
+ * authors only think about the chunk shapes, not the wire format.
+ */
+export const sseEvents = (...chunks: ReadonlyArray<unknown>): string =>
+  `${chunks.map(formatChunk).join("")}data: [DONE]\n\n`
+
+const formatChunk = (chunk: unknown) => `data: ${typeof chunk === "string" ? chunk : JSON.stringify(chunk)}\n\n`
+
+/**
+ * Build an SSE body from already-serialized strings (used when the chunk shape
+ * itself is part of what's being tested, e.g. malformed chunks).
+ */
+export const sseRaw = (...lines: ReadonlyArray<string>): string => lines.map((line) => `${line}\n\n`).join("")

packages/llm/test/lib/tool-runtime.ts

@@ -0,0 +1,9 @@
+import { Stream } from "effect"
+import { LLMClient } from "../../src/route"
+import type { Tools } from "../../src/tool"
+import type { RunOptions } from "../../src/tool-runtime"
+
+type CompatRunOptions<T extends Tools> = RunOptions<T> & { readonly maxSteps?: number }
+
+export const runTools = <T extends Tools>(options: CompatRunOptions<T>) =>
+  LLMClient.stream({ ...options, stopWhen: options.stopWhen ?? LLMClient.stepCountIs(options.maxSteps ?? 10) })