refactor(server): split HttpApi exercise harness (#26385)
This commit is contained in:
Kit Langton
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,64 @@
|
|||||||
|
import type { CallResult, JsonObject } from "./types"
|
||||||
|
|
||||||
|
export function parse(text: string): unknown {
|
||||||
|
if (!text) return undefined
|
||||||
|
try {
|
||||||
|
return JSON.parse(text) as unknown
|
||||||
|
} catch {
|
||||||
|
return text
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function looksJson(result: CallResult) {
|
||||||
|
return result.contentType.includes("application/json") || result.text.startsWith("{") || result.text.startsWith("[")
|
||||||
|
}
|
||||||
|
|
||||||
|
export function stable(value: unknown): string {
|
||||||
|
return JSON.stringify(sort(value))
|
||||||
|
}
|
||||||
|
|
||||||
|
function sort(value: unknown): unknown {
|
||||||
|
if (Array.isArray(value)) return value.map(sort)
|
||||||
|
if (!value || typeof value !== "object") return value
|
||||||
|
return Object.fromEntries(
|
||||||
|
Object.entries(value)
|
||||||
|
.sort(([left], [right]) => left.localeCompare(right))
|
||||||
|
.map(([key, item]) => [key, sort(item)]),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
export function array(value: unknown): asserts value is unknown[] {
|
||||||
|
if (!Array.isArray(value)) throw new Error("expected array")
|
||||||
|
}
|
||||||
|
|
||||||
|
export function object(value: unknown): asserts value is JsonObject {
|
||||||
|
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("expected object")
|
||||||
|
}
|
||||||
|
|
||||||
|
export function boolean(value: unknown): asserts value is boolean {
|
||||||
|
if (typeof value !== "boolean") throw new Error("expected boolean")
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isRecord(value: unknown): value is JsonObject {
|
||||||
|
return !!value && typeof value === "object" && !Array.isArray(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
export function check(value: boolean, message: string): asserts value {
|
||||||
|
if (!value) throw new Error(message)
|
||||||
|
}
|
||||||
|
|
||||||
|
export function message(error: unknown) {
|
||||||
|
if (error instanceof Error) return error.message
|
||||||
|
return String(error)
|
||||||
|
}
|
||||||
|
|
||||||
|
export function pad(value: string, size: number) {
|
||||||
|
return value.length >= size ? value : value + " ".repeat(size - value.length)
|
||||||
|
}
|
||||||
|
|
||||||
|
export function indent(value: string) {
|
||||||
|
return value
|
||||||
|
.split("\n")
|
||||||
|
.map((line) => ` ${line}`)
|
||||||
|
.join("\n")
|
||||||
|
}
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
import { Flag } from "@opencode-ai/core/flag/flag"
|
||||||
|
import { ConfigProvider, Effect, Layer } from "effect"
|
||||||
|
import { HttpRouter } from "effect/unstable/http"
|
||||||
|
import { parse } from "./assertions"
|
||||||
|
import { runtime, type Runtime } from "./runtime"
|
||||||
|
import type { ActiveScenario, Backend, BackendApp, CallResult, CaptureMode, SeededContext } from "./types"
|
||||||
|
|
||||||
|
export function call(backend: Backend, scenario: ActiveScenario, ctx: SeededContext<unknown>) {
|
||||||
|
return Effect.promise(async () =>
|
||||||
|
capture(await app(await runtime(), backend).request(toRequest(scenario, ctx)), scenario.capture),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
const appCache: Partial<Record<Backend, BackendApp>> = {}
|
||||||
|
|
||||||
|
function app(modules: Runtime, backend: Backend) {
|
||||||
|
Flag.OPENCODE_EXPERIMENTAL_HTTPAPI = backend === "effect"
|
||||||
|
Flag.OPENCODE_SERVER_PASSWORD = undefined
|
||||||
|
Flag.OPENCODE_SERVER_USERNAME = undefined
|
||||||
|
if (appCache[backend]) return appCache[backend]
|
||||||
|
if (backend === "legacy") {
|
||||||
|
const legacy = modules.Server.Legacy().app
|
||||||
|
return (appCache.legacy = {
|
||||||
|
request: (input, init) => legacy.request(input, init),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
const handler = HttpRouter.toWebHandler(
|
||||||
|
modules.ExperimentalHttpApiServer.routes.pipe(
|
||||||
|
Layer.provide(
|
||||||
|
ConfigProvider.layer(
|
||||||
|
ConfigProvider.fromUnknown({ OPENCODE_SERVER_PASSWORD: undefined, OPENCODE_SERVER_USERNAME: undefined }),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
{ disableLogger: true },
|
||||||
|
).handler
|
||||||
|
return (appCache.effect = {
|
||||||
|
request(input: string | URL | Request, init?: RequestInit) {
|
||||||
|
return handler(
|
||||||
|
input instanceof Request ? input : new Request(new URL(input, "http://localhost"), init),
|
||||||
|
modules.ExperimentalHttpApiServer.context,
|
||||||
|
)
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
function toRequest(scenario: ActiveScenario, ctx: SeededContext<unknown>) {
|
||||||
|
const spec = scenario.request(ctx, ctx.state)
|
||||||
|
return new Request(new URL(spec.path, "http://localhost"), {
|
||||||
|
method: scenario.method,
|
||||||
|
headers: spec.body === undefined ? spec.headers : { "content-type": "application/json", ...spec.headers },
|
||||||
|
body: spec.body === undefined ? undefined : JSON.stringify(spec.body),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async function capture(response: Response, mode: CaptureMode): Promise<CallResult> {
|
||||||
|
const text = mode === "stream" ? await captureStream(response) : await response.text()
|
||||||
|
return {
|
||||||
|
status: response.status,
|
||||||
|
contentType: response.headers.get("content-type") ?? "",
|
||||||
|
text,
|
||||||
|
body: parse(text),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function captureStream(response: Response) {
|
||||||
|
if (!response.body) return ""
|
||||||
|
const reader = response.body.getReader()
|
||||||
|
const read = reader.read().then(
|
||||||
|
(result) => ({ result }),
|
||||||
|
(error: unknown) => ({ error }),
|
||||||
|
)
|
||||||
|
const winner = await Promise.race([read, Bun.sleep(1_000).then(() => ({ timeout: true }))])
|
||||||
|
if ("timeout" in winner) {
|
||||||
|
await reader.cancel("timed out waiting for stream chunk").catch(() => undefined)
|
||||||
|
throw new Error("timed out waiting for stream chunk")
|
||||||
|
}
|
||||||
|
if ("error" in winner) throw winner.error
|
||||||
|
await reader.cancel().catch(() => undefined)
|
||||||
|
if (winner.result.done) return ""
|
||||||
|
return new TextDecoder().decode(winner.result.value)
|
||||||
|
}
|
||||||
@@ -0,0 +1,170 @@
|
|||||||
|
import { Effect } from "effect"
|
||||||
|
import { looksJson } from "./assertions"
|
||||||
|
import type {
|
||||||
|
ActiveScenario,
|
||||||
|
BuilderState,
|
||||||
|
CallResult,
|
||||||
|
Comparison,
|
||||||
|
Method,
|
||||||
|
ProjectOptions,
|
||||||
|
ScenarioContext,
|
||||||
|
SeededContext,
|
||||||
|
TodoScenario,
|
||||||
|
} from "./types"
|
||||||
|
|
||||||
|
class ScenarioBuilder<S = undefined> {
|
||||||
|
private readonly state: BuilderState<S>
|
||||||
|
|
||||||
|
constructor(method: Method, path: string, name: string) {
|
||||||
|
this.state = {
|
||||||
|
method,
|
||||||
|
path,
|
||||||
|
name,
|
||||||
|
project: { git: true },
|
||||||
|
seed: () => Effect.succeed(undefined as S),
|
||||||
|
request: (ctx) => ({ path, headers: ctx.headers() }),
|
||||||
|
capture: "full",
|
||||||
|
mutates: false,
|
||||||
|
reset: true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
global() {
|
||||||
|
return this.clone({ project: undefined, request: () => ({ path: this.state.path }) })
|
||||||
|
}
|
||||||
|
|
||||||
|
inProject(project: ProjectOptions = { git: true }) {
|
||||||
|
return this.clone({ project })
|
||||||
|
}
|
||||||
|
|
||||||
|
withLlm() {
|
||||||
|
return this.clone({ project: { ...(this.state.project ?? { git: true }), llm: true } })
|
||||||
|
}
|
||||||
|
|
||||||
|
at(request: BuilderState<S>["request"]) {
|
||||||
|
return this.clone({ request })
|
||||||
|
}
|
||||||
|
|
||||||
|
mutating() {
|
||||||
|
return this.clone({ mutates: true })
|
||||||
|
}
|
||||||
|
|
||||||
|
preserveDatabase() {
|
||||||
|
return this.clone({ reset: false })
|
||||||
|
}
|
||||||
|
|
||||||
|
stream() {
|
||||||
|
return this.clone({ capture: "stream" })
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Assert a non-JSON or shape-only response. */
|
||||||
|
ok(status = 200, compare: Comparison = "status") {
|
||||||
|
return this.done(compare, (_ctx, result) =>
|
||||||
|
Effect.sync(() => {
|
||||||
|
if (result.status !== status) throw new Error(`expected ${status}, got ${result.status}: ${result.text}`)
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
status(
|
||||||
|
status = 200,
|
||||||
|
inspect?: (ctx: SeededContext<S>, result: CallResult) => Effect.Effect<void>,
|
||||||
|
compare: Comparison = "status",
|
||||||
|
) {
|
||||||
|
return this.done(compare, (ctx, result) =>
|
||||||
|
Effect.gen(function* () {
|
||||||
|
if (result.status !== status) throw new Error(`expected ${status}, got ${result.status}: ${result.text}`)
|
||||||
|
if (inspect) yield* inspect(ctx, result)
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Assert JSON status/content-type plus an optional synchronous body check. */
|
||||||
|
json(status = 200, inspect?: (body: unknown, ctx: SeededContext<S>) => void, compare: Comparison = "json") {
|
||||||
|
return this.jsonEffect(status, inspect ? (body, ctx) => Effect.sync(() => inspect(body, ctx)) : undefined, compare)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Assert JSON status/content-type plus optional Effect assertions, e.g. DB side effects. */
|
||||||
|
jsonEffect(
|
||||||
|
status = 200,
|
||||||
|
inspect?: (body: unknown, ctx: SeededContext<S>) => Effect.Effect<void>,
|
||||||
|
compare: Comparison = "json",
|
||||||
|
) {
|
||||||
|
return this.done(compare, (ctx, result) =>
|
||||||
|
Effect.gen(function* () {
|
||||||
|
if (result.status !== status) throw new Error(`expected ${status}, got ${result.status}: ${result.text}`)
|
||||||
|
if (!looksJson(result))
|
||||||
|
throw new Error(`expected JSON response, got ${result.contentType || "no content-type"}`)
|
||||||
|
if (inspect) yield* inspect(result.body, ctx)
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
private clone(next: Partial<BuilderState<S>>) {
|
||||||
|
const builder = new ScenarioBuilder<S>(this.state.method, this.state.path, this.state.name)
|
||||||
|
Object.assign(builder.state, this.state, next)
|
||||||
|
return builder
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Seed typed state before the HTTP request. The returned value becomes `ctx.state`
|
||||||
|
* for `.at(...)` and assertions, giving stateful route tests type-safe setup.
|
||||||
|
*/
|
||||||
|
seeded<Next>(seed: (ctx: ScenarioContext) => Effect.Effect<Next>) {
|
||||||
|
const builder = new ScenarioBuilder<Next>(this.state.method, this.state.path, this.state.name)
|
||||||
|
Object.assign(builder.state, this.state, { seed })
|
||||||
|
return builder
|
||||||
|
}
|
||||||
|
|
||||||
|
private done(
|
||||||
|
compare: Comparison,
|
||||||
|
expect: (ctx: SeededContext<S>, result: CallResult) => Effect.Effect<void>,
|
||||||
|
): ActiveScenario {
|
||||||
|
const state = this.state
|
||||||
|
return {
|
||||||
|
kind: "active",
|
||||||
|
method: state.method,
|
||||||
|
path: state.path,
|
||||||
|
name: state.name,
|
||||||
|
project: state.project,
|
||||||
|
seed: state.seed,
|
||||||
|
request: (ctx, seeded) => state.request({ ...ctx, state: seeded as S }),
|
||||||
|
expect: (ctx, seeded, result) => expect({ ...ctx, state: seeded as S }, result),
|
||||||
|
compare,
|
||||||
|
capture: state.capture,
|
||||||
|
mutates: state.mutates,
|
||||||
|
reset: state.reset,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export const http = {
|
||||||
|
get: (path: string, name: string) => new ScenarioBuilder("GET", path, name),
|
||||||
|
post: (path: string, name: string) => new ScenarioBuilder("POST", path, name),
|
||||||
|
put: (path: string, name: string) => new ScenarioBuilder("PUT", path, name),
|
||||||
|
patch: (path: string, name: string) => new ScenarioBuilder("PATCH", path, name),
|
||||||
|
delete: (path: string, name: string) => new ScenarioBuilder("DELETE", path, name),
|
||||||
|
}
|
||||||
|
|
||||||
|
export const pending = (method: Method, path: string, name: string, reason: string): TodoScenario => ({
|
||||||
|
kind: "todo",
|
||||||
|
method,
|
||||||
|
path,
|
||||||
|
name,
|
||||||
|
reason,
|
||||||
|
})
|
||||||
|
|
||||||
|
export function route(template: string, params: Record<string, string>) {
|
||||||
|
return Object.entries(params).reduce(
|
||||||
|
(next, [key, value]) => next.replaceAll(`{${key}}`, value).replaceAll(`:${key}`, value),
|
||||||
|
template,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
export function controlledPtyInput(title: string | undefined) {
|
||||||
|
return {
|
||||||
|
command: "/bin/sh",
|
||||||
|
args: ["-c", "sleep 30"],
|
||||||
|
...(title ? { title } : {}),
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
import { Flag } from "@opencode-ai/core/flag/flag"
|
||||||
|
import { Effect } from "effect"
|
||||||
|
import path from "path"
|
||||||
|
|
||||||
|
const preserveExerciseGlobalRoot = !!process.env.OPENCODE_HTTPAPI_EXERCISE_GLOBAL
|
||||||
|
export const exerciseGlobalRoot =
|
||||||
|
process.env.OPENCODE_HTTPAPI_EXERCISE_GLOBAL ??
|
||||||
|
path.join(process.env.TMPDIR ?? "/tmp", `opencode-httpapi-global-${process.pid}`)
|
||||||
|
process.env.XDG_DATA_HOME = path.join(exerciseGlobalRoot, "data")
|
||||||
|
process.env.XDG_CONFIG_HOME = path.join(exerciseGlobalRoot, "config")
|
||||||
|
process.env.XDG_STATE_HOME = path.join(exerciseGlobalRoot, "state")
|
||||||
|
process.env.XDG_CACHE_HOME = path.join(exerciseGlobalRoot, "cache")
|
||||||
|
process.env.OPENCODE_DISABLE_SHARE = "true"
|
||||||
|
export const exerciseConfigDirectory = path.join(exerciseGlobalRoot, "config", "opencode")
|
||||||
|
export const exerciseDataDirectory = path.join(exerciseGlobalRoot, "data", "opencode")
|
||||||
|
|
||||||
|
const preserveExerciseDatabase = !!process.env.OPENCODE_HTTPAPI_EXERCISE_DB
|
||||||
|
export const exerciseDatabasePath =
|
||||||
|
process.env.OPENCODE_HTTPAPI_EXERCISE_DB ??
|
||||||
|
path.join(process.env.TMPDIR ?? "/tmp", `opencode-httpapi-exercise-${process.pid}.db`)
|
||||||
|
process.env.OPENCODE_DB = exerciseDatabasePath
|
||||||
|
Flag.OPENCODE_DB = exerciseDatabasePath
|
||||||
|
|
||||||
|
export const original = {
|
||||||
|
OPENCODE_EXPERIMENTAL_HTTPAPI: Flag.OPENCODE_EXPERIMENTAL_HTTPAPI,
|
||||||
|
OPENCODE_SERVER_PASSWORD: Flag.OPENCODE_SERVER_PASSWORD,
|
||||||
|
OPENCODE_SERVER_USERNAME: Flag.OPENCODE_SERVER_USERNAME,
|
||||||
|
}
|
||||||
|
|
||||||
|
export const cleanupExercisePaths = Effect.promise(async () => {
|
||||||
|
const fs = await import("fs/promises")
|
||||||
|
if (!preserveExerciseDatabase) {
|
||||||
|
await Promise.all(
|
||||||
|
[exerciseDatabasePath, `${exerciseDatabasePath}-wal`, `${exerciseDatabasePath}-shm`].map((file) =>
|
||||||
|
fs.rm(file, { force: true }).catch(() => undefined),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if (!preserveExerciseGlobalRoot)
|
||||||
|
await fs.rm(exerciseGlobalRoot, { recursive: true, force: true }).catch(() => undefined)
|
||||||
|
})
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,66 @@
|
|||||||
|
import { indent, pad } from "./assertions"
|
||||||
|
import type { Options, Result, Scenario } from "./types"
|
||||||
|
|
||||||
|
export const color = {
|
||||||
|
dim: "\x1b[2m",
|
||||||
|
green: "\x1b[32m",
|
||||||
|
red: "\x1b[31m",
|
||||||
|
yellow: "\x1b[33m",
|
||||||
|
cyan: "\x1b[36m",
|
||||||
|
reset: "\x1b[0m",
|
||||||
|
}
|
||||||
|
|
||||||
|
export function printHeader(
|
||||||
|
options: Options,
|
||||||
|
effectRoutes: string[],
|
||||||
|
honoRoutes: string[],
|
||||||
|
selected: Scenario[],
|
||||||
|
missing: string[],
|
||||||
|
extra: Scenario[],
|
||||||
|
paths: { database: string; global: string },
|
||||||
|
) {
|
||||||
|
console.log(`${color.cyan}HttpApi exerciser${color.reset}`)
|
||||||
|
console.log(`${color.dim}db=${paths.database}${color.reset}`)
|
||||||
|
console.log(`${color.dim}global=${paths.global}${color.reset}`)
|
||||||
|
console.log(
|
||||||
|
`${color.dim}mode=${options.mode} selected=${selected.length} effectRoutes=${effectRoutes.length} missing=${missing.length} extra=${extra.length} onlyEffect=${effectRoutes.filter((route) => !honoRoutes.includes(route)).length} onlyHono=${honoRoutes.filter((route) => !effectRoutes.includes(route)).length}${color.reset}`,
|
||||||
|
)
|
||||||
|
console.log("")
|
||||||
|
}
|
||||||
|
|
||||||
|
export function printResults(results: Result[], missing: string[], extra: Scenario[]) {
|
||||||
|
for (const result of results) {
|
||||||
|
if (result.status === "pass") {
|
||||||
|
console.log(
|
||||||
|
`${color.green}PASS${color.reset} ${pad(result.scenario.method, 6)} ${pad(result.scenario.path, 48)} ${result.scenario.name}`,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if (result.status === "skip") {
|
||||||
|
console.log(
|
||||||
|
`${color.yellow}SKIP${color.reset} ${pad(result.scenario.method, 6)} ${pad(result.scenario.path, 48)} ${result.scenario.name} ${color.dim}${result.scenario.reason}${color.reset}`,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
console.log(
|
||||||
|
`${color.red}FAIL${color.reset} ${pad(result.scenario.method, 6)} ${pad(result.scenario.path, 48)} ${result.scenario.name}`,
|
||||||
|
)
|
||||||
|
console.log(`${color.red}${indent(result.message)}${color.reset}`)
|
||||||
|
}
|
||||||
|
if (missing.length > 0) {
|
||||||
|
console.log("\nMissing scenarios")
|
||||||
|
for (const route of missing) console.log(`${color.red}MISS${color.reset} ${route}`)
|
||||||
|
}
|
||||||
|
if (extra.length > 0) {
|
||||||
|
console.log("\nExtra scenarios")
|
||||||
|
for (const scenario of extra)
|
||||||
|
console.log(`${color.yellow}EXTRA${color.reset} ${routeKey(scenario)} ${scenario.name}`)
|
||||||
|
}
|
||||||
|
console.log(
|
||||||
|
`\n${color.dim}summary pass=${results.filter((result) => result.status === "pass").length} fail=${results.filter((result) => result.status === "fail").length} skip=${results.filter((result) => result.status === "skip").length} missing=${missing.length} extra=${extra.length}${color.reset}`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
function routeKey(scenario: Scenario) {
|
||||||
|
return `${scenario.method} ${scenario.path}`
|
||||||
|
}
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
import { OpenApiMethods, type OpenApiSpec, type Options, type Result, type Scenario } from "./types"
|
||||||
|
|
||||||
|
export function routeKeys(spec: OpenApiSpec) {
|
||||||
|
return Object.entries(spec.paths ?? {})
|
||||||
|
.flatMap(([path, item]) =>
|
||||||
|
OpenApiMethods.filter((method) => item[method]).map((method) => `${method.toUpperCase()} ${path}`),
|
||||||
|
)
|
||||||
|
.sort()
|
||||||
|
}
|
||||||
|
|
||||||
|
export function routeKey(scenario: Scenario) {
|
||||||
|
return `${scenario.method} ${scenario.path}`
|
||||||
|
}
|
||||||
|
|
||||||
|
export function coverageResult(scenario: Scenario): Result {
|
||||||
|
if (scenario.kind === "todo") return { status: "skip", scenario }
|
||||||
|
return { status: "pass", scenario }
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseOptions(args: string[]): Options {
|
||||||
|
const mode = option(args, "--mode") ?? "effect"
|
||||||
|
if (mode !== "effect" && mode !== "parity" && mode !== "coverage") throw new Error(`invalid --mode ${mode}`)
|
||||||
|
return {
|
||||||
|
mode,
|
||||||
|
include: option(args, "--include"),
|
||||||
|
failOnMissing: args.includes("--fail-on-missing"),
|
||||||
|
failOnSkip: args.includes("--fail-on-skip"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function matches(options: Options, scenario: Scenario) {
|
||||||
|
if (!options.include) return true
|
||||||
|
return (
|
||||||
|
scenario.name.includes(options.include) ||
|
||||||
|
scenario.path.includes(options.include) ||
|
||||||
|
scenario.method.includes(options.include.toUpperCase())
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
function option(args: string[], name: string) {
|
||||||
|
const index = args.indexOf(name)
|
||||||
|
if (index === -1) return undefined
|
||||||
|
return args[index + 1]
|
||||||
|
}
|
||||||
@@ -0,0 +1,245 @@
|
|||||||
|
import { Flag } from "@opencode-ai/core/flag/flag"
|
||||||
|
import { Cause, Effect } from "effect"
|
||||||
|
import { TestLLMServer } from "../../lib/llm-server"
|
||||||
|
import type { Config } from "../../../src/config/config"
|
||||||
|
import { ModelID, ProviderID } from "../../../src/provider/schema"
|
||||||
|
import type { MessageV2 } from "../../../src/session/message-v2"
|
||||||
|
import { MessageID, PartID } from "../../../src/session/schema"
|
||||||
|
import { stable } from "./assertions"
|
||||||
|
import { call } from "./backend"
|
||||||
|
import { original } from "./environment"
|
||||||
|
import { runtime } from "./runtime"
|
||||||
|
import type {
|
||||||
|
ActiveScenario,
|
||||||
|
CallResult,
|
||||||
|
Options,
|
||||||
|
ProjectOptions,
|
||||||
|
Result,
|
||||||
|
Scenario,
|
||||||
|
ScenarioContext,
|
||||||
|
SeededContext,
|
||||||
|
} from "./types"
|
||||||
|
|
||||||
|
export function runScenario(options: Options) {
|
||||||
|
return (scenario: Scenario) => {
|
||||||
|
if (scenario.kind === "todo") return Effect.succeed({ status: "skip", scenario } as Result)
|
||||||
|
return runActive(options, scenario).pipe(
|
||||||
|
Effect.as({ status: "pass", scenario } as Result),
|
||||||
|
Effect.catchCause((cause) => Effect.succeed({ status: "fail" as const, scenario, message: Cause.pretty(cause) })),
|
||||||
|
Effect.scoped,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function runActive(options: Options, scenario: ActiveScenario) {
|
||||||
|
if (options.mode === "parity" && scenario.mutates && scenario.compare !== "none") {
|
||||||
|
return Effect.gen(function* () {
|
||||||
|
const effect = yield* runBackend("effect", scenario)
|
||||||
|
const legacy = yield* runBackend("legacy", scenario)
|
||||||
|
yield* compare(scenario, effect, legacy)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return withContext(scenario, (ctx) =>
|
||||||
|
Effect.gen(function* () {
|
||||||
|
const effect = yield* call("effect", scenario, ctx)
|
||||||
|
yield* scenario.expect(ctx, ctx.state, effect)
|
||||||
|
if (options.mode === "parity" && scenario.compare !== "none") {
|
||||||
|
const legacy = yield* call("legacy", scenario, ctx)
|
||||||
|
yield* scenario.expect(ctx, ctx.state, legacy)
|
||||||
|
yield* compare(scenario, effect, legacy)
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
function runBackend(backend: "effect" | "legacy", scenario: ActiveScenario) {
|
||||||
|
return withContext(scenario, (ctx) =>
|
||||||
|
Effect.gen(function* () {
|
||||||
|
const result = yield* call(backend, scenario, ctx)
|
||||||
|
yield* scenario.expect(ctx, ctx.state, result)
|
||||||
|
return result
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
function withContext<A, E>(scenario: ActiveScenario, use: (ctx: SeededContext<unknown>) => Effect.Effect<A, E>) {
|
||||||
|
return Effect.acquireRelease(
|
||||||
|
Effect.gen(function* () {
|
||||||
|
const llm = scenario.project?.llm ? yield* TestLLMServer : undefined
|
||||||
|
const project = scenario.project
|
||||||
|
const dir = project
|
||||||
|
? yield* Effect.promise(async () => (await runtime()).tmpdir(projectOptions(project, llm?.url)))
|
||||||
|
: undefined
|
||||||
|
return { dir, llm }
|
||||||
|
}),
|
||||||
|
(ctx) => Effect.promise(async () => void (await ctx.dir?.[Symbol.asyncDispose]())).pipe(Effect.ignore),
|
||||||
|
).pipe(
|
||||||
|
Effect.flatMap((context) =>
|
||||||
|
Effect.gen(function* () {
|
||||||
|
const modules = yield* Effect.promise(() => runtime())
|
||||||
|
const path = context.dir?.path
|
||||||
|
const instance = path
|
||||||
|
? yield* modules.InstanceStore.Service.use((store) => store.load({ directory: path })).pipe(
|
||||||
|
Effect.provide(modules.AppLayer),
|
||||||
|
Effect.catchCause((cause) =>
|
||||||
|
Effect.sleep("100 millis").pipe(
|
||||||
|
Effect.andThen(
|
||||||
|
modules.InstanceStore.Service.use((store) => store.load({ directory: path })).pipe(
|
||||||
|
Effect.provide(modules.AppLayer),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
Effect.catchCause(() => Effect.failCause(cause)),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
: undefined
|
||||||
|
const run = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
|
||||||
|
effect.pipe(Effect.provideService(modules.InstanceRef, instance), Effect.provide(modules.AppLayer))
|
||||||
|
const directory = () => {
|
||||||
|
if (!context.dir?.path) throw new Error("scenario needs a project directory")
|
||||||
|
return context.dir.path
|
||||||
|
}
|
||||||
|
const llm = () => {
|
||||||
|
if (!context.llm) throw new Error("scenario needs fake LLM")
|
||||||
|
return context.llm
|
||||||
|
}
|
||||||
|
const base: ScenarioContext = {
|
||||||
|
directory: context.dir?.path,
|
||||||
|
headers: (extra) => ({
|
||||||
|
...(context.dir?.path ? { "x-opencode-directory": context.dir.path } : {}),
|
||||||
|
...extra,
|
||||||
|
}),
|
||||||
|
file: (name, content) =>
|
||||||
|
Effect.promise(() => {
|
||||||
|
return Bun.write(`${directory()}/${name}`, content)
|
||||||
|
}).pipe(Effect.asVoid),
|
||||||
|
session: (input) =>
|
||||||
|
run(modules.Session.Service.use((svc) => svc.create({ title: input?.title, parentID: input?.parentID }))),
|
||||||
|
sessionGet: (sessionID) =>
|
||||||
|
run(modules.Session.Service.use((svc) => svc.get(sessionID))).pipe(
|
||||||
|
Effect.catchCause(() => Effect.succeed(undefined)),
|
||||||
|
),
|
||||||
|
project: () =>
|
||||||
|
Effect.sync(() => {
|
||||||
|
if (!instance) throw new Error("scenario needs a project directory")
|
||||||
|
return instance.project
|
||||||
|
}),
|
||||||
|
message: (sessionID, input) =>
|
||||||
|
Effect.gen(function* () {
|
||||||
|
const info: MessageV2.User = {
|
||||||
|
id: MessageID.ascending(),
|
||||||
|
sessionID,
|
||||||
|
role: "user",
|
||||||
|
time: { created: Date.now() },
|
||||||
|
agent: "build",
|
||||||
|
model: {
|
||||||
|
providerID: ProviderID.opencode,
|
||||||
|
modelID: ModelID.make("test"),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
const part: MessageV2.TextPart = {
|
||||||
|
id: PartID.ascending(),
|
||||||
|
sessionID,
|
||||||
|
messageID: info.id,
|
||||||
|
type: "text",
|
||||||
|
text: input?.text ?? "hello",
|
||||||
|
}
|
||||||
|
yield* run(
|
||||||
|
modules.Session.Service.use((svc) =>
|
||||||
|
Effect.gen(function* () {
|
||||||
|
yield* svc.updateMessage(info)
|
||||||
|
yield* svc.updatePart(part)
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return { info, part }
|
||||||
|
}),
|
||||||
|
messages: (sessionID) => run(modules.Session.Service.use((svc) => svc.messages({ sessionID }))),
|
||||||
|
todos: (sessionID, todos) => run(modules.Todo.Service.use((svc) => svc.update({ sessionID, todos }))),
|
||||||
|
worktree: (input) => run(modules.Worktree.Service.use((svc) => svc.create(input))),
|
||||||
|
worktreeRemove: (directory) =>
|
||||||
|
run(modules.Worktree.Service.use((svc) => svc.remove({ directory })).pipe(Effect.ignore)),
|
||||||
|
llmText: (value) => Effect.suspend(() => llm().text(value)),
|
||||||
|
llmWait: (count) => Effect.suspend(() => llm().wait(count)),
|
||||||
|
tuiRequest: (request) => Effect.sync(() => modules.Tui.submitTuiRequest(request)),
|
||||||
|
}
|
||||||
|
const state = yield* scenario.seed(base)
|
||||||
|
return yield* use({ ...base, state })
|
||||||
|
}).pipe(Effect.ensuring(context.llm ? context.llm.reset : Effect.void)),
|
||||||
|
),
|
||||||
|
Effect.ensuring(scenario.reset ? resetState : Effect.void),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
function projectOptions(
|
||||||
|
project: ProjectOptions,
|
||||||
|
llmUrl: string | undefined,
|
||||||
|
): { git?: boolean; config?: Partial<Config.Info> } {
|
||||||
|
if (!project.llm || !llmUrl) return { git: project.git, config: project.config }
|
||||||
|
const fake = fakeLlmConfig(llmUrl)
|
||||||
|
return {
|
||||||
|
git: project.git,
|
||||||
|
config: {
|
||||||
|
...fake,
|
||||||
|
...project.config,
|
||||||
|
provider: {
|
||||||
|
...fake.provider,
|
||||||
|
...project.config?.provider,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function fakeLlmConfig(url: string): Partial<Config.Info> {
|
||||||
|
return {
|
||||||
|
model: "test/test-model",
|
||||||
|
small_model: "test/test-model",
|
||||||
|
provider: {
|
||||||
|
test: {
|
||||||
|
name: "Test",
|
||||||
|
id: "test",
|
||||||
|
env: [],
|
||||||
|
npm: "@ai-sdk/openai-compatible",
|
||||||
|
models: {
|
||||||
|
"test-model": {
|
||||||
|
id: "test-model",
|
||||||
|
name: "Test Model",
|
||||||
|
attachment: false,
|
||||||
|
reasoning: false,
|
||||||
|
temperature: false,
|
||||||
|
tool_call: true,
|
||||||
|
release_date: "2025-01-01",
|
||||||
|
limit: { context: 100000, output: 10000 },
|
||||||
|
cost: { input: 0, output: 0 },
|
||||||
|
options: {},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
options: {
|
||||||
|
apiKey: "test-key",
|
||||||
|
baseURL: url,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function compare(scenario: ActiveScenario, effect: CallResult, legacy: CallResult) {
|
||||||
|
return Effect.sync(() => {
|
||||||
|
if (effect.status !== legacy.status)
|
||||||
|
throw new Error(`legacy returned ${legacy.status}, effect returned ${effect.status}`)
|
||||||
|
if (scenario.compare === "status") return
|
||||||
|
if (stable(effect.body) !== stable(legacy.body))
|
||||||
|
throw new Error(`JSON parity mismatch\nlegacy: ${stable(legacy.body)}\neffect: ${stable(effect.body)}`)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
const resetState = Effect.promise(async () => {
|
||||||
|
const modules = await runtime()
|
||||||
|
Flag.OPENCODE_EXPERIMENTAL_HTTPAPI = original.OPENCODE_EXPERIMENTAL_HTTPAPI
|
||||||
|
Flag.OPENCODE_SERVER_PASSWORD = original.OPENCODE_SERVER_PASSWORD
|
||||||
|
Flag.OPENCODE_SERVER_USERNAME = original.OPENCODE_SERVER_USERNAME
|
||||||
|
await modules.disposeAllInstances()
|
||||||
|
await modules.resetDatabase()
|
||||||
|
await Bun.sleep(25)
|
||||||
|
})
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
export type Runtime = {
|
||||||
|
PublicApi: (typeof import("../../../src/server/routes/instance/httpapi/public"))["PublicApi"]
|
||||||
|
ExperimentalHttpApiServer: (typeof import("../../../src/server/routes/instance/httpapi/server"))["ExperimentalHttpApiServer"]
|
||||||
|
Server: (typeof import("../../../src/server/server"))["Server"]
|
||||||
|
AppLayer: (typeof import("../../../src/effect/app-runtime"))["AppLayer"]
|
||||||
|
InstanceRef: (typeof import("../../../src/effect/instance-ref"))["InstanceRef"]
|
||||||
|
Instance: (typeof import("../../../src/project/instance"))["Instance"]
|
||||||
|
InstanceStore: (typeof import("../../../src/project/instance-store"))["InstanceStore"]
|
||||||
|
Session: (typeof import("../../../src/session/session"))["Session"]
|
||||||
|
Todo: (typeof import("../../../src/session/todo"))["Todo"]
|
||||||
|
Worktree: (typeof import("../../../src/worktree"))["Worktree"]
|
||||||
|
Project: (typeof import("../../../src/project/project"))["Project"]
|
||||||
|
Tui: typeof import("../../../src/server/shared/tui-control")
|
||||||
|
disposeAllInstances: (typeof import("../../fixture/fixture"))["disposeAllInstances"]
|
||||||
|
tmpdir: (typeof import("../../fixture/fixture"))["tmpdir"]
|
||||||
|
resetDatabase: (typeof import("../../fixture/db"))["resetDatabase"]
|
||||||
|
}
|
||||||
|
|
||||||
|
let runtimePromise: Promise<Runtime> | undefined
|
||||||
|
|
||||||
|
export function runtime() {
|
||||||
|
return (runtimePromise ??= (async () => {
|
||||||
|
const publicApi = await import("../../../src/server/routes/instance/httpapi/public")
|
||||||
|
const httpApiServer = await import("../../../src/server/routes/instance/httpapi/server")
|
||||||
|
const server = await import("../../../src/server/server")
|
||||||
|
const appRuntime = await import("../../../src/effect/app-runtime")
|
||||||
|
const instanceRef = await import("../../../src/effect/instance-ref")
|
||||||
|
const instance = await import("../../../src/project/instance")
|
||||||
|
const instanceStore = await import("../../../src/project/instance-store")
|
||||||
|
const session = await import("../../../src/session/session")
|
||||||
|
const todo = await import("../../../src/session/todo")
|
||||||
|
const worktree = await import("../../../src/worktree")
|
||||||
|
const project = await import("../../../src/project/project")
|
||||||
|
const tui = await import("../../../src/server/shared/tui-control")
|
||||||
|
const fixture = await import("../../fixture/fixture")
|
||||||
|
const db = await import("../../fixture/db")
|
||||||
|
return {
|
||||||
|
PublicApi: publicApi.PublicApi,
|
||||||
|
ExperimentalHttpApiServer: httpApiServer.ExperimentalHttpApiServer,
|
||||||
|
Server: server.Server,
|
||||||
|
AppLayer: appRuntime.AppLayer,
|
||||||
|
InstanceRef: instanceRef.InstanceRef,
|
||||||
|
Instance: instance.Instance,
|
||||||
|
InstanceStore: instanceStore.InstanceStore,
|
||||||
|
Session: session.Session,
|
||||||
|
Todo: todo.Todo,
|
||||||
|
Worktree: worktree.Worktree,
|
||||||
|
Project: project.Project,
|
||||||
|
Tui: tui,
|
||||||
|
disposeAllInstances: fixture.disposeAllInstances,
|
||||||
|
tmpdir: fixture.tmpdir,
|
||||||
|
resetDatabase: db.resetDatabase,
|
||||||
|
}
|
||||||
|
})())
|
||||||
|
}
|
||||||
@@ -0,0 +1,111 @@
|
|||||||
|
import type { Effect } from "effect"
|
||||||
|
import type { Config } from "../../../src/config/config"
|
||||||
|
import type { Project } from "../../../src/project/project"
|
||||||
|
import type { Worktree } from "../../../src/worktree"
|
||||||
|
import type { MessageV2 } from "../../../src/session/message-v2"
|
||||||
|
import type { SessionID } from "../../../src/session/schema"
|
||||||
|
|
||||||
|
export const OpenApiMethods = ["get", "post", "put", "delete", "patch"] as const
|
||||||
|
export const Methods = ["GET", "POST", "PUT", "DELETE", "PATCH"] as const
|
||||||
|
|
||||||
|
export type Method = (typeof Methods)[number]
|
||||||
|
export type OpenApiMethod = (typeof OpenApiMethods)[number]
|
||||||
|
export type Mode = "effect" | "parity" | "coverage"
|
||||||
|
export type Backend = "effect" | "legacy"
|
||||||
|
export type Comparison = "none" | "status" | "json"
|
||||||
|
export type CaptureMode = "full" | "stream"
|
||||||
|
export type ProjectOptions = { git?: boolean; config?: Partial<Config.Info>; llm?: boolean }
|
||||||
|
export type OpenApiSpec = { paths?: Record<string, Partial<Record<OpenApiMethod, unknown>>> }
|
||||||
|
export type JsonObject = Record<string, unknown>
|
||||||
|
|
||||||
|
export type Options = {
|
||||||
|
mode: Mode
|
||||||
|
include: string | undefined
|
||||||
|
failOnMissing: boolean
|
||||||
|
failOnSkip: boolean
|
||||||
|
}
|
||||||
|
|
||||||
|
export type RequestSpec = {
|
||||||
|
path: string
|
||||||
|
headers?: Record<string, string>
|
||||||
|
body?: unknown
|
||||||
|
}
|
||||||
|
|
||||||
|
export type CallResult = {
|
||||||
|
status: number
|
||||||
|
contentType: string
|
||||||
|
body: unknown
|
||||||
|
text: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export type BackendApp = {
|
||||||
|
request(input: string | URL | Request, init?: RequestInit): Response | Promise<Response>
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Effect-native helpers available while setting up and asserting a scenario. */
|
||||||
|
export type ScenarioContext = {
|
||||||
|
directory: string | undefined
|
||||||
|
headers: (extra?: Record<string, string>) => Record<string, string>
|
||||||
|
file: (name: string, content: string) => Effect.Effect<void>
|
||||||
|
session: (input?: { title?: string; parentID?: SessionID }) => Effect.Effect<SessionInfo>
|
||||||
|
sessionGet: (sessionID: SessionID) => Effect.Effect<SessionInfo | undefined>
|
||||||
|
project: () => Effect.Effect<Project.Info>
|
||||||
|
message: (sessionID: SessionID, input?: { text?: string }) => Effect.Effect<MessageSeed>
|
||||||
|
messages: (sessionID: SessionID) => Effect.Effect<MessageV2.WithParts[]>
|
||||||
|
todos: (sessionID: SessionID, todos: TodoInfo[]) => Effect.Effect<void>
|
||||||
|
worktree: (input?: { name?: string }) => Effect.Effect<Worktree.Info>
|
||||||
|
worktreeRemove: (directory: string) => Effect.Effect<void>
|
||||||
|
llmText: (value: string) => Effect.Effect<void>
|
||||||
|
llmWait: (count: number) => Effect.Effect<void>
|
||||||
|
tuiRequest: (request: { path: string; body: unknown }) => Effect.Effect<void>
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Scenario context after `.seeded(...)`; `state` preserves the seed return type in the DSL. */
|
||||||
|
export type SeededContext<S> = ScenarioContext & {
|
||||||
|
state: S
|
||||||
|
}
|
||||||
|
|
||||||
|
export type Scenario = ActiveScenario | TodoScenario
|
||||||
|
export type ActiveScenario = {
|
||||||
|
kind: "active"
|
||||||
|
method: Method
|
||||||
|
path: string
|
||||||
|
name: string
|
||||||
|
project: ProjectOptions | undefined
|
||||||
|
seed: (ctx: ScenarioContext) => Effect.Effect<unknown>
|
||||||
|
request: (ctx: ScenarioContext, state: unknown) => RequestSpec
|
||||||
|
expect: (ctx: ScenarioContext, state: unknown, result: CallResult) => Effect.Effect<void>
|
||||||
|
compare: Comparison
|
||||||
|
capture: CaptureMode
|
||||||
|
mutates: boolean
|
||||||
|
reset: boolean
|
||||||
|
}
|
||||||
|
|
||||||
|
export type BuilderState<S> = {
|
||||||
|
method: Method
|
||||||
|
path: string
|
||||||
|
name: string
|
||||||
|
project: ProjectOptions | undefined
|
||||||
|
seed: (ctx: ScenarioContext) => Effect.Effect<S>
|
||||||
|
request: (ctx: SeededContext<S>) => RequestSpec
|
||||||
|
capture: CaptureMode
|
||||||
|
mutates: boolean
|
||||||
|
reset: boolean
|
||||||
|
}
|
||||||
|
|
||||||
|
export type TodoScenario = {
|
||||||
|
kind: "todo"
|
||||||
|
method: Method
|
||||||
|
path: string
|
||||||
|
name: string
|
||||||
|
reason: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export type Result =
|
||||||
|
| { status: "pass"; scenario: ActiveScenario }
|
||||||
|
| { status: "fail"; scenario: ActiveScenario; message: string }
|
||||||
|
| { status: "skip"; scenario: TodoScenario }
|
||||||
|
|
||||||
|
export type SessionInfo = { id: SessionID; title: string; parentID?: SessionID }
|
||||||
|
export type TodoInfo = { content: string; status: string; priority: string }
|
||||||
|
export type MessageSeed = { info: MessageV2.User; part: MessageV2.TextPart }
|
||||||
Reference in New Issue
Block a user