refactor(test/lib): generalize run-process harness into cli-process (#28253)

This commit is contained in:
Kit Langton
2026-05-18 19:16:10 -04:00
committed by GitHub
parent ee5cf45ef9
commit 7b8a1037a0
2 changed files with 44 additions and 43 deletions
@@ -1,16 +1,16 @@
// Subprocess integration tests for `opencode run` (non-interactive mode). // Subprocess integration tests for `opencode run` (non-interactive mode).
// These exercise the real CLI binary against a TestLLMServer running in the // These exercise the real CLI binary against a TestLLMServer running in the
// same process. See `test/lib/run-process.ts` for the harness — each test uses // same process. See `test/lib/cli-process.ts` for the harness — each test uses
// `opencode.run(message, opts?)` to spawn `bun src/index.ts run ...` with // `opencode.run(message, opts?)` to spawn `bun src/index.ts run ...` with
// `OPENCODE_CONFIG_CONTENT` providing the test provider config inline. // `OPENCODE_CONFIG_CONTENT` providing the test provider config inline.
import { describe, expect } from "bun:test" import { describe, expect } from "bun:test"
import { Effect } from "effect" import { Effect } from "effect"
import { runIt } from "../../lib/run-process" import { cliIt } from "../../lib/cli-process"
describe("opencode run (non-interactive subprocess)", () => { describe("opencode run (non-interactive subprocess)", () => {
// Happy path: prompt completes, output reaches stdout, process exits 0. // Happy path: prompt completes, output reaches stdout, process exits 0.
// If this fails, all the others likely will too — debug here first. // If this fails, all the others likely will too — debug here first.
runIt.live( cliIt.live(
"exits 0 and writes the response to stdout on a successful prompt", "exits 0 and writes the response to stdout on a successful prompt",
({ llm, opencode }) => ({ llm, opencode }) =>
Effect.gen(function* () { Effect.gen(function* () {
@@ -27,7 +27,7 @@ describe("opencode run (non-interactive subprocess)", () => {
// makes the SDK call surface an error promptly so the process exits nonzero. // makes the SDK call surface an error promptly so the process exits nonzero.
// We assert nonzero exit AND wall-clock under the harness timeout — a hang // We assert nonzero exit AND wall-clock under the harness timeout — a hang
// would expire the timeout and produce a different (signal-killed) failure. // would expire the timeout and produce a different (signal-killed) failure.
runIt.live( cliIt.live(
"exits nonzero promptly when the model is unknown (regression for #27371)", "exits nonzero promptly when the model is unknown (regression for #27371)",
({ opencode }) => ({ opencode }) =>
Effect.gen(function* () { Effect.gen(function* () {
@@ -47,7 +47,7 @@ describe("opencode run (non-interactive subprocess)", () => {
// //
// This is debatable — a future cleanup might flip it to exit 1. If you're // This is debatable — a future cleanup might flip it to exit 1. If you're
// changing this expectation, do it deliberately and say so in the PR. // changing this expectation, do it deliberately and say so in the PR.
runIt.live( cliIt.live(
"mid-stream LLM error still exits 0 today (contract lock-in)", "mid-stream LLM error still exits 0 today (contract lock-in)",
({ llm, opencode }) => ({ llm, opencode }) =>
Effect.gen(function* () { Effect.gen(function* () {
@@ -61,7 +61,7 @@ describe("opencode run (non-interactive subprocess)", () => {
// --format json puts one JSON object per line on stdout for each emitted // --format json puts one JSON object per line on stdout for each emitted
// event. Consumers (CI scripts, tooling) parse this stream. Asserts the // event. Consumers (CI scripts, tooling) parse this stream. Asserts the
// shape so a future event-emit change has to update this expectation. // shape so a future event-emit change has to update this expectation.
runIt.live( cliIt.live(
"--format json emits parseable line-delimited JSON to stdout", "--format json emits parseable line-delimited JSON to stdout",
({ llm, opencode }) => ({ llm, opencode }) =>
Effect.gen(function* () { Effect.gen(function* () {
@@ -1,23 +1,22 @@
// Subprocess test harness for the `opencode run` CLI. // Subprocess test harness for the opencode CLI. Spawns the real binary against
// a TestLLMServer running in-process at a random port, with full env isolation.
// //
// This is the missing test tier: every other `cli/run/*.test.ts` is a unit // This is the missing test tier: in-process tests can't catch bugs that span
// test of an extracted helper. Nothing actually exercises the `RunCommand` // argv parsing → server boot → SDK call → event consumption → exit code (like
// handler end-to-end. Bugs that span argv parsing → server boot → SDK call → // the original /event race or #27371's invalid-model hang).
// event consumption → exit code (like the original /event race or the
// non-interactive hang #27371) are invisible to in-process tests.
// //
// The harness uses opencode's built-in test affordances to spawn the real CLI // Configuration flows through opencode's built-in test affordances:
// hermetically: // - OPENCODE_CONFIG_CONTENT : provider config inline, no files to find
// - OPENCODE_CONFIG_CONTENT : provider config inline, no files to find // - OPENCODE_TEST_HOME : pins os.homedir() → tmpdir
// - OPENCODE_TEST_HOME : pins os.homedir() → tmpdir
// - OPENCODE_DISABLE_PROJECT_CONFIG : skip walking up for opencode.json // - OPENCODE_DISABLE_PROJECT_CONFIG : skip walking up for opencode.json
// - OPENCODE_PURE : skip external plugin discovery + install // - OPENCODE_PURE : skip external plugin discovery + install
// - OPENCODE_DISABLE_AUTOUPDATE / AUTOCOMPACT / MODELS_FETCH : no background work // - OPENCODE_DISABLE_AUTOUPDATE / AUTOCOMPACT / MODELS_FETCH : no background work
//
// Plus HOME / XDG_* pointing at the tmpdir for belt-and-suspenders isolation. // Plus HOME / XDG_* pointing at the tmpdir for belt-and-suspenders isolation.
// //
// The custom `test` provider points at a TestLLMServer running in the same // Today only `opencode.run` is fully wired. The shape supports adding more
// process at a random port. The CLI subprocess talks to it over real HTTP. // builders (`opencode.serve(opts)`, `opencode.acp(opts)`, `opencode.auth(...)`)
// without changing the fixture. Long-lived commands like `serve` will need a
// different return shape — see the TODO at the bottom of OpencodeCli.
import type { TestOptions } from "bun:test" import type { TestOptions } from "bun:test"
import * as Scope from "effect/Scope" import * as Scope from "effect/Scope"
import { Effect } from "effect" import { Effect } from "effect"
@@ -59,10 +58,10 @@ export type RunResult = {
readonly durationMs: number readonly durationMs: number
} }
type SpawnOpts = { readonly timeoutMs?: number; readonly env?: Record<string, string> } export type SpawnOpts = { readonly timeoutMs?: number; readonly env?: Record<string, string> }
// A `RunOpts` is the typed equivalent of constructing argv for `opencode run`. // Typed equivalent of constructing argv for `opencode run`. New flags should
// New flags should land here so tests stay grep-able and refactor-safe. // land here so tests stay grep-able and refactor-safe.
export type RunOpts = SpawnOpts & { export type RunOpts = SpawnOpts & {
readonly model?: string readonly model?: string
readonly agent?: string readonly agent?: string
@@ -73,39 +72,41 @@ export type RunOpts = SpawnOpts & {
} }
export type OpencodeCli = { export type OpencodeCli = {
// High-level: run a single prompt against the test model. // High-level: run a single prompt against the test model. Short-lived.
readonly run: (message: string, opts?: RunOpts) => Effect.Effect<RunResult> readonly run: (message: string, opts?: RunOpts) => Effect.Effect<RunResult>
// Escape hatch: any CLI invocation with full control over argv. // Escape hatch: any CLI invocation with full control over argv. Used to test
// commands that don't yet have a typed builder.
readonly spawn: (args: string[], opts?: SpawnOpts) => Effect.Effect<RunResult> readonly spawn: (args: string[], opts?: SpawnOpts) => Effect.Effect<RunResult>
// Convenience assertion. Dumps captured stderr/stdout on mismatch so CI // Convenience assertion. Dumps captured stderr/stdout on mismatch so CI
// failures are debuggable without re-running locally. // failures are debuggable without re-running locally.
readonly expectExit: (result: RunResult, expected: number, label?: string) => void readonly expectExit: (result: RunResult, expected: number, label?: string) => void
// Parse `--format json` stdout into one event object per non-empty line. // Parse `--format json` stdout into one event object per non-empty line.
// The CLI writes `JSON.stringify({ type, sessionID, ... }) + EOL` for each // The CLI writes `JSON.stringify({ type, sessionID, ... }) + EOL` for each
// event (see src/cli/cmd/run.ts `emit`). Throws if any line is malformed // event (see src/cli/cmd/run.ts `emit`). Throws on a malformed line so
// so tests fail loudly rather than silently skipping data. // tests fail loudly rather than silently skipping data.
readonly parseJsonEvents: (stdout: string) => Array<Record<string, unknown>> readonly parseJsonEvents: (stdout: string) => Array<Record<string, unknown>>
// TODO: long-lived builders for `serve` / `acp` / etc. need a different
// return shape — they yield a handle with .url / .kill and live inside the
// surrounding Scope. Add when the first long-lived command is tested.
} }
export type RunFixture = { export type CliFixture = {
readonly llm: TestLLMServer["Service"] readonly llm: TestLLMServer["Service"]
readonly home: string readonly home: string
readonly opencode: OpencodeCli readonly opencode: OpencodeCli
} }
// `withRunFixture(fn)` provisions a TestLLMServer + tmpdir + spawn helper and // Provisions a TestLLMServer + tmpdir + spawn helper and invokes fn. Cleans
// invokes fn. Cleans up the tmpdir on scope exit. // up the tmpdir on scope exit. TestLLMServer.layer is provided internally so
// // the caller doesn't need to wire it up — the fixture's lifetime is tied to
// Note on the R channel: TestLLMServer.layer is provided internally so the // the surrounding Scope.
// caller doesn't need to wire it up. The fixture's lifetime is tied to the export function withCliFixture<A, E>(
// surrounding Scope. fn: (input: CliFixture) => Effect.Effect<A, E>,
export function withRunFixture<A, E>(
fn: (input: RunFixture) => Effect.Effect<A, E>,
): Effect.Effect<A, E | unknown, Scope.Scope> { ): Effect.Effect<A, E | unknown, Scope.Scope> {
return Effect.gen(function* () { return Effect.gen(function* () {
const llm = yield* TestLLMServer const llm = yield* TestLLMServer
const home = path.join(os.tmpdir(), "oc-run-" + Math.random().toString(36).slice(2)) const home = path.join(os.tmpdir(), "oc-cli-" + Math.random().toString(36).slice(2))
yield* Effect.promise(() => fs.mkdir(home, { recursive: true })) yield* Effect.promise(() => fs.mkdir(home, { recursive: true }))
yield* Effect.addFinalizer(() => yield* Effect.addFinalizer(() =>
Effect.promise(() => fs.rm(home, { recursive: true, force: true }).catch(() => undefined)), Effect.promise(() => fs.rm(home, { recursive: true, force: true }).catch(() => undefined)),
@@ -172,14 +173,14 @@ function expectExit(result: RunResult, expected: number, label = "opencode") {
throw new Error(`${label}: expected exit ${expected}, got ${result.exitCode}`) throw new Error(`${label}: expected exit ${expected}, got ${result.exitCode}`)
} }
// `runIt.live(name, fixture => effect)` is the same as // `cliIt.live(name, fixture => effect)` is the same as
// `it.live(name, () => withRunFixture(fixture))` — one fewer nesting level at // `it.live(name, () => withCliFixture(fixture))` — one fewer nesting level at
// every call site. Use this for any test that needs the opencode CLI fixture. // every call site. Use this for any test that needs the opencode CLI fixture.
// //
// Only `.live` is exposed because subprocess tests must run against the real // Only `.live` is exposed because subprocess tests must run against the real
// clock — a TestClock-paused environment can't drive a child process. If you // clock — a TestClock-paused environment can't drive a child process. If you
// need `.only` or `.skip`, fall back to `it.live` + `withRunFixture` directly. // need `.only` or `.skip`, fall back to `it.live` + `withCliFixture` directly.
export const runIt = { export const cliIt = {
live: <A, E>(name: string, body: (input: RunFixture) => Effect.Effect<A, E>, opts?: number | TestOptions) => live: <A, E>(name: string, body: (input: CliFixture) => Effect.Effect<A, E>, opts?: number | TestOptions) =>
it.live(name, () => withRunFixture(body), opts), it.live(name, () => withCliFixture(body), opts),
} }