lmin/zmo-cli
1
0
Fork 0
Code Issues Pull Requests Actions 91 Packages Projects Releases Wiki Activity

Skip LLM server for prompt tests without LLM calls (#28391)

Browse Source
This commit is contained in:
Kit Langton
2026-05-19 16:27:36 -04:00
committed by GitHub
parent e3c8d221ec
commit b70b45964a
2 changed files with 107 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/opencode/test/session/prompt.test.ts
+106 -95
View File
@@ -164,7 +164,7 @@ const blockingProcessor = Layer.succeed(
}), }),
) )
function makeHttp(input?: { processor?: "blocking" }) { function makePrompt(input?: { processor?: "blocking" }) {
const deps = Layer.mergeAll( const deps = Layer.mergeAll(
Session.defaultLayer, Session.defaultLayer,
Snapshot.defaultLayer, Snapshot.defaultLayer,
@@ -215,29 +215,37 @@ function makeHttp(input?: { processor?: "blocking" }) {
Layer.provideMerge(proc), Layer.provideMerge(proc),
Layer.provideMerge(deps), Layer.provideMerge(deps),
) )
return Layer.mergeAll( return SessionPrompt.layer.pipe(
TestLLMServer.layer, Layer.provide(SessionRevert.defaultLayer),
SessionPrompt.layer.pipe( Layer.provide(Image.defaultLayer),
Layer.provide(SessionRevert.defaultLayer), Layer.provide(Reference.defaultLayer),
Layer.provide(Image.defaultLayer), Layer.provide(summary),
Layer.provide(Reference.defaultLayer), Layer.provideMerge(run),
Layer.provide(summary), Layer.provideMerge(compact),
Layer.provideMerge(run), Layer.provideMerge(proc),
Layer.provideMerge(compact), Layer.provideMerge(registry),
Layer.provideMerge(proc), Layer.provideMerge(trunc),
Layer.provideMerge(registry), Layer.provide(Instruction.defaultLayer),
Layer.provideMerge(trunc), Layer.provide(SystemPrompt.defaultLayer),
Layer.provide(Instruction.defaultLayer), Layer.provide(RuntimeFlags.layer({ experimentalEventSystem: true })),
Layer.provide(SystemPrompt.defaultLayer), Layer.provideMerge(deps),
Layer.provide(RuntimeFlags.layer({ experimentalEventSystem: true })), Layer.provide(summary),
Layer.provideMerge(deps), )
), }
).pipe(Layer.provide(summary))
function makeHttp(input?: { processor?: "blocking" }) {
return Layer.mergeAll(TestLLMServer.layer, makePrompt(input))
}
function makeHttpNoLLMServer(input?: { processor?: "blocking" }) {
return makePrompt(input)
} }
const it = testEffect(makeHttp()) const it = testEffect(makeHttp())
const race = testEffect(makeHttp({ processor: "blocking" })) const noLLMServer = testEffect(makeHttpNoLLMServer())
const raceNoLLMServer = testEffect(makeHttpNoLLMServer({ processor: "blocking" }))
const unix = process.platform !== "win32" ? it.instance : it.instance.skip const unix = process.platform !== "win32" ? it.instance : it.instance.skip
const unixNoLLMServer = process.platform !== "win32" ? noLLMServer.instance : noLLMServer.instance.skip
// Config that registers a custom "test" provider with a "test-model" model // Config that registers a custom "test" provider with a "test-model" model
// so provider model lookup succeeds inside the loop. // so provider model lookup succeeds inside the loop.
@@ -433,19 +441,20 @@ const boot = Effect.fn("test.boot")(function* (input?: { title?: string }) {
// Loop semantics // Loop semantics
it.instance("loop exits immediately when last assistant has stop finish", () => noLLMServer.instance(
Effect.gen(function* () { "loop exits immediately when last assistant has stop finish",
const { llm } = yield* useServerConfig(providerCfg) () =>
const prompt = yield* SessionPrompt.Service Effect.gen(function* () {
const sessions = yield* Session.Service const prompt = yield* SessionPrompt.Service
const chat = yield* sessions.create({ title: "Pinned" }) const sessions = yield* Session.Service
yield* seed(chat.id, { finish: "stop" }) const chat = yield* sessions.create({ title: "Pinned" })
yield* seed(chat.id, { finish: "stop" })
const result = yield* prompt.loop({ sessionID: chat.id }) const result = yield* prompt.loop({ sessionID: chat.id })
expect(result.info.role).toBe("assistant") expect(result.info.role).toBe("assistant")
if (result.info.role === "assistant") expect(result.info.finish).toBe("stop") if (result.info.role === "assistant") expect(result.info.finish).toBe("stop")
expect(yield* llm.calls).toBe(0) }),
}), { config: cfg },
) )
it.instance("loop calls LLM and returns assistant message", () => it.instance("loop calls LLM and returns assistant message", () =>
@@ -473,43 +482,45 @@ it.instance("loop calls LLM and returns assistant message", () =>
}), }),
) )
it.instance("prompt emits v2 prompted and synthetic events", () => noLLMServer.instance(
Effect.gen(function* () { "prompt emits v2 prompted and synthetic events",
yield* useServerConfig(providerCfg) () =>
const prompt = yield* SessionPrompt.Service Effect.gen(function* () {
const sessions = yield* Session.Service const prompt = yield* SessionPrompt.Service
const chat = yield* sessions.create({ title: "Pinned" }) const sessions = yield* Session.Service
const chat = yield* sessions.create({ title: "Pinned" })
yield* prompt.prompt({ yield* prompt.prompt({
sessionID: chat.id, sessionID: chat.id,
agent: "build", agent: "build",
noReply: true, noReply: true,
parts: [ parts: [
{ type: "text", text: "hello v2" }, { type: "text", text: "hello v2" },
{ {
type: "file", type: "file",
mime: "text/plain", mime: "text/plain",
filename: "note.txt", filename: "note.txt",
url: "data:text/plain;base64,bm90ZSBjb250ZW50", url: "data:text/plain;base64,bm90ZSBjb250ZW50",
}, },
], ],
}) })
const messages = yield* SessionV2.Service.use((session) => session.messages({ sessionID: chat.id })).pipe( const messages = yield* SessionV2.Service.use((session) => session.messages({ sessionID: chat.id })).pipe(
Effect.provide(SessionV2.layer), Effect.provide(SessionV2.layer),
) )
const row = Database.use((db) => const row = Database.use((db) =>
db.select().from(SessionMessageTable).where(Database.eq(SessionMessageTable.session_id, chat.id)).get(), db.select().from(SessionMessageTable).where(Database.eq(SessionMessageTable.session_id, chat.id)).get(),
) )
expect(messages.find((message) => message.type === "user")).toMatchObject({ type: "user", text: "hello v2" }) expect(messages.find((message) => message.type === "user")).toMatchObject({ type: "user", text: "hello v2" })
expect(typeof row?.data.time.created).toBe("number") expect(typeof row?.data.time.created).toBe("number")
expect(messages).toEqual( expect(messages).toEqual(
expect.arrayContaining([ expect.arrayContaining([
expect.objectContaining({ type: "synthetic", text: expect.stringContaining("Called the Read tool") }), expect.objectContaining({ type: "synthetic", text: expect.stringContaining("Called the Read tool") }),
expect.objectContaining({ type: "synthetic", text: "note content" }), expect.objectContaining({ type: "synthetic", text: "note content" }),
]), ]),
) )
}), }),
{ config: cfg },
) )
it.instance("static loop returns assistant text through local provider", () => it.instance("static loop returns assistant text through local provider", () =>
@@ -876,11 +887,10 @@ it.instance(
3_000, 3_000,
) )
race.instance( raceNoLLMServer.instance(
"finalizes assistant when cancelled before processor creation completes", "finalizes assistant when cancelled before processor creation completes",
() => () =>
Effect.gen(function* () { Effect.gen(function* () {
yield* useServerConfig(providerCfg)
processorCreateStarted.length = 0 processorCreateStarted.length = 0
yield* Effect.addFinalizer(() => yield* Effect.addFinalizer(() =>
Effect.sync(() => { Effect.sync(() => {
@@ -962,10 +972,11 @@ race.instance(
expect(lastAssistant.info.parentID).toBe(lastUser?.info.id) expect(lastAssistant.info.parentID).toBe(lastUser?.info.id)
} }
}), }),
{ config: cfg },
3_000, 3_000,
) )
it.instance( noLLMServer.instance(
"cancel finalizes subtask tool state", "cancel finalizes subtask tool state",
() => () =>
Effect.gen(function* () { Effect.gen(function* () {
@@ -1077,7 +1088,7 @@ it.instance(
// Queue semantics // Queue semantics
it.instance("concurrent loop callers get same result", () => noLLMServer.instance("concurrent loop callers get same result", () =>
Effect.gen(function* () { Effect.gen(function* () {
const { prompt, run, chat } = yield* boot() const { prompt, run, chat } = yield* boot()
yield* seed(chat.id, { finish: "stop" }) yield* seed(chat.id, { finish: "stop" })
@@ -1210,7 +1221,7 @@ it.instance(
3_000, 3_000,
) )
it.instance("assertNotBusy succeeds when idle", () => noLLMServer.instance("assertNotBusy succeeds when idle", () =>
Effect.gen(function* () { Effect.gen(function* () {
const run = yield* SessionRunState.Service const run = yield* SessionRunState.Service
const sessions = yield* Session.Service const sessions = yield* Session.Service
@@ -1250,7 +1261,7 @@ it.instance(
3_000, 3_000,
) )
unix( unixNoLLMServer(
"shell captures stdout and stderr in completed tool output", "shell captures stdout and stderr in completed tool output",
() => () =>
Effect.gen(function* () { Effect.gen(function* () {
@@ -1274,7 +1285,7 @@ unix(
{ config: cfg }, { config: cfg },
) )
unix( unixNoLLMServer(
"shell completes a fast command on the preferred shell", "shell completes a fast command on the preferred shell",
() => () =>
Effect.gen(function* () { Effect.gen(function* () {
@@ -1298,7 +1309,7 @@ unix(
{ config: cfg }, { config: cfg },
) )
unix( unixNoLLMServer(
"shell uses configured shell over env shell", "shell uses configured shell over env shell",
() => () =>
withSh(() => withSh(() =>
@@ -1321,7 +1332,7 @@ unix(
30_000, 30_000,
) )
unix( unixNoLLMServer(
"shell commands can change directory after startup", "shell commands can change directory after startup",
() => () =>
Effect.gen(function* () { Effect.gen(function* () {
@@ -1345,7 +1356,7 @@ unix(
{ config: cfg }, { config: cfg },
) )
unix( unixNoLLMServer(
"shell lists files from the project directory", "shell lists files from the project directory",
() => () =>
Effect.gen(function* () { Effect.gen(function* () {
@@ -1371,7 +1382,7 @@ unix(
{ config: cfg }, { config: cfg },
) )
unix( unixNoLLMServer(
"shell captures stderr from a failing command", "shell captures stderr from a failing command",
() => () =>
Effect.gen(function* () { Effect.gen(function* () {
@@ -1393,7 +1404,7 @@ unix(
{ config: cfg }, { config: cfg },
) )
unix( unixNoLLMServer(
"shell updates running metadata before process exit", "shell updates running metadata before process exit",
() => () =>
withSh(() => withSh(() =>
@@ -1531,7 +1542,7 @@ unix(
30_000, 30_000,
) )
unix( unixNoLLMServer(
"cancel interrupts shell and resolves cleanly", "cancel interrupts shell and resolves cleanly",
() => () =>
withSh(() => withSh(() =>
@@ -1565,7 +1576,7 @@ unix(
30_000, 30_000,
) )
unix( unixNoLLMServer(
"cancel persists aborted shell result when shell ignores TERM", "cancel persists aborted shell result when shell ignores TERM",
() => () =>
withSh(() => withSh(() =>
@@ -1658,7 +1669,7 @@ unix(
30_000, 30_000,
) )
unix( unixNoLLMServer(
"cancel interrupts loop queued behind shell", "cancel interrupts loop queued behind shell",
() => () =>
Effect.gen(function* () { Effect.gen(function* () {
@@ -1685,7 +1696,7 @@ unix(
30_000, 30_000,
) )
unix( unixNoLLMServer(
"shell rejects when another shell is already running", "shell rejects when another shell is already running",
() => () =>
withSh(() => withSh(() =>
@@ -1729,7 +1740,7 @@ function hangUntilAborted(tool: { execute: (...args: any[]) => any }) {
}) })
} }
it.instance( noLLMServer.instance(
"interrupt propagates abort signal to read tool via file part (text/plain)", "interrupt propagates abort signal to read tool via file part (text/plain)",
() => () =>
Effect.gen(function* () { Effect.gen(function* () {
@@ -1767,7 +1778,7 @@ it.instance(
30_000, 30_000,
) )
it.instance( noLLMServer.instance(
"interrupt propagates abort signal to read tool via file part (directory)", "interrupt propagates abort signal to read tool via file part (directory)",
() => () =>
Effect.gen(function* () { Effect.gen(function* () {
@@ -1804,7 +1815,7 @@ it.instance(
// Missing file handling // Missing file handling
it.instance( noLLMServer.instance(
"does not fail the prompt when a file part is missing", "does not fail the prompt when a file part is missing",
() => () =>
Effect.gen(function* () { Effect.gen(function* () {
@@ -1840,7 +1851,7 @@ it.instance(
{ config: cfg }, { config: cfg },
) )
it.instance( noLLMServer.instance(
"keeps stored part order stable when file resolution is async", "keeps stored part order stable when file resolution is async",
() => () =>
Effect.gen(function* () { Effect.gen(function* () {
@@ -1882,7 +1893,7 @@ it.instance(
{ config: cfg }, { config: cfg },
) )
it.instance( noLLMServer.instance(
"resolves configured reference mentions before workspace paths and agents", "resolves configured reference mentions before workspace paths and agents",
() => () =>
Effect.gen(function* () { Effect.gen(function* () {
@@ -1937,7 +1948,7 @@ it.instance(
}, },
) )
it.instance( noLLMServer.instance(
"injects metadata for bare configured reference mentions", "injects metadata for bare configured reference mentions",
() => () =>
Effect.gen(function* () { Effect.gen(function* () {
@@ -1976,7 +1987,7 @@ it.instance(
}, },
) )
it.instance( noLLMServer.instance(
"injects metadata for configured reference file attachments", "injects metadata for configured reference file attachments",
() => () =>
Effect.gen(function* () { Effect.gen(function* () {
@@ -2043,7 +2054,7 @@ it.instance(
// Special characters in filenames // Special characters in filenames
it.instance( noLLMServer.instance(
"handles filenames with # character", "handles filenames with # character",
() => () =>
Effect.gen(function* () { Effect.gen(function* () {
@@ -2147,7 +2158,7 @@ it.instance(
// Agent variant // Agent variant
it.instance( noLLMServer.instance(
"applies agent variant only when using agent model", "applies agent variant only when using agent model",
() => () =>
Effect.gen(function* () { Effect.gen(function* () {
@@ -2218,7 +2229,7 @@ it.instance(
// Agent / command resolution errors // Agent / command resolution errors
it.instance( noLLMServer.instance(
"unknown agent throws typed error", "unknown agent throws typed error",
() => () =>
Effect.gen(function* () { Effect.gen(function* () {
@@ -2247,7 +2258,7 @@ it.instance(
30_000, 30_000,
) )
it.instance( noLLMServer.instance(
"unknown agent error includes available agent names", "unknown agent error includes available agent names",
() => () =>
Effect.gen(function* () { Effect.gen(function* () {
@@ -2275,7 +2286,7 @@ it.instance(
30_000, 30_000,
) )
it.instance( noLLMServer.instance(
"unknown command throws typed error with available names", "unknown command throws typed error with available names",
() => () =>
Effect.gen(function* () { Effect.gen(function* () {
perf/test-suite.md
+1
View File
@@ -83,6 +83,7 @@ Repeated setup work, long sleeps/timeouts, serial integration tests, filesystem/
| MCP merge config cases can use Effect-aware instance fixtures | Migrated three MCP merge/override cases to `it.instance` | 1.98s | 1.95s | keep | Neutral timing within noise; removes manual `tmpdir` + `withTestInstance` setup from isolated filesystem-only config cases. | | MCP merge config cases can use Effect-aware instance fixtures | Migrated three MCP merge/override cases to `it.instance` | 1.98s | 1.95s | keep | Neutral timing within noise; removes manual `tmpdir` + `withTestInstance` setup from isolated filesystem-only config cases. |
| Remaining legacy tools config cases can use Effect-aware instance fixtures | Migrated allow/deny legacy `tools` permission cases to `it.instance` | 2.65s | 1.90s | keep | Single baseline before edit; after median from three sequential reruns (2.58, 1.90, 1.90). | | Remaining legacy tools config cases can use Effect-aware instance fixtures | Migrated allow/deny legacy `tools` permission cases to `it.instance` | 2.65s | 1.90s | keep | Single baseline before edit; after median from three sequential reruns (2.58, 1.90, 1.90). |
| Oversized snapshot batch tests only need to cross the 100-file boundary | Reduced large diff/revert fixture sizes while keeping each case above the batch boundary | 4.32s | 3.66s | keep | Three affected snapshot tests; after median from three reruns (4.32, 3.66, 3.66) while still crossing the 100-file boundary. | | Oversized snapshot batch tests only need to cross the 100-file boundary | Reduced large diff/revert fixture sizes while keeping each case above the batch boundary | 4.32s | 3.66s | keep | Three affected snapshot tests; after median from three reruns (4.32, 3.66, 3.66) while still crossing the 100-file boundary. |
| Prompt tests without LLM calls do not need the test LLM server | Added a no-server runner and moved obvious non-LLM prompt/shell cases to it | 25.41s | 21.03s | keep | Full prompt file after simplify pass median from three reruns (20.66, 21.03, 21.64); LLM-backed tests stay on original runner. |
## Profiling Results ## Profiling Results