lmin/zmo-cli
1
0
Fork 0
Code Issues Pull Requests Actions 88 Packages Projects Releases Wiki Activity

fix(test): auto-acknowledge tool-result follow-ups in mock LLM server (#20528)

Browse Source
This commit is contained in:
Kit Langton
2026-04-01 23:47:26 +00:00
committed by GitHub
parent 48db7cf07a
commit d9d4f895bc
13 changed files with 482 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/app/e2e/session/session-child-navigation.spec.ts
+11 -4
View File
@@ -1,8 +1,9 @@
import { seedSessionTask, withSession } from "../actions"
import { test, expect } from "../fixtures"
import { inputMatch } from "../prompt/mock"
import { promptSelector } from "../selectors"
test("task tool child-session link does not trigger stale show errors", async ({ page, withBackendProject }) => {
test("task tool child-session link does not trigger stale show errors", async ({ page, llm, withMockProject }) => {
test.setTimeout(120_000)
const errs: string[] = []
@@ -12,12 +13,18 @@ test("task tool child-session link does not trigger stale show errors", async ({
page.on("pageerror", onError)
try {
await withBackendProject(async ({ gotoSession, trackSession, sdk }) => {
await withMockProject(async ({ gotoSession, trackSession, sdk }) => {
await withSession(sdk, `e2e child nav ${Date.now()}`, async (session) => {
const child = await seedSessionTask(sdk, {
sessionID: session.id,
const taskInput = {
description: "Open child session",
prompt: "Search the repository for AssistantParts and then reply with exactly CHILD_OK.",
subagent_type: "general",
}
await llm.toolMatch(inputMatch(taskInput), "task", taskInput)
const child = await seedSessionTask(sdk, {
sessionID: session.id,
description: taskInput.description,
prompt: taskInput.prompt,
})
trackSession(child.sessionID)
packages/app/e2e/session/session-composer-dock.spec.ts
+50 -57
View File
@@ -14,6 +14,7 @@ import {
sessionTodoToggleButtonSelector,
} from "../selectors"
import { modKey } from "../utils"
import { inputMatch } from "../prompt/mock"
type Sdk = Parameters<typeof clearSessionDockSeed>[0]
type PermissionRule = { permission: string; pattern: string; action: "allow" | "deny" | "ask" }
@@ -36,6 +37,17 @@ async function withDockSession<T>(
}
}
const defaultQuestions = [
{
header: "Need input",
question: "Pick one option",
options: [
{ label: "Continue", description: "Continue now" },
{ label: "Stop", description: "Stop here" },
],
},
]
test.setTimeout(120_000)
async function withDockSeed<T>(sdk: Sdk, sessionID: string, fn: () => Promise<T>) {
@@ -291,8 +303,8 @@ test("auto-accept toggle works before first submit", async ({ page, withBackendP
})
})
test("blocked question flow unblocks after submit", async ({ page, withBackendProject }) => {
await withBackendProject(async (project) => {
test("blocked question flow unblocks after submit", async ({ page, llm, withMockProject }) => {
await withMockProject(async (project) => {
await withDockSession(
project.sdk,
"e2e composer dock question",
@@ -300,18 +312,10 @@ test("blocked question flow unblocks after submit", async ({ page, withBackendPr
await withDockSeed(project.sdk, session.id, async () => {
await project.gotoSession(session.id)
await llm.toolMatch(inputMatch({ questions: defaultQuestions }), "question", { questions: defaultQuestions })
await seedSessionQuestion(project.sdk, {
sessionID: session.id,
questions: [
{
header: "Need input",
question: "Pick one option",
options: [
{ label: "Continue", description: "Continue now" },
{ label: "Stop", description: "Stop here" },
],
},
],
questions: defaultQuestions,
})
const dock = page.locator(questionDockSelector)
@@ -328,8 +332,8 @@ test("blocked question flow unblocks after submit", async ({ page, withBackendPr
})
})
test("blocked question flow supports keyboard shortcuts", async ({ page, withBackendProject }) => {
await withBackendProject(async (project) => {
test("blocked question flow supports keyboard shortcuts", async ({ page, llm, withMockProject }) => {
await withMockProject(async (project) => {
await withDockSession(
project.sdk,
"e2e composer dock question keyboard",
@@ -337,18 +341,10 @@ test("blocked question flow supports keyboard shortcuts", async ({ page, withBac
await withDockSeed(project.sdk, session.id, async () => {
await project.gotoSession(session.id)
await llm.toolMatch(inputMatch({ questions: defaultQuestions }), "question", { questions: defaultQuestions })
await seedSessionQuestion(project.sdk, {
sessionID: session.id,
questions: [
{
header: "Need input",
question: "Pick one option",
options: [
{ label: "Continue", description: "Continue now" },
{ label: "Stop", description: "Stop here" },
],
},
],
questions: defaultQuestions,
})
const dock = page.locator(questionDockSelector)
@@ -371,8 +367,8 @@ test("blocked question flow supports keyboard shortcuts", async ({ page, withBac
})
})
test("blocked question flow supports escape dismiss", async ({ page, withBackendProject }) => {
await withBackendProject(async (project) => {
test("blocked question flow supports escape dismiss", async ({ page, llm, withMockProject }) => {
await withMockProject(async (project) => {
await withDockSession(
project.sdk,
"e2e composer dock question escape",
@@ -380,18 +376,10 @@ test("blocked question flow supports escape dismiss", async ({ page, withBackend
await withDockSeed(project.sdk, session.id, async () => {
await project.gotoSession(session.id)
await llm.toolMatch(inputMatch({ questions: defaultQuestions }), "question", { questions: defaultQuestions })
await seedSessionQuestion(project.sdk, {
sessionID: session.id,
questions: [
{
header: "Need input",
question: "Pick one option",
options: [
{ label: "Continue", description: "Continue now" },
{ label: "Stop", description: "Stop here" },
],
},
],
questions: defaultQuestions,
})
const dock = page.locator(questionDockSelector)
@@ -512,9 +500,20 @@ test("blocked permission flow supports allow always", async ({ page, withBackend
test("child session question request blocks parent dock and unblocks after submit", async ({
page,
withBackendProject,
llm,
withMockProject,
}) => {
await withBackendProject(async (project) => {
const questions = [
{
header: "Child input",
question: "Pick one child option",
options: [
{ label: "Continue", description: "Continue child" },
{ label: "Stop", description: "Stop child" },
],
},
]
await withMockProject(async (project) => {
await withDockSession(
project.sdk,
"e2e composer dock child question parent",
@@ -532,18 +531,10 @@ test("child session question request blocks parent dock and unblocks after submi
try {
await withDockSeed(project.sdk, child.id, async () => {
await llm.toolMatch(inputMatch({ questions }), "question", { questions })
await seedSessionQuestion(project.sdk, {
sessionID: child.id,
questions: [
{
header: "Child input",
question: "Pick one child option",
options: [
{ label: "Continue", description: "Continue child" },
{ label: "Stop", description: "Stop child" },
],
},
],
questions,
})
const dock = page.locator(questionDockSelector)
@@ -652,8 +643,15 @@ test("todo dock transitions and collapse behavior", async ({ page, withBackendPr
})
})
test("keyboard focus stays off prompt while blocked", async ({ page, withBackendProject }) => {
await withBackendProject(async (project) => {
test("keyboard focus stays off prompt while blocked", async ({ page, llm, withMockProject }) => {
const questions = [
{
header: "Need input",
question: "Pick one option",
options: [{ label: "Continue", description: "Continue now" }],
},
]
await withMockProject(async (project) => {
await withDockSession(
project.sdk,
"e2e composer dock keyboard",
@@ -661,15 +659,10 @@ test("keyboard focus stays off prompt while blocked", async ({ page, withBackend
await withDockSeed(project.sdk, session.id, async () => {
await project.gotoSession(session.id)
await llm.toolMatch(inputMatch({ questions }), "question", { questions })
await seedSessionQuestion(project.sdk, {
sessionID: session.id,
questions: [
{
header: "Need input",
question: "Pick one option",
options: [{ label: "Continue", description: "Continue now" }],
},
],
questions,
})
await expectQuestionBlocked(page)
packages/app/e2e/session/session-review.spec.ts
+42 -33
View File
@@ -1,6 +1,6 @@
import { waitSessionIdle, withSession } from "../actions"
import { test, expect } from "../fixtures"
import { createSdk } from "../utils"
import { inputMatch } from "../prompt/mock"
const count = 14
@@ -40,7 +40,14 @@ function edit(file: string, prev: string, next: string) {
)
}
async function patch(sdk: ReturnType<typeof createSdk>, sessionID: string, patchText: string) {
async function patchWithMock(
llm: Parameters<typeof test>[0]["llm"],
sdk: Parameters<typeof withSession>[0],
sessionID: string,
patchText: string,
) {
const callsBefore = await llm.calls()
await llm.toolMatch(inputMatch({ patchText }), "apply_patch", { patchText })
await sdk.session.promptAsync({
sessionID,
agent: "build",
@@ -54,6 +61,13 @@ async function patch(sdk: ReturnType<typeof createSdk>, sessionID: string, patch
parts: [{ type: "text", text: "Apply the provided patch exactly once." }],
})
// Wait for the agent loop to actually start before checking idle.
// promptAsync is fire-and-forget — without this, waitSessionIdle can
// return immediately because the session status is still undefined.
await expect
.poll(() => llm.calls().then((c) => c > callsBefore), { timeout: 30_000 })
.toBe(true)
await waitSessionIdle(sdk, sessionID, 120_000)
}
@@ -233,8 +247,7 @@ async function fileOverflow(page: Parameters<typeof test>[0]["page"]) {
}
}
test("review applies inline comment clicks without horizontal overflow", async ({ page, withProject }) => {
test.skip(true, "Flaky in CI for now.")
test("review applies inline comment clicks without horizontal overflow", async ({ page, llm, withMockProject }) => {
test.setTimeout(180_000)
const tag = `review-comment-${Date.now()}`
@@ -243,16 +256,15 @@ test("review applies inline comment clicks without horizontal overflow", async (
await page.setViewportSize({ width: 1280, height: 900 })
await withProject(async (project) => {
const sdk = createSdk(project.directory)
await withSession(sdk, `e2e review comment ${tag}`, async (session) => {
await patch(sdk, session.id, seed([{ file, mark: tag }]))
await withMockProject(async (project) => {
await withSession(project.sdk, `e2e review comment ${tag}`, async (session) => {
project.trackSession(session.id)
await patchWithMock(llm, project.sdk, session.id, seed([{ file, mark: tag }]))
await expect
.poll(
async () => {
const diff = await sdk.session.diff({ sessionID: session.id }).then((res) => res.data ?? [])
const diff = await project.sdk.session.diff({ sessionID: session.id }).then((res) => res.data ?? [])
return diff.length
},
{ timeout: 60_000 },
@@ -283,8 +295,7 @@ test("review applies inline comment clicks without horizontal overflow", async (
})
})
test("review file comments submit on click without clipping actions", async ({ page, withProject }) => {
test.skip(true, "Flaky in CI for now.")
test("review file comments submit on click without clipping actions", async ({ page, llm, withMockProject }) => {
test.setTimeout(180_000)
const tag = `review-file-comment-${Date.now()}`
@@ -293,16 +304,15 @@ test("review file comments submit on click without clipping actions", async ({ p
await page.setViewportSize({ width: 1280, height: 900 })
await withProject(async (project) => {
const sdk = createSdk(project.directory)
await withSession(sdk, `e2e review file comment ${tag}`, async (session) => {
await patch(sdk, session.id, seed([{ file, mark: tag }]))
await withMockProject(async (project) => {
await withSession(project.sdk, `e2e review file comment ${tag}`, async (session) => {
project.trackSession(session.id)
await patchWithMock(llm, project.sdk, session.id, seed([{ file, mark: tag }]))
await expect
.poll(
async () => {
const diff = await sdk.session.diff({ sessionID: session.id }).then((res) => res.data ?? [])
const diff = await project.sdk.session.diff({ sessionID: session.id }).then((res) => res.data ?? [])
return diff.length
},
{ timeout: 60_000 },
@@ -334,8 +344,7 @@ test("review file comments submit on click without clipping actions", async ({ p
})
})
test("review keeps scroll position after a live diff update", async ({ page, withProject }) => {
test.skip(Boolean(process.env.CI), "Flaky in CI for now.")
test.fixme("review keeps scroll position after a live diff update", async ({ page, llm, withMockProject }) => {
test.setTimeout(180_000)
const tag = `review-${Date.now()}`
@@ -345,16 +354,15 @@ test("review keeps scroll position after a live diff update", async ({ page, wit
await page.setViewportSize({ width: 1600, height: 1000 })
await withProject(async (project) => {
const sdk = createSdk(project.directory)
await withSession(sdk, `e2e review ${tag}`, async (session) => {
await patch(sdk, session.id, seed(list))
await withMockProject(async (project) => {
await withSession(project.sdk, `e2e review ${tag}`, async (session) => {
project.trackSession(session.id)
await patchWithMock(llm, project.sdk, session.id, seed(list))
await expect
.poll(
async () => {
const info = await sdk.session.get({ sessionID: session.id }).then((res) => res.data)
const info = await project.sdk.session.get({ sessionID: session.id }).then((res) => res.data)
return info?.summary?.files ?? 0
},
{ timeout: 60_000 },
@@ -364,7 +372,7 @@ test("review keeps scroll position after a live diff update", async ({ page, wit
await expect
.poll(
async () => {
const diff = await sdk.session.diff({ sessionID: session.id }).then((res) => res.data ?? [])
const diff = await project.sdk.session.diff({ sessionID: session.id }).then((res) => res.data ?? [])
return diff.length
},
{ timeout: 60_000 },
@@ -381,15 +389,16 @@ test("review keeps scroll position after a live diff update", async ({ page, wit
const view = page.locator('[data-slot="session-review-scroll"] .scroll-view__viewport').first()
await expect(view).toBeVisible()
const heads = page.getByRole("heading", { level: 3 }).filter({ hasText: /^review-scroll-/ })
await expect(heads).toHaveCount(list.length, {
timeout: 60_000,
})
await expect(heads).toHaveCount(list.length, { timeout: 60_000 })
await expand(page)
await waitMark(page, hit.file, hit.mark)
const row = page
.getByRole("heading", { level: 3, name: new RegExp(hit.file.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")) })
.getByRole("heading", {
level: 3,
name: new RegExp(hit.file.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")),
})
.first()
await expect(row).toBeVisible()
await row.evaluate((el) => el.scrollIntoView({ block: "center" }))
@@ -398,12 +407,12 @@ test("review keeps scroll position after a live diff update", async ({ page, wit
const prev = await spot(page, hit.file)
if (!prev) throw new Error(`missing review row for ${hit.file}`)
await patch(sdk, session.id, edit(hit.file, hit.mark, next))
await patchWithMock(llm, project.sdk, session.id, edit(hit.file, hit.mark, next))
await expect
.poll(
async () => {
const diff = await sdk.session.diff({ sessionID: session.id }).then((res) => res.data ?? [])
const diff = await project.sdk.session.diff({ sessionID: session.id }).then((res) => res.data ?? [])
const item = diff.find((item) => item.file === hit.file)
return typeof item?.after === "string" ? item.after : ""
},