fix(acp): include tool image attachments in updates (#25128)
This commit is contained in:
@@ -347,33 +347,7 @@ export class Agent implements ACPAgent {
|
|||||||
this.toolStarts.delete(part.callID)
|
this.toolStarts.delete(part.callID)
|
||||||
this.shellSnapshots.delete(part.callID)
|
this.shellSnapshots.delete(part.callID)
|
||||||
const kind = toToolKind(part.tool)
|
const kind = toToolKind(part.tool)
|
||||||
const content: ToolCallContent[] = [
|
const content = completedToolContent(part, kind)
|
||||||
{
|
|
||||||
type: "content",
|
|
||||||
content: {
|
|
||||||
type: "text",
|
|
||||||
text: part.state.output,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
]
|
|
||||||
|
|
||||||
if (kind === "edit") {
|
|
||||||
const input = part.state.input
|
|
||||||
const filePath = typeof input["filePath"] === "string" ? input["filePath"] : ""
|
|
||||||
const oldText = typeof input["oldString"] === "string" ? input["oldString"] : ""
|
|
||||||
const newText =
|
|
||||||
typeof input["newString"] === "string"
|
|
||||||
? input["newString"]
|
|
||||||
: typeof input["content"] === "string"
|
|
||||||
? input["content"]
|
|
||||||
: ""
|
|
||||||
content.push({
|
|
||||||
type: "diff",
|
|
||||||
path: filePath,
|
|
||||||
oldText,
|
|
||||||
newText,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
if (part.tool === "todowrite") {
|
if (part.tool === "todowrite") {
|
||||||
const parsedTodos = decodeTodos(part.state.output)
|
const parsedTodos = decodeTodos(part.state.output)
|
||||||
@@ -413,10 +387,7 @@ export class Agent implements ACPAgent {
|
|||||||
content,
|
content,
|
||||||
title: part.state.title,
|
title: part.state.title,
|
||||||
rawInput: part.state.input,
|
rawInput: part.state.input,
|
||||||
rawOutput: {
|
rawOutput: completedToolRawOutput(part),
|
||||||
output: part.state.output,
|
|
||||||
metadata: part.state.metadata,
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
.catch((error) => {
|
.catch((error) => {
|
||||||
@@ -860,33 +831,7 @@ export class Agent implements ACPAgent {
|
|||||||
this.toolStarts.delete(part.callID)
|
this.toolStarts.delete(part.callID)
|
||||||
this.shellSnapshots.delete(part.callID)
|
this.shellSnapshots.delete(part.callID)
|
||||||
const kind = toToolKind(part.tool)
|
const kind = toToolKind(part.tool)
|
||||||
const content: ToolCallContent[] = [
|
const content = completedToolContent(part, kind)
|
||||||
{
|
|
||||||
type: "content",
|
|
||||||
content: {
|
|
||||||
type: "text",
|
|
||||||
text: part.state.output,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
]
|
|
||||||
|
|
||||||
if (kind === "edit") {
|
|
||||||
const input = part.state.input
|
|
||||||
const filePath = typeof input["filePath"] === "string" ? input["filePath"] : ""
|
|
||||||
const oldText = typeof input["oldString"] === "string" ? input["oldString"] : ""
|
|
||||||
const newText =
|
|
||||||
typeof input["newString"] === "string"
|
|
||||||
? input["newString"]
|
|
||||||
: typeof input["content"] === "string"
|
|
||||||
? input["content"]
|
|
||||||
: ""
|
|
||||||
content.push({
|
|
||||||
type: "diff",
|
|
||||||
path: filePath,
|
|
||||||
oldText,
|
|
||||||
newText,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
if (part.tool === "todowrite") {
|
if (part.tool === "todowrite") {
|
||||||
const parsedTodos = decodeTodos(part.state.output)
|
const parsedTodos = decodeTodos(part.state.output)
|
||||||
@@ -926,10 +871,7 @@ export class Agent implements ACPAgent {
|
|||||||
content,
|
content,
|
||||||
title: part.state.title,
|
title: part.state.title,
|
||||||
rawInput: part.state.input,
|
rawInput: part.state.input,
|
||||||
rawOutput: {
|
rawOutput: completedToolRawOutput(part),
|
||||||
output: part.state.output,
|
|
||||||
metadata: part.state.metadata,
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
.catch((err) => {
|
.catch((err) => {
|
||||||
@@ -1655,6 +1597,70 @@ function toLocations(toolName: string, input: Record<string, any>): { path: stri
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function completedToolContent(part: ToolPart, kind: ToolKind): ToolCallContent[] {
|
||||||
|
if (part.state.status !== "completed") return []
|
||||||
|
|
||||||
|
const content: ToolCallContent[] = [
|
||||||
|
{
|
||||||
|
type: "content",
|
||||||
|
content: {
|
||||||
|
type: "text",
|
||||||
|
text: part.state.output,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
if (kind === "edit") {
|
||||||
|
const input = part.state.input
|
||||||
|
const filePath = typeof input["filePath"] === "string" ? input["filePath"] : ""
|
||||||
|
const oldText = typeof input["oldString"] === "string" ? input["oldString"] : ""
|
||||||
|
const newText =
|
||||||
|
typeof input["newString"] === "string"
|
||||||
|
? input["newString"]
|
||||||
|
: typeof input["content"] === "string"
|
||||||
|
? input["content"]
|
||||||
|
: ""
|
||||||
|
content.push({
|
||||||
|
type: "diff",
|
||||||
|
path: filePath,
|
||||||
|
oldText,
|
||||||
|
newText,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
content.push(...imageContents(part.state.attachments ?? []))
|
||||||
|
return content
|
||||||
|
}
|
||||||
|
|
||||||
|
function completedToolRawOutput(part: ToolPart) {
|
||||||
|
if (part.state.status !== "completed") return {}
|
||||||
|
return {
|
||||||
|
output: part.state.output,
|
||||||
|
metadata: part.state.metadata,
|
||||||
|
...(part.state.attachments?.length ? { attachments: part.state.attachments } : {}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function imageContents(attachments: Array<{ mime: string; url: string }>): ToolCallContent[] {
|
||||||
|
return attachments.flatMap((attachment): ToolCallContent[] => {
|
||||||
|
const match = attachment.url.match(/^data:([^;,]+)(?:;[^,]*)*;base64,(.*)$/)
|
||||||
|
const mime = match?.[1] ?? attachment.mime
|
||||||
|
if (!mime.startsWith("image/")) return []
|
||||||
|
const data = match?.[2]
|
||||||
|
if (data === undefined) return []
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
type: "content" as const,
|
||||||
|
content: {
|
||||||
|
type: "image" as const,
|
||||||
|
mimeType: mime,
|
||||||
|
data,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
async function defaultModel(config: ACPConfig, cwd?: string): Promise<{ providerID: ProviderID; modelID: ModelID }> {
|
async function defaultModel(config: ACPConfig, cwd?: string): Promise<{ providerID: ProviderID; modelID: ModelID }> {
|
||||||
const sdk = config.sdk
|
const sdk = config.sdk
|
||||||
const configured = config.defaultModel
|
const configured = config.defaultModel
|
||||||
|
|||||||
@@ -1,7 +1,13 @@
|
|||||||
import { describe, expect, test } from "bun:test"
|
import { describe, expect, test } from "bun:test"
|
||||||
import { ACP } from "../../src/acp/agent"
|
import { ACP } from "../../src/acp/agent"
|
||||||
import type { AgentSideConnection } from "@agentclientprotocol/sdk"
|
import type { AgentSideConnection } from "@agentclientprotocol/sdk"
|
||||||
import type { Event, EventMessagePartUpdated, ToolStatePending, ToolStateRunning } from "@opencode-ai/sdk/v2"
|
import type {
|
||||||
|
Event,
|
||||||
|
EventMessagePartUpdated,
|
||||||
|
ToolStateCompleted,
|
||||||
|
ToolStatePending,
|
||||||
|
ToolStateRunning,
|
||||||
|
} from "@opencode-ai/sdk/v2"
|
||||||
import { Instance } from "../../src/project/instance"
|
import { Instance } from "../../src/project/instance"
|
||||||
import { WithInstance } from "../../src/project/with-instance"
|
import { WithInstance } from "../../src/project/with-instance"
|
||||||
import { tmpdir } from "../fixture/fixture"
|
import { tmpdir } from "../fixture/fixture"
|
||||||
@@ -36,6 +42,14 @@ function isToolCallUpdate(
|
|||||||
return update.sessionUpdate === "tool_call_update"
|
return update.sessionUpdate === "tool_call_update"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function completedToolUpdate(sessionUpdates: SessionUpdateParams[], sessionId: string, callID: string) {
|
||||||
|
return sessionUpdates
|
||||||
|
.filter((u) => u.sessionId === sessionId)
|
||||||
|
.map((u) => u.update)
|
||||||
|
.filter(isToolCallUpdate)
|
||||||
|
.find((u) => u.toolCallId === callID && u.status === "completed")
|
||||||
|
}
|
||||||
|
|
||||||
function toolEvent(
|
function toolEvent(
|
||||||
sessionId: string,
|
sessionId: string,
|
||||||
cwd: string,
|
cwd: string,
|
||||||
@@ -78,6 +92,45 @@ function toolEvent(
|
|||||||
return { directory: cwd, payload }
|
return { directory: cwd, payload }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function completedToolEvent(
|
||||||
|
sessionId: string,
|
||||||
|
cwd: string,
|
||||||
|
opts: {
|
||||||
|
callID: string
|
||||||
|
tool: string
|
||||||
|
input: Record<string, unknown>
|
||||||
|
output: string
|
||||||
|
attachments?: ToolStateCompleted["attachments"]
|
||||||
|
},
|
||||||
|
): GlobalEventEnvelope {
|
||||||
|
const state: ToolStateCompleted = {
|
||||||
|
status: "completed",
|
||||||
|
input: opts.input,
|
||||||
|
output: opts.output,
|
||||||
|
title: opts.tool,
|
||||||
|
metadata: {},
|
||||||
|
time: { start: Date.now() - 1, end: Date.now() },
|
||||||
|
...(opts.attachments && { attachments: opts.attachments }),
|
||||||
|
}
|
||||||
|
const payload: EventMessagePartUpdated = {
|
||||||
|
type: "message.part.updated",
|
||||||
|
properties: {
|
||||||
|
sessionID: sessionId,
|
||||||
|
time: Date.now(),
|
||||||
|
part: {
|
||||||
|
id: `part_${opts.callID}`,
|
||||||
|
sessionID: sessionId,
|
||||||
|
messageID: `msg_${opts.callID}`,
|
||||||
|
type: "tool",
|
||||||
|
callID: opts.callID,
|
||||||
|
tool: opts.tool,
|
||||||
|
state,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
return { directory: cwd, payload }
|
||||||
|
}
|
||||||
|
|
||||||
function createEventStream() {
|
function createEventStream() {
|
||||||
const queue: GlobalEventEnvelope[] = []
|
const queue: GlobalEventEnvelope[] = []
|
||||||
const waiters: Array<(value: GlobalEventEnvelope | undefined) => void> = []
|
const waiters: Array<(value: GlobalEventEnvelope | undefined) => void> = []
|
||||||
@@ -616,6 +669,130 @@ describe("acp.agent event subscription", () => {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
test("emits image attachments as ACP tool content blocks on live completed tool updates", async () => {
|
||||||
|
await using tmp = await tmpdir()
|
||||||
|
await Instance.provide({
|
||||||
|
directory: tmp.path,
|
||||||
|
fn: async () => {
|
||||||
|
const { agent, controller, sessionUpdates, stop } = createFakeAgent()
|
||||||
|
const cwd = "/tmp/opencode-acp-test"
|
||||||
|
const sessionId = await agent.newSession({ cwd, mcpServers: [] } as any).then((x) => x.sessionId)
|
||||||
|
const data = Buffer.from("image-data").toString("base64")
|
||||||
|
|
||||||
|
controller.push(
|
||||||
|
completedToolEvent(sessionId, cwd, {
|
||||||
|
callID: "call_image",
|
||||||
|
tool: "read",
|
||||||
|
input: { filePath: "/tmp/image.png" },
|
||||||
|
output: "Image read successfully",
|
||||||
|
attachments: [
|
||||||
|
{
|
||||||
|
id: "part_image",
|
||||||
|
sessionID: sessionId,
|
||||||
|
messageID: "msg_image",
|
||||||
|
type: "file",
|
||||||
|
mime: "image/png",
|
||||||
|
filename: "image.png",
|
||||||
|
url: `data:image/png;base64,${data}`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "part_text",
|
||||||
|
sessionID: sessionId,
|
||||||
|
messageID: "msg_image",
|
||||||
|
type: "file",
|
||||||
|
mime: "text/plain",
|
||||||
|
filename: "note.txt",
|
||||||
|
url: "data:text/plain;base64,Zm9v",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
await new Promise((r) => setTimeout(r, 20))
|
||||||
|
|
||||||
|
const update = completedToolUpdate(sessionUpdates, sessionId, "call_image")
|
||||||
|
expect(update?.content).toContainEqual({
|
||||||
|
type: "content",
|
||||||
|
content: { type: "text", text: "Image read successfully" },
|
||||||
|
})
|
||||||
|
expect(update?.content).toContainEqual({
|
||||||
|
type: "content",
|
||||||
|
content: { type: "image", mimeType: "image/png", data },
|
||||||
|
})
|
||||||
|
expect(update?.content?.some((item) => item.type === "content" && item.content.type === "resource")).toBe(false)
|
||||||
|
expect((update?.rawOutput as { attachments?: unknown[] } | undefined)?.attachments?.length).toBe(2)
|
||||||
|
|
||||||
|
stop()
|
||||||
|
},
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
test("replays completed tool image attachments as ACP tool content blocks", async () => {
|
||||||
|
await using tmp = await tmpdir()
|
||||||
|
await Instance.provide({
|
||||||
|
directory: tmp.path,
|
||||||
|
fn: async () => {
|
||||||
|
const { agent, sessionUpdates, stop, sdk } = createFakeAgent()
|
||||||
|
const cwd = "/tmp/opencode-acp-test"
|
||||||
|
const sessionId = await agent.newSession({ cwd, mcpServers: [] } as any).then((x) => x.sessionId)
|
||||||
|
const data = Buffer.from("replay-image").toString("base64")
|
||||||
|
|
||||||
|
sdk.session.messages = async () => ({
|
||||||
|
data: [
|
||||||
|
{
|
||||||
|
info: {
|
||||||
|
role: "assistant",
|
||||||
|
sessionID: sessionId,
|
||||||
|
},
|
||||||
|
parts: [
|
||||||
|
{
|
||||||
|
id: "part_replay",
|
||||||
|
sessionID: sessionId,
|
||||||
|
messageID: "msg_replay",
|
||||||
|
type: "tool",
|
||||||
|
callID: "call_replay_image",
|
||||||
|
tool: "webfetch",
|
||||||
|
state: {
|
||||||
|
status: "completed",
|
||||||
|
input: { url: "https://example.com/image.png" },
|
||||||
|
output: "Image fetched successfully",
|
||||||
|
title: "webfetch",
|
||||||
|
metadata: {},
|
||||||
|
time: { start: Date.now() - 1, end: Date.now() },
|
||||||
|
attachments: [
|
||||||
|
{
|
||||||
|
id: "part_replay_image",
|
||||||
|
sessionID: sessionId,
|
||||||
|
messageID: "msg_replay",
|
||||||
|
type: "file",
|
||||||
|
mime: "image/jpeg",
|
||||||
|
filename: "image.jpg",
|
||||||
|
url: `data:image/jpeg;base64,${data}`,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
})
|
||||||
|
|
||||||
|
await agent.loadSession({ sessionId, cwd, mcpServers: [] } as any)
|
||||||
|
|
||||||
|
const update = completedToolUpdate(sessionUpdates, sessionId, "call_replay_image")
|
||||||
|
expect(update?.content).toContainEqual({
|
||||||
|
type: "content",
|
||||||
|
content: { type: "text", text: "Image fetched successfully" },
|
||||||
|
})
|
||||||
|
expect(update?.content).toContainEqual({
|
||||||
|
type: "content",
|
||||||
|
content: { type: "image", mimeType: "image/jpeg", data },
|
||||||
|
})
|
||||||
|
|
||||||
|
stop()
|
||||||
|
},
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
test("does not emit duplicate synthetic pending after replayed running tool", async () => {
|
test("does not emit duplicate synthetic pending after replayed running tool", async () => {
|
||||||
await using tmp = await tmpdir()
|
await using tmp = await tmpdir()
|
||||||
await WithInstance.provide({
|
await WithInstance.provide({
|
||||||
|
|||||||
Reference in New Issue
Block a user