lmin/zmo-cli
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity

feat: better image handling (auto resize & max size constraints) (#26401)

Browse Source
This commit is contained in:
Aiden Cline
2026-05-10 01:48:19 -05:00
committed by GitHub
parent 5217e6c1af
commit 85ce6a5f95
18 changed files with 417 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
packages/opencode/src/session/processor.ts
View File

@@ -1,4 +1,4 @@
import { Cause, Deferred, Effect, Layer, Context, Scope } from "effect"
import { Cause, Deferred, Effect, Exit, Layer, Context, Scope } from "effect"
import * as Stream from "effect/Stream"
import { Agent } from "@/agent/agent"
import { Bus } from "@/bus"
@@ -9,6 +9,7 @@ import { Snapshot } from "@/snapshot"
import * as Session from "./session"
import { LLM } from "./llm"
import { MessageV2 } from "./message-v2"
import { Image } from "@/image/image"
import { isOverflow } from "./overflow"
import { PartID } from "./schema"
import type { SessionID } from "./schema"
@@ -92,6 +93,7 @@ export const layer: Layer.Layer<
| LLM.Service
| Permission.Service
| Plugin.Service
| Image.Service
| SessionSummary.Service
| SessionStatus.Service
> = Layer.effect(
@@ -108,6 +110,7 @@ export const layer: Layer.Layer<
const summary = yield* SessionSummary.Service
const scope = yield* Scope.Scope
const status = yield* SessionStatus.Service
const image = yield* Image.Service
const create = Effect.fn("SessionProcessor.create")(function* (input: Input) {
// Pre-capture snapshot before the LLM stream starts. The AI SDK
@@ -377,17 +380,43 @@ export const layer: Layer.Layer<
case "tool-result": {
const toolCall = yield* readToolCall(value.toolCallId)
const toolAttachments: MessageV2.FilePart[] = (
Array.isArray(value.output.attachments) ? value.output.attachments : []
).filter(
(attachment: unknown): attachment is MessageV2.FilePart =>
isRecord(attachment) &&
attachment.type === "file" &&
typeof attachment.mime === "string" &&
typeof attachment.url === "string",
)
const normalized = yield* Effect.forEach(
toolAttachments,
(attachment) =>
attachment.mime.startsWith("image/")
? image.normalize(attachment).pipe(Effect.exit)
: Effect.succeed(Exit.succeed<MessageV2.FilePart>(attachment)),
)
const omitted = normalized.filter(Exit.isFailure).length
const attachments = normalized.filter(Exit.isSuccess).map((item) => item.value)
const output = {
...value.output,
output:
omitted === 0
? value.output.output
: `${value.output.output}\n\n[${omitted} image${omitted === 1 ? "" : "s"} omitted: could not be resized below the inline image size limit.]`,
attachments: attachments?.length ? attachments : undefined,
}
// TODO(v2): Temporary dual-write while migrating session messages to v2 events.
EventV2.run(SessionEvent.Tool.Success.Sync, {
sessionID: ctx.sessionID,
callID: value.toolCallId,
structured: value.output.metadata,
structured: output.metadata,
content: [
{
type: "text",
text: value.output.output,
text: output.output,
},
...(value.output.attachments?.map((item: MessageV2.FilePart) => ({
...(output.attachments?.map((item: MessageV2.FilePart) => ({
type: "file",
uri: item.url,
mime: item.mime,
@@ -399,7 +428,7 @@ export const layer: Layer.Layer<
},
timestamp: DateTime.makeUnsafe(Date.now()),
})
yield* completeToolCall(value.toolCallId, value.output)
yield* completeToolCall(value.toolCallId, output)
return
}
@@ -758,6 +787,7 @@ export const defaultLayer = Layer.suspend(() =>
Layer.provide(Plugin.defaultLayer),
Layer.provide(SessionSummary.defaultLayer),
Layer.provide(SessionStatus.defaultLayer),
Layer.provide(Image.defaultLayer),
Layer.provide(Bus.layer),
Layer.provide(Config.defaultLayer),
),

21
packages/opencode/src/session/prompt.ts
View File

@@ -43,6 +43,7 @@ import { Shell } from "@/shell/shell"
import { ShellID } from "@/tool/shell/id"
import { AppFileSystem } from "@opencode-ai/core/filesystem"
import { Truncate } from "@/tool/truncate"
import { Image } from "@/image/image"
import { decodeDataUrl } from "@/util/data-url"
import { Process } from "@/util/process"
import { Cause, Effect, Exit, Latch, Layer, Option, Scope, Context, Schema, Types } from "effect"
@@ -80,10 +81,10 @@ const elog = EffectLogger.create({ service: "session.prompt" })
export interface Interface {
readonly cancel: (sessionID: SessionID) => Effect.Effect<void>
readonly prompt: (input: PromptInput) => Effect.Effect<MessageV2.WithParts>
readonly prompt: (input: PromptInput) => Effect.Effect<MessageV2.WithParts, Image.Error>
readonly loop: (input: LoopInput) => Effect.Effect<MessageV2.WithParts>
readonly shell: (input: ShellInput) => Effect.Effect<MessageV2.WithParts>
readonly command: (input: CommandInput) => Effect.Effect<MessageV2.WithParts>
readonly command: (input: CommandInput) => Effect.Effect<MessageV2.WithParts, Image.Error>
readonly resolvePromptParts: (template: string) => Effect.Effect<PromptInput["parts"]>
}
@@ -108,6 +109,7 @@ export const layer = Layer.effect(
const lsp = yield* LSP.Service
const registry = yield* ToolRegistry.Service
const truncate = yield* Truncate.Service
const image = yield* Image.Service
const spawner = yield* ChildProcessSpawner.ChildProcessSpawner
const scope = yield* Scope.Scope
const instruction = yield* Instruction.Service
@@ -123,7 +125,7 @@ export const layer = Layer.effect(
return {
cancel: (sessionID: SessionID) => cancel(sessionID),
resolvePromptParts: (template: string) => resolvePromptParts(template),
prompt: (input: PromptInput) => prompt(input),
prompt: (input: PromptInput) => prompt(input).pipe(Effect.catch(Effect.die)),
} satisfies TaskPromptOps
})
@@ -1259,7 +1261,7 @@ NOTE: At any point in time through this workflow you should feel free to ask the
return [{ ...part, messageID: info.id, sessionID: input.sessionID }]
})
const parts = yield* Effect.forEach(input.parts, resolvePart, { concurrency: "unbounded" }).pipe(
const resolvedParts = yield* Effect.forEach(input.parts, resolvePart, { concurrency: "unbounded" }).pipe(
Effect.map((x) => x.flat().map(assign)),
)
@@ -1272,7 +1274,13 @@ NOTE: At any point in time through this workflow you should feel free to ask the
messageID: input.messageID,
variant: input.variant,
},
{ message: info, parts },
{ message: info, parts: resolvedParts },
)
const parts = yield* Effect.forEach(resolvedParts, (part) =>
part.type === "file" && part.mime.startsWith("image/")
? image.normalize(part)
: Effect.succeed(part),
)
const parsed = MessageV2.Info.zod.safeParse(info)
@@ -1368,7 +1376,7 @@ NOTE: At any point in time through this workflow you should feel free to ask the
return { info, parts }
}, Effect.scoped)
const prompt: (input: PromptInput) => Effect.Effect<MessageV2.WithParts> = Effect.fn("SessionPrompt.prompt")(
const prompt: (input: PromptInput) => Effect.Effect<MessageV2.WithParts, Image.Error> = Effect.fn("SessionPrompt.prompt")(
function* (input: PromptInput) {
const session = yield* sessions.get(input.sessionID).pipe(Effect.orDie)
yield* revert.cleanup(session)
@@ -1788,6 +1796,7 @@ export const defaultLayer = Layer.suspend(() =>
Layer.provide(Session.defaultLayer),
Layer.provide(SessionRevert.defaultLayer),
Layer.provide(SessionSummary.defaultLayer),
Layer.provide(Image.defaultLayer),
Layer.provide(
Layer.mergeAll(
Agent.defaultLayer,