lmin/zmo-cli
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity

feat: better image handling (auto resize & max size constraints) (#26401)

Browse Source
This commit is contained in:
Aiden Cline
2026-05-10 01:48:19 -05:00
committed by GitHub
parent 5217e6c1af
commit 85ce6a5f95
18 changed files with 417 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
packages/opencode/src/audio.d.ts vendored
View File

@@ -2,3 +2,8 @@ declare module "*.wav" {
const file: string
export default file
}
declare module "*.wasm" {
const file: string
export default file
}

30
packages/opencode/src/config/attachment.ts Normal file
View File

@@ -0,0 +1,30 @@
export * as ConfigAttachment from "./attachment"
import { Schema } from "effect"
import { zod } from "@opencode-ai/core/effect-zod"
import { PositiveInt, withStatics } from "@opencode-ai/core/schema"
export const Image = Schema.Struct({
auto_resize: Schema.optional(Schema.Boolean).annotate({
description: "Resize images before sending them to the model when they exceed configured limits (default: true)",
}),
max_width: Schema.optional(PositiveInt).annotate({
description: "Maximum image width before resizing or rejecting the attachment (default: 2000)",
}),
max_height: Schema.optional(PositiveInt).annotate({
description: "Maximum image height before resizing or rejecting the attachment (default: 2000)",
}),
max_base64_bytes: Schema.optional(PositiveInt).annotate({
description: "Maximum base64 payload bytes for an image attachment (default: 4718592)",
}),
})
.annotate({ identifier: "ImageAttachmentConfig" })
.pipe(withStatics((s) => ({ zod: zod(s) })))
export type Image = Schema.Schema.Type<typeof Image>
export const Info = Schema.Struct({
image: Schema.optional(Image).annotate({ description: "Image attachment configuration" }),
})
.annotate({ identifier: "AttachmentConfig" })
.pipe(withStatics((s) => ({ zod: zod(s) })))
export type Info = Schema.Schema.Type<typeof Info>

4
packages/opencode/src/config/config.ts
View File

@@ -25,6 +25,7 @@ import { containsPath } from "../project/instance-context"
import { zod } from "@opencode-ai/core/effect-zod"
import { NonNegativeInt, PositiveInt, withStatics, type DeepMutable } from "@opencode-ai/core/schema"
import { ConfigAgent } from "./agent"
import { ConfigAttachment } from "./attachment"
import { ConfigCommand } from "./command"
import { ConfigFormatter } from "./formatter"
import { ConfigLayout } from "./layout"
@@ -241,6 +242,9 @@ export const Info = Schema.Struct({
layout: Schema.optional(ConfigLayout.Layout).annotate({ description: "@deprecated Always uses stretch layout." }),
permission: Schema.optional(ConfigPermission.Info),
tools: Schema.optional(Schema.Record(Schema.String, Schema.Boolean)),
attachment: Schema.optional(ConfigAttachment.Info).annotate({
description: "Attachment processing configuration, including image size limits and resizing behavior",
}),
enterprise: Schema.optional(
Schema.Struct({
url: Schema.optional(Schema.String).annotate({ description: "Enterprise URL" }),

180
packages/opencode/src/image/image.ts Normal file
View File

@@ -0,0 +1,180 @@
import { Config } from "@/config/config"
import type { MessageV2 } from "@/session/message-v2"
import * as Log from "@opencode-ai/core/util/log"
import { Context, Effect, Layer, Schema } from "effect"
const MAX_BASE64_BYTES = 4.5 * 1024 * 1024
const MAX_WIDTH = 2000
const MAX_HEIGHT = 2000
const AUTO_RESIZE = true
const JPEG_QUALITIES = [80, 85, 70, 55, 40]
const log = Log.create({ service: "image" })
export class PhotonUnavailableError extends Schema.TaggedErrorClass<PhotonUnavailableError>()(
"ImagePhotonUnavailableError",
{},
) {
override get message() {
return "Photon image processor is unavailable"
}
}
export class InvalidDataUrlError extends Schema.TaggedErrorClass<InvalidDataUrlError>()("ImageInvalidDataUrlError", {
url: Schema.String,
}) {
override get message() {
return "Image URL must be a base64 data URL"
}
}
export class DecodeError extends Schema.TaggedErrorClass<DecodeError>()("ImageDecodeError", {}) {
override get message() {
return "Image could not be decoded"
}
}
export class SizeError extends Schema.TaggedErrorClass<SizeError>()("ImageSizeError", {
bytes: Schema.Number,
max: Schema.Number,
width: Schema.Number,
height: Schema.Number,
max_width: Schema.Number,
max_height: Schema.Number,
}) {
override get message() {
return `Image ${this.width}x${this.height} with base64 size ${this.bytes} exceeds configured limits and could not be resized below ${this.max_width}x${this.max_height}/${this.max} bytes`
}
}
export type Error = PhotonUnavailableError | InvalidDataUrlError | DecodeError | SizeError
export interface Interface {
readonly normalize: (input: MessageV2.FilePart) => Effect.Effect<MessageV2.FilePart, Error>
}
export class Service extends Context.Service<Service, Interface>()("@opencode/Image") {}
export const layer = Layer.effect(
Service,
Effect.gen(function* () {
const config = yield* Config.Service
const loadPhoton = yield* Effect.cached(
Effect.promise(async () => {
try {
const photonWasm = (await import("@silvia-odwyer/photon-node/photon_rs_bg.wasm", { with: { type: "file" } }))
.default
// Patched photon-node reads this during module init so Bun compiled binaries use the embedded wasm path.
;(globalThis as typeof globalThis & { __OPENCODE_PHOTON_WASM_PATH?: string }).__OPENCODE_PHOTON_WASM_PATH =
photonWasm
return await import("@silvia-odwyer/photon-node")
} catch {
return null
}
}),
)
const normalize = Effect.fn("Image.normalize")(function* (input: MessageV2.FilePart) {
const image = (yield* config.get()).attachment?.image
const info = {
autoResize: image?.auto_resize ?? AUTO_RESIZE,
maxWidth: image?.max_width ?? MAX_WIDTH,
maxHeight: image?.max_height ?? MAX_HEIGHT,
maxBase64Bytes: image?.max_base64_bytes ?? MAX_BASE64_BYTES,
}
if (!input.url.startsWith("data:") || !input.url.includes(";base64,"))
return yield* new InvalidDataUrlError({ url: input.url })
const base64 = input.url.slice(input.url.indexOf(";base64,") + ";base64,".length)
const photon = yield* loadPhoton
if (!photon) return yield* new PhotonUnavailableError()
const decoded = yield* Effect.sync(() => {
try {
return photon.PhotonImage.new_from_byteslice(Buffer.from(base64, "base64"))
} catch {
return undefined
}
})
if (!decoded) return yield* new DecodeError()
try {
const originalWidth = decoded.get_width()
const originalHeight = decoded.get_height()
if (
originalWidth <= info.maxWidth &&
originalHeight <= info.maxHeight &&
Buffer.byteLength(base64, "utf8") <= info.maxBase64Bytes
)
return input
if (!info.autoResize)
return yield* new SizeError({
bytes: Buffer.byteLength(base64, "utf8"),
max: info.maxBase64Bytes,
width: originalWidth,
height: originalHeight,
max_width: info.maxWidth,
max_height: info.maxHeight,
})
const scale = Math.min(1, info.maxWidth / originalWidth, info.maxHeight / originalHeight)
for (const size of Array.from({ length: 32 }).reduce<Array<{ width: number; height: number }>>((acc) => {
const previous = acc.at(-1) ?? {
width: Math.max(1, Math.round(originalWidth * scale)),
height: Math.max(1, Math.round(originalHeight * scale)),
}
const next =
acc.length === 0
? previous
: {
width: previous.width === 1 ? 1 : Math.max(1, Math.floor(previous.width * 0.75)),
height: previous.height === 1 ? 1 : Math.max(1, Math.floor(previous.height * 0.75)),
}
return acc.some((item) => item.width === next.width && item.height === next.height) ? acc : [...acc, next]
}, [])) {
const resized = photon.resize(decoded, size.width, size.height, photon.SamplingFilter.Lanczos3)
const candidate = [
{ data: Buffer.from(resized.get_bytes()).toString("base64"), mime: "image/png" },
...JPEG_QUALITIES.map((quality) => ({
data: Buffer.from(resized.get_bytes_jpeg(quality)).toString("base64"),
mime: "image/jpeg",
})),
]
.map((item) => ({ ...item, bytes: Buffer.byteLength(item.data, "utf8") }))
.find((item) => item.bytes <= info.maxBase64Bytes)
resized.free()
if (candidate) {
log.info("using resized image", {
from_mime: input.mime,
to_mime: candidate.mime,
from: `${originalWidth}x${originalHeight}`,
to: `${size.width}x${size.height}`,
})
return {
...input,
mime: candidate.mime,
url: `data:${candidate.mime};base64,${candidate.data}`,
}
}
}
return yield* new SizeError({
bytes: Buffer.byteLength(base64, "utf8"),
max: info.maxBase64Bytes,
width: originalWidth,
height: originalHeight,
max_width: info.maxWidth,
max_height: info.maxHeight,
})
} finally {
decoded.free()
}
})
return Service.of({ normalize })
}),
)
export const defaultLayer = layer.pipe(Layer.provide(Config.defaultLayer))
export * as Image from "./image"

42
packages/opencode/src/server/routes/instance/httpapi/handlers/session.ts
View File

@@ -203,13 +203,15 @@ export const sessionHandlers = HttpApiBuilder.group(InstanceHttpApi, "session",
params: { sessionID: SessionID }
payload: typeof InitPayload.Type
}) {
yield* promptSvc.command({
sessionID: ctx.params.sessionID,
messageID: ctx.payload.messageID,
model: `${ctx.payload.providerID}/${ctx.payload.modelID}`,
command: Command.Default.INIT,
arguments: "",
})
yield* promptSvc
.command({
sessionID: ctx.params.sessionID,
messageID: ctx.payload.messageID,
model: `${ctx.payload.providerID}/${ctx.payload.modelID}`,
command: Command.Default.INIT,
arguments: "",
})
.pipe(Effect.mapError(() => new HttpApiError.BadRequest({})))
return true
})
@@ -258,18 +260,18 @@ export const sessionHandlers = HttpApiBuilder.group(InstanceHttpApi, "session",
}) {
const instance = yield* InstanceState.context
const workspace = yield* InstanceState.workspaceID
const message = yield* promptSvc
.prompt({
...ctx.payload,
sessionID: ctx.params.sessionID,
})
.pipe(
Effect.provideService(InstanceRef, instance),
Effect.provideService(WorkspaceRef, workspace),
Effect.mapError(() => new HttpApiError.BadRequest({})),
)
return HttpServerResponse.stream(
Stream.fromEffect(
promptSvc
.prompt({
...ctx.payload,
sessionID: ctx.params.sessionID,
})
.pipe(Effect.provideService(InstanceRef, instance), Effect.provideService(WorkspaceRef, workspace)),
).pipe(
Stream.map((message) => JSON.stringify(message)),
Stream.encodeText,
),
Stream.make(JSON.stringify(message)).pipe(Stream.encodeText),
{ contentType: "application/json" },
)
})
@@ -297,7 +299,9 @@ export const sessionHandlers = HttpApiBuilder.group(InstanceHttpApi, "session",
params: { sessionID: SessionID }
payload: typeof CommandPayload.Type
}) {
return yield* promptSvc.command({ ...ctx.payload, sessionID: ctx.params.sessionID })
return yield* promptSvc
.command({ ...ctx.payload, sessionID: ctx.params.sessionID })
.pipe(Effect.mapError(() => new HttpApiError.BadRequest({})))
})
const shell = Effect.fn("SessionHttpApi.shell")(function* (ctx: {

40
packages/opencode/src/session/processor.ts
View File

@@ -1,4 +1,4 @@
import { Cause, Deferred, Effect, Layer, Context, Scope } from "effect"
import { Cause, Deferred, Effect, Exit, Layer, Context, Scope } from "effect"
import * as Stream from "effect/Stream"
import { Agent } from "@/agent/agent"
import { Bus } from "@/bus"
@@ -9,6 +9,7 @@ import { Snapshot } from "@/snapshot"
import * as Session from "./session"
import { LLM } from "./llm"
import { MessageV2 } from "./message-v2"
import { Image } from "@/image/image"
import { isOverflow } from "./overflow"
import { PartID } from "./schema"
import type { SessionID } from "./schema"
@@ -92,6 +93,7 @@ export const layer: Layer.Layer<
| LLM.Service
| Permission.Service
| Plugin.Service
| Image.Service
| SessionSummary.Service
| SessionStatus.Service
> = Layer.effect(
@@ -108,6 +110,7 @@ export const layer: Layer.Layer<
const summary = yield* SessionSummary.Service
const scope = yield* Scope.Scope
const status = yield* SessionStatus.Service
const image = yield* Image.Service
const create = Effect.fn("SessionProcessor.create")(function* (input: Input) {
// Pre-capture snapshot before the LLM stream starts. The AI SDK
@@ -377,17 +380,43 @@ export const layer: Layer.Layer<
case "tool-result": {
const toolCall = yield* readToolCall(value.toolCallId)
const toolAttachments: MessageV2.FilePart[] = (
Array.isArray(value.output.attachments) ? value.output.attachments : []
).filter(
(attachment: unknown): attachment is MessageV2.FilePart =>
isRecord(attachment) &&
attachment.type === "file" &&
typeof attachment.mime === "string" &&
typeof attachment.url === "string",
)
const normalized = yield* Effect.forEach(
toolAttachments,
(attachment) =>
attachment.mime.startsWith("image/")
? image.normalize(attachment).pipe(Effect.exit)
: Effect.succeed(Exit.succeed<MessageV2.FilePart>(attachment)),
)
const omitted = normalized.filter(Exit.isFailure).length
const attachments = normalized.filter(Exit.isSuccess).map((item) => item.value)
const output = {
...value.output,
output:
omitted === 0
? value.output.output
: `${value.output.output}\n\n[${omitted} image${omitted === 1 ? "" : "s"} omitted: could not be resized below the inline image size limit.]`,
attachments: attachments?.length ? attachments : undefined,
}
// TODO(v2): Temporary dual-write while migrating session messages to v2 events.
EventV2.run(SessionEvent.Tool.Success.Sync, {
sessionID: ctx.sessionID,
callID: value.toolCallId,
structured: value.output.metadata,
structured: output.metadata,
content: [
{
type: "text",
text: value.output.output,
text: output.output,
},
...(value.output.attachments?.map((item: MessageV2.FilePart) => ({
...(output.attachments?.map((item: MessageV2.FilePart) => ({
type: "file",
uri: item.url,
mime: item.mime,
@@ -399,7 +428,7 @@ export const layer: Layer.Layer<
},
timestamp: DateTime.makeUnsafe(Date.now()),
})
yield* completeToolCall(value.toolCallId, value.output)
yield* completeToolCall(value.toolCallId, output)
return
}
@@ -758,6 +787,7 @@ export const defaultLayer = Layer.suspend(() =>
Layer.provide(Plugin.defaultLayer),
Layer.provide(SessionSummary.defaultLayer),
Layer.provide(SessionStatus.defaultLayer),
Layer.provide(Image.defaultLayer),
Layer.provide(Bus.layer),
Layer.provide(Config.defaultLayer),
),

21
packages/opencode/src/session/prompt.ts
View File

@@ -43,6 +43,7 @@ import { Shell } from "@/shell/shell"
import { ShellID } from "@/tool/shell/id"
import { AppFileSystem } from "@opencode-ai/core/filesystem"
import { Truncate } from "@/tool/truncate"
import { Image } from "@/image/image"
import { decodeDataUrl } from "@/util/data-url"
import { Process } from "@/util/process"
import { Cause, Effect, Exit, Latch, Layer, Option, Scope, Context, Schema, Types } from "effect"
@@ -80,10 +81,10 @@ const elog = EffectLogger.create({ service: "session.prompt" })
export interface Interface {
readonly cancel: (sessionID: SessionID) => Effect.Effect<void>
readonly prompt: (input: PromptInput) => Effect.Effect<MessageV2.WithParts>
readonly prompt: (input: PromptInput) => Effect.Effect<MessageV2.WithParts, Image.Error>
readonly loop: (input: LoopInput) => Effect.Effect<MessageV2.WithParts>
readonly shell: (input: ShellInput) => Effect.Effect<MessageV2.WithParts>
readonly command: (input: CommandInput) => Effect.Effect<MessageV2.WithParts>
readonly command: (input: CommandInput) => Effect.Effect<MessageV2.WithParts, Image.Error>
readonly resolvePromptParts: (template: string) => Effect.Effect<PromptInput["parts"]>
}
@@ -108,6 +109,7 @@ export const layer = Layer.effect(
const lsp = yield* LSP.Service
const registry = yield* ToolRegistry.Service
const truncate = yield* Truncate.Service
const image = yield* Image.Service
const spawner = yield* ChildProcessSpawner.ChildProcessSpawner
const scope = yield* Scope.Scope
const instruction = yield* Instruction.Service
@@ -123,7 +125,7 @@ export const layer = Layer.effect(
return {
cancel: (sessionID: SessionID) => cancel(sessionID),
resolvePromptParts: (template: string) => resolvePromptParts(template),
prompt: (input: PromptInput) => prompt(input),
prompt: (input: PromptInput) => prompt(input).pipe(Effect.catch(Effect.die)),
} satisfies TaskPromptOps
})
@@ -1259,7 +1261,7 @@ NOTE: At any point in time through this workflow you should feel free to ask the
return [{ ...part, messageID: info.id, sessionID: input.sessionID }]
})
const parts = yield* Effect.forEach(input.parts, resolvePart, { concurrency: "unbounded" }).pipe(
const resolvedParts = yield* Effect.forEach(input.parts, resolvePart, { concurrency: "unbounded" }).pipe(
Effect.map((x) => x.flat().map(assign)),
)
@@ -1272,7 +1274,13 @@ NOTE: At any point in time through this workflow you should feel free to ask the
messageID: input.messageID,
variant: input.variant,
},
{ message: info, parts },
{ message: info, parts: resolvedParts },
)
const parts = yield* Effect.forEach(resolvedParts, (part) =>
part.type === "file" && part.mime.startsWith("image/")
? image.normalize(part)
: Effect.succeed(part),
)
const parsed = MessageV2.Info.zod.safeParse(info)
@@ -1368,7 +1376,7 @@ NOTE: At any point in time through this workflow you should feel free to ask the
return { info, parts }
}, Effect.scoped)
const prompt: (input: PromptInput) => Effect.Effect<MessageV2.WithParts> = Effect.fn("SessionPrompt.prompt")(
const prompt: (input: PromptInput) => Effect.Effect<MessageV2.WithParts, Image.Error> = Effect.fn("SessionPrompt.prompt")(
function* (input: PromptInput) {
const session = yield* sessions.get(input.sessionID).pipe(Effect.orDie)
yield* revert.cleanup(session)
@@ -1788,6 +1796,7 @@ export const defaultLayer = Layer.suspend(() =>
Layer.provide(Session.defaultLayer),
Layer.provide(SessionRevert.defaultLayer),
Layer.provide(SessionSummary.defaultLayer),
Layer.provide(Image.defaultLayer),
Layer.provide(
Layer.mergeAll(
Agent.defaultLayer,