fix(vcs): avoid unbounded diff memory usage (#25581)

packages/opencode/src/project/vcs.ts

@@ -1,10 +1,8 @@
 import { Effect, Layer, Context, Schema, Stream, Scope } from "effect"
 import { formatPatch, structuredPatch } from "diff"
-import path from "path"
 import { Bus } from "@/bus"
 import { BusEvent } from "@/bus/bus-event"
 import { InstanceState } from "@/effect/instance-state"
-import { AppFileSystem } from "@opencode-ai/core/filesystem"
 import { FileWatcher } from "@/file/watcher"
 import { Git } from "@/git"
 import * as Log from "@opencode-ai/core/util/log"
@@ -12,20 +10,11 @@ import { zod } from "@/util/effect-zod"
 import { NonNegativeInt, withStatics } from "@/util/schema"
 
 const log = Log.create({ service: "vcs" })
+const PATCH_CONTEXT_LINES = 2_147_483_647
+const MAX_PATCH_BYTES = 10_000_000
+const MAX_TOTAL_PATCH_BYTES = 10_000_000
 
-const count = (text: string) => {
-  if (!text) return 0
-  if (!text.endsWith("\n")) return text.split("\n").length
-  return text.slice(0, -1).split("\n").length
-}
-
-const work = Effect.fnUntraced(function* (fs: AppFileSystem.Interface, cwd: string, file: string) {
-  const full = path.join(cwd, file)
-  if (!(yield* fs.exists(full).pipe(Effect.orDie))) return ""
-  const buf = yield* fs.readFile(full).pipe(Effect.catch(() => Effect.succeed(new Uint8Array())))
-  if (Buffer.from(buf).includes(0)) return ""
-  return Buffer.from(buf).toString("utf8")
-})
+const emptyPatch = (file: string) => formatPatch(structuredPatch(file, file, "", "", "", "", { context: 0 }))
 
 const nums = (list: Git.Stat[]) =>
   new Map(list.map((item) => [item.file, { additions: item.additions, deletions: item.deletions }] as const))
@@ -38,59 +27,168 @@ const merge = (...lists: Git.Item[][]) => {
   return [...out.values()]
 }
 
+const emptyBatch = () => ({ patches: new Map<string, string>(), capped: false })
+
+const parseQuotedPath = (value: string) => {
+  let out = ""
+  for (let idx = 1; idx < value.length; idx++) {
+    const char = value[idx]
+    if (char === '"') return { value: out, end: idx + 1 }
+    if (char !== "\\") {
+      out += char
+      continue
+    }
+
+    const next = value[++idx]
+    if (next === "t") out += "\t"
+    else if (next === "n") out += "\n"
+    else if (next === "r") out += "\r"
+    else if (next === '"' || next === "\\") out += next
+    else out += next ?? ""
+  }
+}
+
+const parsePathToken = (value: string) => {
+  if (!value.startsWith('"')) return value.split("\t")[0]
+  return parseQuotedPath(value)?.value ?? value
+}
+
+const fileFromDiffPath = (value: string | undefined) => {
+  if (!value || value === "/dev/null") return
+  const file = parsePathToken(value)
+  if (file.startsWith("a/") || file.startsWith("b/")) return file.slice(2)
+  return file
+}
+
+const fileFromGitHeader = (header: string) => {
+  if (header.startsWith('"')) {
+    const first = parseQuotedPath(header)
+    const second = first ? header.slice(first.end).trimStart() : undefined
+    if (!second) return
+    if (!second.startsWith('"')) return fileFromDiffPath(second)
+    return fileFromDiffPath(parseQuotedPath(second)?.value)
+  }
+
+  const separator = header.indexOf(" b/")
+  if (separator === -1) return
+  return fileFromDiffPath(header.slice(separator + 1))
+}
+
+const fileFromPatchChunk = (chunk: string) => {
+  const next = /^\+\+\+ (.+)$/m.exec(chunk)?.[1]
+  const before = /^--- (.+)$/m.exec(chunk)?.[1]
+  const file = fileFromDiffPath(next) ?? fileFromDiffPath(before)
+  if (file) return file
+
+  const header = /^diff --git (.+)$/m.exec(chunk)?.[1]
+  return fileFromGitHeader(header ?? "")
+}
+
+const splitGitPatch = (patch: Git.Patch) => {
+  const starts = [...patch.text.matchAll(/^diff --git /gm)].map((match) => match.index)
+  const chunks = starts.map((start, index) => patch.text.slice(start, starts[index + 1] ?? patch.text.length))
+  if (!patch.truncated) return chunks
+  return chunks.slice(0, -1)
+}
+
+const batchPatches = Effect.fnUntraced(function* (git: Git.Interface, cwd: string, ref: string, list: Git.Item[]) {
+  if (list.length === 0) return { patches: new Map<string, string>(), capped: false }
+
+  const result = yield* git.patchAll(cwd, ref, {
+    context: PATCH_CONTEXT_LINES,
+    maxOutputBytes: MAX_TOTAL_PATCH_BYTES,
+  })
+  if (result.truncated) log.warn("batched patch exceeded byte limit", { max: MAX_TOTAL_PATCH_BYTES })
+
+  return {
+    patches: splitGitPatch(result).reduce((acc, patch, index) => {
+      const file = fileFromPatchChunk(patch) ?? list[index]?.file
+      if (!file) return acc
+      acc.set(file, (acc.get(file) ?? "") + patch)
+      return acc
+    }, new Map<string, string>()),
+    capped: result.truncated,
+  }
+})
+
+const nativePatch = Effect.fnUntraced(function* (
+  git: Git.Interface,
+  cwd: string,
+  ref: string | undefined,
+  item: Git.Item,
+) {
+  const result =
+    item.code === "??" || !ref
+      ? yield* git.patchUntracked(cwd, item.file, { context: PATCH_CONTEXT_LINES, maxOutputBytes: MAX_PATCH_BYTES })
+      : yield* git.patch(cwd, ref, item.file, { context: PATCH_CONTEXT_LINES, maxOutputBytes: MAX_PATCH_BYTES })
+  if (!result.truncated && result.text) return result.text
+
+  if (result.truncated) log.warn("patch exceeded byte limit", { file: item.file, max: MAX_PATCH_BYTES })
+  return emptyPatch(item.file)
+})
+
+const totalPatch = (file: string, patch: string, total: number) => {
+  if (total + Buffer.byteLength(patch) <= MAX_TOTAL_PATCH_BYTES) return { patch, capped: false }
+  log.warn("total patch budget exceeded", { file, max: MAX_TOTAL_PATCH_BYTES })
+  return { patch: emptyPatch(file), capped: true }
+}
+
+const patchForItem = Effect.fnUntraced(function* (
+  git: Git.Interface,
+  cwd: string,
+  ref: string | undefined,
+  item: Git.Item,
+  batch: { patches: Map<string, string>; capped: boolean },
+  capped: boolean,
+) {
+  if (capped) return emptyPatch(item.file)
+
+  const batched = batch.patches.get(item.file)
+  if (batched !== undefined) return batched
+  if (item.code !== "??" && batch.capped) return emptyPatch(item.file)
+  return yield* nativePatch(git, cwd, ref, item)
+})
+
 const files = Effect.fnUntraced(function* (
-  fs: AppFileSystem.Interface,
   git: Git.Interface,
   cwd: string,
   ref: string | undefined,
   list: Git.Item[],
   map: Map<string, { additions: number; deletions: number }>,
+  batch: { patches: Map<string, string>; capped: boolean },
 ) {
-  const base = ref ? yield* git.prefix(cwd) : ""
-  const patch = (file: string, before: string, after: string) =>
-    formatPatch(structuredPatch(file, file, before, after, "", "", { context: Number.MAX_SAFE_INTEGER }))
-  const next = yield* Effect.forEach(
-    list,
-    (item) =>
-      Effect.gen(function* () {
-        const before = item.status === "added" || !ref ? "" : yield* git.show(cwd, ref, item.file, base)
-        const after = item.status === "deleted" ? "" : yield* work(fs, cwd, item.file)
-        const stat = map.get(item.file)
-        return {
-          file: item.file,
-          patch: patch(item.file, before, after),
-          additions: stat?.additions ?? (item.status === "added" ? count(after) : 0),
-          deletions: stat?.deletions ?? (item.status === "deleted" ? count(before) : 0),
-          status: item.status,
-        } satisfies FileDiff
-      }),
-    { concurrency: 8 },
-  )
-  return next.toSorted((a, b) => a.file.localeCompare(b.file))
+  const next: FileDiff[] = []
+  let total = 0
+  let capped = false
+
+  for (const item of list.toSorted((a, b) => a.file.localeCompare(b.file))) {
+    const stat = map.get(item.file) ?? (item.status === "added" ? yield* git.statUntracked(cwd, item.file) : undefined)
+    const patch = yield* patchForItem(git, cwd, ref, item, batch, capped)
+    const result: { patch: string; capped: boolean } = capped
+      ? { patch, capped: true }
+      : totalPatch(item.file, patch, total)
+    capped = capped || result.capped
+    if (!capped) {
+      total += Buffer.byteLength(result.patch)
+      capped = total >= MAX_TOTAL_PATCH_BYTES
+    }
+    next.push({
+      file: item.file,
+      patch: result.patch,
+      additions: stat?.additions ?? 0,
+      deletions: stat?.deletions ?? 0,
+      status: item.status,
+    })
+  }
+
+  return next
 })
 
-const track = Effect.fnUntraced(function* (
-  fs: AppFileSystem.Interface,
-  git: Git.Interface,
-  cwd: string,
-  ref: string | undefined,
-) {
-  if (!ref) return yield* files(fs, git, cwd, ref, yield* git.status(cwd), new Map())
-  const [list, stats] = yield* Effect.all([git.status(cwd), git.stats(cwd, ref)], { concurrency: 2 })
-  return yield* files(fs, git, cwd, ref, list, nums(stats))
-})
-
-const compare = Effect.fnUntraced(function* (
-  fs: AppFileSystem.Interface,
-  git: Git.Interface,
-  cwd: string,
-  ref: string,
-) {
+const diffAgainstRef = Effect.fnUntraced(function* (git: Git.Interface, cwd: string, ref: string) {
   const [list, stats, extra] = yield* Effect.all([git.diff(cwd, ref), git.stats(cwd, ref), git.status(cwd)], {
     concurrency: 3,
   })
   return yield* files(
-    fs,
     git,
     cwd,
     ref,
@@ -99,9 +197,15 @@ const compare = Effect.fnUntraced(function* (
       extra.filter((item) => item.code === "??"),
     ),
     nums(stats),
+    yield* batchPatches(git, cwd, ref, list),
   )
 })
 
+const track = Effect.fnUntraced(function* (git: Git.Interface, cwd: string, ref: string | undefined) {
+  if (!ref) return yield* files(git, cwd, ref, yield* git.status(cwd), new Map(), emptyBatch())
+  return yield* diffAgainstRef(git, cwd, ref)
+})
+
 export const Mode = Schema.Literals(["git", "branch"]).pipe(withStatics((s) => ({ zod: zod(s) })))
 export type Mode = Schema.Schema.Type<typeof Mode>
 
@@ -147,10 +251,9 @@ interface State {
 
 export class Service extends Context.Service<Service, Interface>()("@opencode/Vcs") {}
 
-export const layer: Layer.Layer<Service, never, AppFileSystem.Service | Git.Service | Bus.Service> = Layer.effect(
+export const layer: Layer.Layer<Service, never, Git.Service | Bus.Service> = Layer.effect(
   Service,
   Effect.gen(function* () {
-    const fs = yield* AppFileSystem.Service
     const git = yield* Git.Service
     const bus = yield* Bus.Service
     const scope = yield* Scope.Scope
@@ -204,23 +307,19 @@ export const layer: Layer.Layer<Service, never, AppFileSystem.Service | Git.Serv
         const ctx = yield* InstanceState.context
         if (ctx.project.vcs !== "git") return []
         if (mode === "git") {
-          return yield* track(fs, git, ctx.directory, (yield* git.hasHead(ctx.directory)) ? "HEAD" : undefined)
+          return yield* track(git, ctx.directory, (yield* git.hasHead(ctx.directory)) ? "HEAD" : undefined)
         }
 
         if (!value.root) return []
         if (value.current && value.current === value.root.name) return []
         const ref = yield* git.mergeBase(ctx.directory, value.root.ref)
         if (!ref) return []
-        return yield* compare(fs, git, ctx.directory, ref)
+        return yield* diffAgainstRef(git, ctx.directory, ref)
       }),
     })
   }),
 )
 
-export const defaultLayer = layer.pipe(
-  Layer.provide(Git.defaultLayer),
-  Layer.provide(AppFileSystem.defaultLayer),
-  Layer.provide(Bus.layer),
-)
+export const defaultLayer = layer.pipe(Layer.provide(Git.defaultLayer), Layer.provide(Bus.layer))
 
 export * as Vcs from "./vcs"