feat(core): add fence to make all methods strongly consistent when syncing (#22679)

packages/opencode/src/server/fence.ts

@@ -0,0 +1,81 @@
+import type { MiddlewareHandler } from "hono"
+import { Database, inArray } from "@/storage/db"
+import { EventSequenceTable } from "@/sync/event.sql"
+import { Workspace } from "@/control-plane/workspace"
+import type { WorkspaceID } from "@/control-plane/schema"
+import { Log } from "@/util/log"
+
+const HEADER = "x-opencode-sync"
+type State = Record<string, number>
+const log = Log.create({ service: "fence" })
+
+export function load(ids?: string[]) {
+  const rows = Database.use((db) => {
+    if (!ids?.length) {
+      return db.select().from(EventSequenceTable).all()
+    }
+
+    return db.select().from(EventSequenceTable).where(inArray(EventSequenceTable.aggregate_id, ids)).all()
+  })
+
+  return Object.fromEntries(rows.map((row) => [row.aggregate_id, row.seq])) as State
+}
+
+export function diff(prev: State, next: State) {
+  const ids = new Set([...Object.keys(prev), ...Object.keys(next)])
+  return Object.fromEntries(
+    [...ids]
+      .map((id) => [id, next[id] ?? -1] as const)
+      .filter(([id, seq]) => {
+        return (prev[id] ?? -1) !== seq
+      }),
+  ) as State
+}
+
+export function parse(headers: Headers) {
+  const raw = headers.get(HEADER)
+  if (!raw) return
+
+  let data
+
+  try {
+    data = JSON.parse(raw)
+  } catch (err) {
+    return
+  }
+
+  if (!data || typeof data !== "object") return
+
+  return Object.fromEntries(
+    Object.entries(data).filter(([id, seq]) => {
+      return typeof id === "string" && Number.isInteger(seq)
+    }),
+  ) as State
+}
+
+export async function wait(workspaceID: WorkspaceID, state: State, signal?: AbortSignal) {
+  log.info("waiting for state", {
+    workspaceID,
+    state,
+  })
+  await Workspace.waitForSync(workspaceID, state, signal)
+  log.info("state fully synced", {
+    workspaceID,
+    state,
+  })
+}
+
+export const FenceMiddleware: MiddlewareHandler = async (c, next) => {
+  if (c.req.method === "GET" || c.req.method === "HEAD" || c.req.method === "OPTIONS") return next()
+
+  const prev = load()
+  await next()
+  const current = diff(prev, load())
+
+  if (Object.keys(current).length > 0) {
+    log.info("header", {
+      diff: current,
+    })
+    c.res.headers.set(HEADER, JSON.stringify(current))
+  }
+}

packages/opencode/src/server/instance/middleware.ts

@@ -6,6 +6,7 @@ import { Workspace } from "@/control-plane/workspace"
 import { ServerProxy } from "../proxy"
 import { Instance } from "@/project/instance"
 import { InstanceBootstrap } from "@/project/bootstrap"
+import { Flag } from "@/flag/flag"
 import { Session } from "@/session"
 import { SessionID } from "@/session/schema"
 import { WorkspaceContext } from "@/control-plane/workspace-context"
@@ -68,10 +69,10 @@ export function WorkspaceRouterMiddleware(upgrade: UpgradeWebSocket): Middleware
     const sessionWorkspaceID = await getSessionWorkspace(url)
     const workspaceID = sessionWorkspaceID || url.searchParams.get("workspace")
 
-    if (!workspaceID || url.pathname.startsWith("/console") || OPENCODE_WORKSPACE) {
-      if (OPENCODE_WORKSPACE) {
+    if (!workspaceID || url.pathname.startsWith("/console") || Flag.OPENCODE_WORKSPACE_ID) {
+      if (Flag.OPENCODE_WORKSPACE_ID) {
         return WorkspaceContext.provide({
-          workspaceID: WorkspaceID.make(OPENCODE_WORKSPACE),
+          workspaceID: WorkspaceID.make(Flag.OPENCODE_WORKSPACE_ID),
           async fn() {
             return Instance.provide({
               directory,
@@ -148,6 +149,6 @@ export function WorkspaceRouterMiddleware(upgrade: UpgradeWebSocket): Middleware
     headers.delete("x-opencode-workspace")
 
     const req = new Request(c.req.raw, { headers })
-    return ServerProxy.http(proxyURL, target.headers, req)
+    return ServerProxy.http(proxyURL, target.headers, req, workspace.id)
   }
 }

packages/opencode/src/server/proxy.ts

@@ -1,6 +1,9 @@
 import { Hono } from "hono"
 import type { UpgradeWebSocket } from "hono/ws"
 import { Log } from "@/util/log"
+import * as Fence from "./fence"
+import type { WorkspaceID } from "@/control-plane/schema"
+import { Workspace } from "@/control-plane/workspace"
 
 const hop = new Set([
   "connection",
@@ -101,12 +104,27 @@ const app = (upgrade: UpgradeWebSocket) =>
 export namespace ServerProxy {
   const log = Log.Default.clone().tag("service", "server-proxy")
 
-  export function http(url: string | URL, extra: HeadersInit | undefined, req: Request) {
+  export async function http(
+    url: string | URL,
+    extra: HeadersInit | undefined,
+    req: Request,
+    workspaceID: WorkspaceID,
+  ) {
     console.log("proxy http request", {
       method: req.method,
       request: req.url,
       url: String(url),
     })
+
+    if (!Workspace.isSyncing(workspaceID)) {
+      return new Response(`broken sync connection for workspace: ${workspaceID}`, {
+        status: 503,
+        headers: {
+          "content-type": "text/plain; charset=utf-8",
+        },
+      })
+    }
+
     return fetch(
       new Request(url, {
         method: req.method,
@@ -116,21 +134,26 @@ export namespace ServerProxy {
         signal: req.signal,
       }),
     ).then((res) => {
+      const sync = Fence.parse(res.headers)
       const next = new Headers(res.headers)
       next.delete("content-encoding")
       next.delete("content-length")
 
-      console.log("proxy http response", {
-        method: req.method,
-        request: req.url,
-        url: String(url),
-        status: res.status,
-        statusText: res.statusText,
-      })
-      return new Response(res.body, {
-        status: res.status,
-        statusText: res.statusText,
-        headers: next,
+      const done = sync ? Fence.wait(workspaceID, sync, req.signal) : Promise.resolve()
+
+      return done.then(async () => {
+        console.log("proxy http response", {
+          method: req.method,
+          request: req.url,
+          url: String(url),
+          status: res.status,
+          statusText: res.statusText,
+        })
+        return new Response(res.body, {
+          status: res.status,
+          statusText: res.statusText,
+          headers: next,
+        })
       })
     })
   }

packages/opencode/src/server/server.ts

@@ -4,9 +4,11 @@ import { adapter } from "#hono"
 import { MDNS } from "./mdns"
 import { lazy } from "@/util/lazy"
 import { AuthMiddleware, CompressionMiddleware, CorsMiddleware, ErrorMiddleware, LoggerMiddleware } from "./middleware"
+import { FenceMiddleware } from "./fence"
 import { InstanceRoutes } from "./instance"
 import { initProjectors } from "./projectors"
 import { Log } from "@/util/log"
+import { Flag } from "@/flag/flag"
 import { ControlPlaneRoutes } from "./control"
 import { UIRoutes } from "./ui"
 
@@ -30,6 +32,22 @@ export namespace Server {
   function create(opts: { cors?: string[] }) {
     const app = new Hono()
     const runtime = adapter.create(app)
+
+    if (Flag.OPENCODE_WORKSPACE_ID) {
+      return {
+        app: app
+          .onError(ErrorMiddleware)
+          .use(AuthMiddleware)
+          .use(LoggerMiddleware)
+          .use(CompressionMiddleware)
+          .use(CorsMiddleware(opts))
+          .use(FenceMiddleware)
+          .route("/", ControlPlaneRoutes())
+          .route("/", InstanceRoutes(runtime.upgradeWebSocket)),
+        runtime,
+      }
+    }
+
     return {
       app: app
         .onError(ErrorMiddleware)