feat: initial datalake and stats site (#28666)

packages/stats/server/Dockerfile

@@ -0,0 +1,43 @@
+FROM oven/bun:1.3.14-alpine
+
+WORKDIR /app
+
+ENV NODE_ENV=production
+ENV BUN_RUNTIME_TRANSPILER_CACHE_PATH=0
+
+COPY package.json bun.lock ./
+COPY patches ./patches
+COPY packages/app/package.json ./packages/app/package.json
+COPY packages/console/app/package.json ./packages/console/app/package.json
+COPY packages/console/core/package.json ./packages/console/core/package.json
+COPY packages/console/function/package.json ./packages/console/function/package.json
+COPY packages/console/mail/package.json ./packages/console/mail/package.json
+COPY packages/console/resource/package.json ./packages/console/resource/package.json
+COPY packages/core/package.json ./packages/core/package.json
+COPY packages/desktop/package.json ./packages/desktop/package.json
+COPY packages/effect-drizzle-sqlite/package.json ./packages/effect-drizzle-sqlite/package.json
+COPY packages/enterprise/package.json ./packages/enterprise/package.json
+COPY packages/function/package.json ./packages/function/package.json
+COPY packages/http-recorder/package.json ./packages/http-recorder/package.json
+COPY packages/llm/package.json ./packages/llm/package.json
+COPY packages/opencode/package.json ./packages/opencode/package.json
+COPY packages/plugin/package.json ./packages/plugin/package.json
+COPY packages/script/package.json ./packages/script/package.json
+COPY packages/sdk/js/package.json ./packages/sdk/js/package.json
+COPY packages/slack/package.json ./packages/slack/package.json
+COPY packages/stats/app/package.json ./packages/stats/app/package.json
+COPY packages/stats/core/package.json ./packages/stats/core/package.json
+COPY packages/stats/server/package.json ./packages/stats/server/package.json
+COPY packages/storybook/package.json ./packages/storybook/package.json
+COPY packages/ui/package.json ./packages/ui/package.json
+COPY packages/web/package.json ./packages/web/package.json
+
+RUN bun install --frozen-lockfile --production --ignore-scripts
+
+COPY packages ./packages
+
+WORKDIR /app/packages/stats/server
+
+EXPOSE 3000
+
+CMD ["bun", "src/server.ts"]

packages/stats/server/package.json

@@ -0,0 +1,33 @@
+{
+  "$schema": "https://json.schemastore.org/package.json",
+  "name": "@opencode-ai/stats-server",
+  "version": "1.14.50",
+  "private": true,
+  "type": "module",
+  "license": "MIT",
+  "main": "./src/server.ts",
+  "exports": {
+    ".": "./src/server.ts"
+  },
+  "scripts": {
+    "start": "bun src/server.ts",
+    "typecheck": "tsgo --noEmit"
+  },
+  "dependencies": {
+    "@aws-sdk/client-firehose": "3.933.0",
+    "@effect/platform-node": "catalog:",
+    "@opencode-ai/stats-core": "workspace:*",
+    "effect": "catalog:",
+    "sst": "catalog:"
+  },
+  "devDependencies": {
+    "@tsconfig/node22": "catalog:",
+    "@types/bun": "catalog:",
+    "@types/node": "catalog:",
+    "@typescript/native-preview": "catalog:",
+    "typescript": "catalog:"
+  },
+  "engines": {
+    "node": ">=22"
+  }
+}

packages/stats/server/src/ingest.ts

@@ -0,0 +1,110 @@
+import { Buffer } from "node:buffer"
+import { FirehoseClient, PutRecordBatchCommand } from "@aws-sdk/client-firehose"
+import { Effect, Layer, Schema } from "effect"
+import * as Context from "effect/Context"
+import { Resource } from "sst/resource"
+
+const MAX_FIREHOSE_BATCH_SIZE = 500
+const MAX_FIREHOSE_ATTEMPTS = 3
+const LAKE_TYPE = /^([A-Za-z0-9_]+)\.([A-Za-z0-9_]+)$/
+
+type IngestEvent = Record<string, unknown>
+type RoutedEvent = IngestEvent & { _lake_database: string; _lake_table: string; _lake_operation: "insert" }
+type FirehoseRecord = { Data: Uint8Array }
+
+export class IngestError extends Schema.TaggedErrorClass<IngestError>()("IngestError", {
+  message: Schema.String,
+  failed: Schema.Number,
+  cause: Schema.optional(Schema.Defect),
+}) {}
+
+export declare namespace Ingest {
+  export interface Service {
+    readonly write: (events: IngestEvent[]) => Effect.Effect<{ records: number }, IngestError>
+  }
+}
+
+export class Ingest extends Context.Service<Ingest, Ingest.Service>()("@opencode/stats/Ingest") {
+  static readonly layer: Layer.Layer<Ingest> = Layer.effect(
+    Ingest,
+    Effect.sync(() => {
+      const client = new FirehoseClient({})
+
+      const write = Effect.fn("Ingest.write")(function* (events: IngestEvent[]) {
+        if (events.length === 0) return { records: 0 }
+        const records = events.map(routeEvent).filter((event): event is RoutedEvent => Boolean(event))
+        if (records.length !== events.length) {
+          return yield* new IngestError({
+            message: "Unsupported lake event type",
+            failed: events.length - records.length,
+          })
+        }
+
+        const failed = (
+          yield* Effect.all(
+            chunks(
+              records.map((event) => ({ Data: Buffer.from(JSON.stringify(event)) })),
+              MAX_FIREHOSE_BATCH_SIZE,
+            ).map((batch) => putRecords(client, Resource.LakeIngestConfig.streamName, batch)),
+            { concurrency: 8 },
+          )
+        ).reduce((sum, item) => sum + item, 0)
+
+        if (failed > 0) {
+          return yield* new IngestError({ message: "Failed to ingest all lake records", failed })
+        }
+
+        return { records: records.length }
+      })
+
+      return Ingest.of({ write })
+    }),
+  )
+}
+
+const putRecords: (
+  client: FirehoseClient,
+  streamName: string,
+  records: FirehoseRecord[],
+  attempt?: number,
+) => Effect.Effect<number, IngestError> = Effect.fn("Ingest.putRecords")(function* (
+  client,
+  streamName,
+  records,
+  attempt = 1,
+) {
+  const result = yield* Effect.tryPromise({
+    try: () => client.send(new PutRecordBatchCommand({ DeliveryStreamName: streamName, Records: records })),
+    catch: (cause) => new IngestError({ message: "Failed to write lake records to Firehose", failed: records.length, cause }),
+  })
+  const failed =
+    result.RequestResponses?.flatMap((item, index) => {
+      const record = records[index]
+      if (!item.ErrorCode || !record) return []
+      return [record]
+    }) ?? []
+
+  if (failed.length === 0) return 0
+  if (attempt >= MAX_FIREHOSE_ATTEMPTS) return failed.length
+
+  yield* Effect.sleep(`${250 * 2 ** (attempt - 1)} millis`)
+  return yield* putRecords(client, streamName, failed, attempt + 1)
+})
+
+function routeEvent(event: IngestEvent): RoutedEvent | undefined {
+  if (typeof event._datalake_key !== "string") return
+  const match = event._datalake_key.match(LAKE_TYPE)
+  if (!match?.[1] || !match[2]) return
+  return {
+    ...Object.fromEntries(Object.entries(event).filter(([key]) => key !== "_datalake_key")),
+    _lake_database: match[1],
+    _lake_table: match[2],
+    _lake_operation: "insert" as const,
+  }
+}
+
+function chunks<T>(items: T[], size: number) {
+  return Array.from({ length: Math.ceil(items.length / size) }, (_, index) =>
+    items.slice(index * size, (index + 1) * size),
+  )
+}

packages/stats/server/src/resource.d.ts

@@ -0,0 +1,11 @@
+import "sst/resource"
+
+declare module "sst/resource" {
+  export interface Resource {
+    LakeIngestConfig: {
+      secret: string
+      streamName: string
+      type: "sst.sst.Linkable"
+    }
+  }
+}

packages/stats/server/src/router.ts

@@ -0,0 +1,62 @@
+import { Buffer } from "node:buffer"
+import { timingSafeEqual } from "node:crypto"
+import { Effect, Schema } from "effect"
+import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http"
+import { Resource } from "sst/resource"
+import { Ingest } from "./ingest"
+import { isShuttingDown } from "./shutdown"
+
+const IngestPayload = Schema.Struct({
+  events: Schema.optional(Schema.Unknown),
+})
+
+export const Routes = HttpRouter.use((router) =>
+  Effect.gen(function* () {
+    const ingestService = yield* Ingest
+
+    yield* Effect.all(
+      [
+        router.add("GET", "/health", () => json(200, { ok: true })),
+        router.add("GET", "/ready", () => json(isShuttingDown() ? 503 : 200, { ok: !isShuttingDown() })),
+        router.add("POST", "/", ingest(ingestService)),
+      ],
+      { discard: true },
+    )
+  }),
+)
+
+const ingest = (ingestService: Ingest.Service) => Effect.gen(function* () {
+  const request = yield* HttpServerRequest.HttpServerRequest
+  if (!isAuthorized(request.headers)) return yield* json(401, { ok: false, error: "Unauthorized" })
+
+  const payload = yield* HttpServerRequest.schemaBodyJson(IngestPayload).pipe(
+    Effect.match({
+      onFailure: () => undefined,
+      onSuccess: (value) => value,
+    }),
+  )
+  if (!payload) return yield* json(400, { ok: false, error: "Invalid JSON body" })
+
+  const events = Array.isArray(payload.events) ? payload.events.filter(isRecord) : []
+  if (events.length === 0) return yield* json(202, { ok: true, records: 0 })
+
+  return yield* ingestService.write(events).pipe(
+    Effect.flatMap((result) => json(202, { ok: true, records: result.records })),
+    Effect.catchTag("IngestError", (error) => json(502, { ok: false, records: events.length, failed: error.failed })),
+  )
+})
+
+function isAuthorized(headers: Record<string, string | undefined>) {
+  const actual = Buffer.from(headers.authorization ?? headers.Authorization ?? "")
+  const expected = Buffer.from(`Bearer ${Resource.LakeIngestConfig.secret}`)
+  if (actual.length !== expected.length) return false
+  return timingSafeEqual(actual, expected)
+}
+
+function isRecord(item: unknown): item is Record<string, unknown> {
+  return Boolean(item) && typeof item === "object" && !Array.isArray(item)
+}
+
+function json(status: number, body: Record<string, unknown>) {
+  return HttpServerResponse.json(body, { status }).pipe(Effect.orDie)
+}

packages/stats/server/src/server.ts

@@ -0,0 +1,28 @@
+import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer"
+import * as NodeRuntime from "@effect/platform-node/NodeRuntime"
+import { Config, Layer } from "effect"
+import { HttpRouter } from "effect/unstable/http"
+import { createServer } from "node:http"
+import { Ingest } from "./ingest"
+import { Routes } from "./router"
+import { registerShutdownSignalHandlers } from "./shutdown"
+
+registerShutdownSignalHandlers()
+
+const ServerLive = NodeHttpServer.layerConfig(
+  () => createServer(),
+  Config.all({
+    port: Config.number("PORT").pipe(Config.withDefault(3000)),
+    host: Config.string("HOST").pipe(Config.withDefault("0.0.0.0")),
+  }),
+)
+
+const runtimeLayer = Ingest.layer
+const programLayer = Routes.pipe(Layer.provide(runtimeLayer))
+const main = Layer.launch(
+  HttpRouter.serve(programLayer, {
+    disableLogger: true,
+  }).pipe(Layer.provideMerge(ServerLive)),
+)
+
+NodeRuntime.runMain(main, { disableErrorReporting: true })

packages/stats/server/src/shutdown.ts

@@ -0,0 +1,17 @@
+let shuttingDown = false
+let signalHandlersRegistered = false
+
+export function isShuttingDown() {
+  return shuttingDown
+}
+
+export function registerShutdownSignalHandlers() {
+  if (signalHandlersRegistered) return
+  signalHandlersRegistered = true
+  process.once("SIGTERM", markShuttingDown)
+  process.once("SIGINT", markShuttingDown)
+}
+
+function markShuttingDown() {
+  shuttingDown = true
+}

packages/stats/server/src/stat-sync.ts

@@ -0,0 +1,22 @@
+import * as NodeRuntime from "@effect/platform-node/NodeRuntime"
+import { Athena } from "@opencode-ai/stats-core/athena"
+import { layer as statsLayer } from "@opencode-ai/stats-core/runtime"
+import { syncStats } from "@opencode-ai/stats-core/stat-sync"
+import { Cause, Effect, Layer, Schedule } from "effect"
+
+const SYNC_INTERVAL = "1 hour"
+
+const runtimeLayer = Layer.mergeAll(statsLayer, Athena.layer)
+const syncPass = syncStats().pipe(
+  Effect.catchCause((cause) =>
+    Effect.logWarning("stats sync failed").pipe(Effect.annotateLogs({ cause: Cause.pretty(cause) })),
+  ),
+)
+const daemon = Effect.logInfo("stats sync daemon started").pipe(
+  Effect.andThen(syncPass.pipe(Effect.repeat(Schedule.fixed(SYNC_INTERVAL)))),
+  Effect.forkScoped,
+)
+
+NodeRuntime.runMain(Layer.launch(Layer.effectDiscard(daemon).pipe(Layer.provide(runtimeLayer))), {
+  disableErrorReporting: true,
+})

packages/stats/server/sst-env.d.ts

@@ -0,0 +1,10 @@
+/* This file is auto-generated by SST. Do not edit. */
+/* tslint:disable */
+/* eslint-disable */
+/* deno-fmt-ignore-file */
+/* biome-ignore-all lint: auto-generated */
+
+/// <reference path="../../../sst-env.d.ts" />
+
+import "sst"
+export {}

packages/stats/server/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "$schema": "https://json.schemastore.org/tsconfig",
+  "extends": "@tsconfig/node22/tsconfig.json",
+  "compilerOptions": {
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "strict": true,
+    "noEmit": true,
+    "types": ["bun", "node"]
+  },
+  "include": ["src", "../core/src/resource.d.ts"]
+}