lmin/zmo-cli
1
0
Fork 0
Code Issues Pull Requests Actions 82 Packages Projects Releases Wiki Activity

fix(stats): batch honeycomb backfill

Browse Source
This commit is contained in:
Adam
2026-05-28 08:37:49 -05:00
parent 9ac0f3e9ac
commit 7504daa602
2 changed files with 125 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package.json
+1 -1
View File
@@ -10,7 +10,7 @@
"dev:desktop": "bun --cwd packages/desktop dev", "dev:desktop": "bun --cwd packages/desktop dev",
"dev:web": "bun --cwd packages/app dev", "dev:web": "bun --cwd packages/app dev",
"dev:console": "ulimit -n 10240 2>/dev/null; bun run --cwd packages/console/app dev", "dev:console": "ulimit -n 10240 2>/dev/null; bun run --cwd packages/console/app dev",
"dev:stats": "bun sst shell --stage=production -- bun run --cwd packages/stats/app dev", "dev:stats": "bun sst shell --stage=dev -- bun run --cwd packages/stats/app dev",
"dev:storybook": "bun --cwd packages/storybook storybook", "dev:storybook": "bun --cwd packages/storybook storybook",
"lint": "oxlint", "lint": "oxlint",
"typecheck": "bun turbo typecheck", "typecheck": "bun turbo typecheck",
packages/stats/core/src/honeycomb-backfill.ts
+124 -121
View File
@@ -19,11 +19,11 @@ import {
statPeriodKey, statPeriodKey,
synthesizeAllTierRows, synthesizeAllTierRows,
toStatBaseRow, toStatBaseRow,
UPSERT_CHUNK_SIZE,
type StatBaseAggregate, type StatBaseAggregate,
} from "./domain/stat" } from "./domain/stat"
const DAY_MS = 86_400_000 const DAY_MS = 86_400_000
const DEFAULT_UPSERT_CHUNK_SIZE = 100
const DEFAULT_TIERS = ["Go", "Free", "Paid"] const DEFAULT_TIERS = ["Go", "Free", "Paid"]
const FREE_MODELS = new Set(["gpt-5-nano", "grok-code", "big-pickle"]) const FREE_MODELS = new Set(["gpt-5-nano", "grok-code", "big-pickle"])
@@ -44,6 +44,7 @@ type ImportOptions = {
directories: string[] directories: string[]
dryRun: boolean dryRun: boolean
periodStart: Date | undefined periodStart: Date | undefined
upsertChunkSize: number
files: Partial<Record<ImportKey, string[]>> files: Partial<Record<ImportKey, string[]>>
} }
type ModelAggregate = StatBaseAggregate & { provider: string; model: string; provider_model: string } type ModelAggregate = StatBaseAggregate & { provider: string; model: string; provider_model: string }
@@ -168,6 +169,7 @@ async function importFiles(args: string[]) {
providerRows: providerRows.length, providerRows: providerRows.length,
geoRows: geoRows.length, geoRows: geoRows.length,
dryRun: opts.dryRun, dryRun: opts.dryRun,
upsertChunkSize: opts.upsertChunkSize,
}, },
null, null,
2, 2,
@@ -178,9 +180,9 @@ async function importFiles(args: string[]) {
if (!opts.databaseUrl) fail("DATABASE_URL is required unless --dry-run is set") if (!opts.databaseUrl) fail("DATABASE_URL is required unless --dry-run is set")
const db = drizzle({ client: new Client({ url: opts.databaseUrl }) }) const db = drizzle({ client: new Client({ url: opts.databaseUrl }) })
await upsertModelRows(db, modelRows) await upsertModelRows(db, modelRows, opts.upsertChunkSize)
await upsertProviderRows(db, providerRows) await upsertProviderRows(db, providerRows, opts.upsertChunkSize)
await upsertGeoRows(db, geoRows) await upsertGeoRows(db, geoRows, opts.upsertChunkSize)
} }
function buildQueries(limit: number, tiers: string[]): QuerySpec[] { function buildQueries(limit: number, tiers: string[]): QuerySpec[] {
@@ -783,125 +785,125 @@ function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value) return typeof value === "object" && value !== null && !Array.isArray(value)
} }
async function upsertModelRows(db: ReturnType<typeof drizzle>, rows: ModelStatRow[]) { async function upsertModelRows(db: ReturnType<typeof drizzle>, rows: ModelStatRow[], chunkSize: number) {
await Promise.all( const batches = chunks(rows, chunkSize)
chunks(rows, UPSERT_CHUNK_SIZE).map((chunk) => console.log(JSON.stringify({ table: "model_stat", batches: batches.length, chunkSize }))
db for (const chunk of batches) {
.insert(modelStat) await db
.values(chunk) .insert(modelStat)
.onDuplicateKeyUpdate({ .values(chunk)
set: { .onDuplicateKeyUpdate({
provider_model: inserted("provider_model"), set: {
sessions: inserted("sessions"), provider_model: inserted("provider_model"),
requests: inserted("requests"), sessions: inserted("sessions"),
input_tokens: inserted("input_tokens"), requests: inserted("requests"),
output_tokens: inserted("output_tokens"), input_tokens: inserted("input_tokens"),
reasoning_tokens: inserted("reasoning_tokens"), output_tokens: inserted("output_tokens"),
cache_read_tokens: inserted("cache_read_tokens"), reasoning_tokens: inserted("reasoning_tokens"),
total_tokens: inserted("total_tokens"), cache_read_tokens: inserted("cache_read_tokens"),
input_cost_microcents: inserted("input_cost_microcents"), total_tokens: inserted("total_tokens"),
output_cost_microcents: inserted("output_cost_microcents"), input_cost_microcents: inserted("input_cost_microcents"),
total_cost_microcents: inserted("total_cost_microcents"), output_cost_microcents: inserted("output_cost_microcents"),
avg_duration_ms: inserted("avg_duration_ms"), total_cost_microcents: inserted("total_cost_microcents"),
p50_duration_ms: inserted("p50_duration_ms"), avg_duration_ms: inserted("avg_duration_ms"),
p95_duration_ms: inserted("p95_duration_ms"), p50_duration_ms: inserted("p50_duration_ms"),
avg_ttfb_ms: inserted("avg_ttfb_ms"), p95_duration_ms: inserted("p95_duration_ms"),
p50_ttfb_ms: inserted("p50_ttfb_ms"), avg_ttfb_ms: inserted("avg_ttfb_ms"),
p95_ttfb_ms: inserted("p95_ttfb_ms"), p50_ttfb_ms: inserted("p50_ttfb_ms"),
avg_output_tps: inserted("avg_output_tps"), p95_ttfb_ms: inserted("p95_ttfb_ms"),
success_count: inserted("success_count"), avg_output_tps: inserted("avg_output_tps"),
error_count: inserted("error_count"), success_count: inserted("success_count"),
sample_count: inserted("sample_count"), error_count: inserted("error_count"),
rank_by_tokens: inserted("rank_by_tokens"), sample_count: inserted("sample_count"),
rank_by_requests: inserted("rank_by_requests"), rank_by_tokens: inserted("rank_by_tokens"),
rank_by_cost: inserted("rank_by_cost"), rank_by_requests: inserted("rank_by_requests"),
}, rank_by_cost: inserted("rank_by_cost"),
}), },
), })
) }
} }
async function upsertProviderRows(db: ReturnType<typeof drizzle>, rows: ProviderStatRow[]) { async function upsertProviderRows(db: ReturnType<typeof drizzle>, rows: ProviderStatRow[], chunkSize: number) {
await Promise.all( const batches = chunks(rows, chunkSize)
chunks(rows, UPSERT_CHUNK_SIZE).map((chunk) => console.log(JSON.stringify({ table: "provider_stat", batches: batches.length, chunkSize }))
db for (const chunk of batches) {
.insert(providerStat) await db
.values(chunk) .insert(providerStat)
.onDuplicateKeyUpdate({ .values(chunk)
set: { .onDuplicateKeyUpdate({
sessions: inserted("sessions"), set: {
requests: inserted("requests"), sessions: inserted("sessions"),
input_tokens: inserted("input_tokens"), requests: inserted("requests"),
output_tokens: inserted("output_tokens"), input_tokens: inserted("input_tokens"),
reasoning_tokens: inserted("reasoning_tokens"), output_tokens: inserted("output_tokens"),
cache_read_tokens: inserted("cache_read_tokens"), reasoning_tokens: inserted("reasoning_tokens"),
total_tokens: inserted("total_tokens"), cache_read_tokens: inserted("cache_read_tokens"),
input_cost_microcents: inserted("input_cost_microcents"), total_tokens: inserted("total_tokens"),
output_cost_microcents: inserted("output_cost_microcents"), input_cost_microcents: inserted("input_cost_microcents"),
total_cost_microcents: inserted("total_cost_microcents"), output_cost_microcents: inserted("output_cost_microcents"),
avg_duration_ms: inserted("avg_duration_ms"), total_cost_microcents: inserted("total_cost_microcents"),
p50_duration_ms: inserted("p50_duration_ms"), avg_duration_ms: inserted("avg_duration_ms"),
p95_duration_ms: inserted("p95_duration_ms"), p50_duration_ms: inserted("p50_duration_ms"),
avg_ttfb_ms: inserted("avg_ttfb_ms"), p95_duration_ms: inserted("p95_duration_ms"),
p50_ttfb_ms: inserted("p50_ttfb_ms"), avg_ttfb_ms: inserted("avg_ttfb_ms"),
p95_ttfb_ms: inserted("p95_ttfb_ms"), p50_ttfb_ms: inserted("p50_ttfb_ms"),
avg_output_tps: inserted("avg_output_tps"), p95_ttfb_ms: inserted("p95_ttfb_ms"),
success_count: inserted("success_count"), avg_output_tps: inserted("avg_output_tps"),
error_count: inserted("error_count"), success_count: inserted("success_count"),
sample_count: inserted("sample_count"), error_count: inserted("error_count"),
market_share_tokens: inserted("market_share_tokens"), sample_count: inserted("sample_count"),
market_share_requests: inserted("market_share_requests"), market_share_tokens: inserted("market_share_tokens"),
market_share_sessions: inserted("market_share_sessions"), market_share_requests: inserted("market_share_requests"),
rank_by_tokens: inserted("rank_by_tokens"), market_share_sessions: inserted("market_share_sessions"),
rank_by_requests: inserted("rank_by_requests"), rank_by_tokens: inserted("rank_by_tokens"),
rank_by_sessions: inserted("rank_by_sessions"), rank_by_requests: inserted("rank_by_requests"),
rank_by_cost: inserted("rank_by_cost"), rank_by_sessions: inserted("rank_by_sessions"),
}, rank_by_cost: inserted("rank_by_cost"),
}), },
), })
) }
} }
async function upsertGeoRows(db: ReturnType<typeof drizzle>, rows: GeoStatRow[]) { async function upsertGeoRows(db: ReturnType<typeof drizzle>, rows: GeoStatRow[], chunkSize: number) {
await Promise.all( const batches = chunks(rows, chunkSize)
chunks(rows, UPSERT_CHUNK_SIZE).map((chunk) => console.log(JSON.stringify({ table: "geo_stat", batches: batches.length, chunkSize }))
db for (const chunk of batches) {
.insert(geoStat) await db
.values(chunk) .insert(geoStat)
.onDuplicateKeyUpdate({ .values(chunk)
set: { .onDuplicateKeyUpdate({
continent: inserted("continent"), set: {
sessions: inserted("sessions"), continent: inserted("continent"),
requests: inserted("requests"), sessions: inserted("sessions"),
input_tokens: inserted("input_tokens"), requests: inserted("requests"),
output_tokens: inserted("output_tokens"), input_tokens: inserted("input_tokens"),
reasoning_tokens: inserted("reasoning_tokens"), output_tokens: inserted("output_tokens"),
cache_read_tokens: inserted("cache_read_tokens"), reasoning_tokens: inserted("reasoning_tokens"),
total_tokens: inserted("total_tokens"), cache_read_tokens: inserted("cache_read_tokens"),
input_cost_microcents: inserted("input_cost_microcents"), total_tokens: inserted("total_tokens"),
output_cost_microcents: inserted("output_cost_microcents"), input_cost_microcents: inserted("input_cost_microcents"),
total_cost_microcents: inserted("total_cost_microcents"), output_cost_microcents: inserted("output_cost_microcents"),
avg_duration_ms: inserted("avg_duration_ms"), total_cost_microcents: inserted("total_cost_microcents"),
p50_duration_ms: inserted("p50_duration_ms"), avg_duration_ms: inserted("avg_duration_ms"),
p95_duration_ms: inserted("p95_duration_ms"), p50_duration_ms: inserted("p50_duration_ms"),
avg_ttfb_ms: inserted("avg_ttfb_ms"), p95_duration_ms: inserted("p95_duration_ms"),
p50_ttfb_ms: inserted("p50_ttfb_ms"), avg_ttfb_ms: inserted("avg_ttfb_ms"),
p95_ttfb_ms: inserted("p95_ttfb_ms"), p50_ttfb_ms: inserted("p50_ttfb_ms"),
avg_output_tps: inserted("avg_output_tps"), p95_ttfb_ms: inserted("p95_ttfb_ms"),
success_count: inserted("success_count"), avg_output_tps: inserted("avg_output_tps"),
error_count: inserted("error_count"), success_count: inserted("success_count"),
sample_count: inserted("sample_count"), error_count: inserted("error_count"),
market_share_tokens: inserted("market_share_tokens"), sample_count: inserted("sample_count"),
market_share_requests: inserted("market_share_requests"), market_share_tokens: inserted("market_share_tokens"),
market_share_sessions: inserted("market_share_sessions"), market_share_requests: inserted("market_share_requests"),
rank_by_tokens: inserted("rank_by_tokens"), market_share_sessions: inserted("market_share_sessions"),
rank_by_requests: inserted("rank_by_requests"), rank_by_tokens: inserted("rank_by_tokens"),
rank_by_sessions: inserted("rank_by_sessions"), rank_by_requests: inserted("rank_by_requests"),
rank_by_cost: inserted("rank_by_cost"), rank_by_sessions: inserted("rank_by_sessions"),
}, rank_by_cost: inserted("rank_by_cost"),
}), },
), })
) }
} }
function parseImportOptions(args: string[]): ImportOptions { function parseImportOptions(args: string[]): ImportOptions {
@@ -917,6 +919,7 @@ function parseImportOptions(args: string[]): ImportOptions {
directories: flags.get("dir") ?? flags.get("directory") ?? [], directories: flags.get("dir") ?? flags.get("directory") ?? [],
dryRun: flags.has("dry-run"), dryRun: flags.has("dry-run"),
periodStart: parseDateFlag(flags, "period-start"), periodStart: parseDateFlag(flags, "period-start"),
upsertChunkSize: parseIntegerFlag(flags, "upsert-chunk-size") ?? DEFAULT_UPSERT_CHUNK_SIZE,
files, files,
} }
} }
@@ -969,8 +972,8 @@ function parseListFlag(flags: Map<string, string[]>, name: string) {
function usage(): never { function usage(): never {
fail(`Usage: fail(`Usage:
bun src/honeycomb-backfill.ts queries [--tiers Go,Free,Paid] [--limit 1000] bun src/honeycomb-backfill.ts queries [--tiers Go,Free,Paid] [--limit 1000]
bun src/honeycomb-backfill.ts import [--dry-run] [--database-url URL] --dir downloads bun src/honeycomb-backfill.ts import [--dry-run] [--upsert-chunk-size 100] [--database-url URL] --dir downloads
bun src/honeycomb-backfill.ts import [--dry-run] [--database-url URL] --model-day file.csv [--model-day more.csv] ...`) bun src/honeycomb-backfill.ts import [--dry-run] [--upsert-chunk-size 100] [--database-url URL] --model-day file.csv [--model-day more.csv] ...`)
} }
function fail(message: string): never { function fail(message: string): never {