lmin/zmo-cli
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity

feat: initial datalake and stats site (#28666)

Browse Source
This commit is contained in:
Adam
2026-05-25 17:34:04 -05:00
committed by GitHub
parent 633b5d6208
commit 5b02ac4d33
68 changed files with 8967 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
packages/stats/core/migrations/20260522121617_common_dust/migration.sql Normal file
View File

@@ -0,0 +1,42 @@
CREATE TABLE `stat` (
`id` bigint AUTO_INCREMENT PRIMARY KEY,
`grain` varchar(16) NOT NULL,
`period_start` datetime NOT NULL,
`period_end` datetime NOT NULL,
`dataset` varchar(64) NOT NULL DEFAULT 'all',
`tier` varchar(64) NOT NULL DEFAULT 'all',
`client` varchar(64) NOT NULL DEFAULT 'all',
`source` varchar(64) NOT NULL DEFAULT 'all',
`provider` varchar(128) NOT NULL,
`model` varchar(256) NOT NULL,
`provider_model` varchar(256) NOT NULL DEFAULT '',
`sessions` bigint NOT NULL DEFAULT 0,
`requests` bigint NOT NULL DEFAULT 0,
`input_tokens` bigint NOT NULL DEFAULT 0,
`output_tokens` bigint NOT NULL DEFAULT 0,
`reasoning_tokens` bigint NOT NULL DEFAULT 0,
`cache_read_tokens` bigint NOT NULL DEFAULT 0,
`total_tokens` bigint NOT NULL DEFAULT 0,
`input_cost_microcents` bigint NOT NULL DEFAULT 0,
`output_cost_microcents` bigint NOT NULL DEFAULT 0,
`total_cost_microcents` bigint NOT NULL DEFAULT 0,
`avg_duration_ms` decimal(12,2),
`p50_duration_ms` int,
`p95_duration_ms` int,
`avg_ttfb_ms` decimal(12,2),
`p50_ttfb_ms` int,
`p95_ttfb_ms` int,
`avg_output_tps` decimal(12,4),
`success_count` bigint NOT NULL DEFAULT 0,
`error_count` bigint NOT NULL DEFAULT 0,
`sample_count` bigint NOT NULL DEFAULT 0,
`rank_by_tokens` int,
`rank_by_requests` int,
`rank_by_cost` int,
`created_at` datetime NOT NULL DEFAULT (now()),
`updated_at` datetime NOT NULL DEFAULT (now()) ON UPDATE CURRENT_TIMESTAMP,
CONSTRAINT `uniq_model_period` UNIQUE INDEX(`grain`,`period_start`,`dataset`,`tier`,`client`,`source`,`provider`,`model`)
);
--> statement-breakpoint
CREATE INDEX `idx_leaderboard_tokens` ON `stat` (`grain`,`period_start`,`dataset`,`tier`,`total_tokens`);--> statement-breakpoint
CREATE INDEX `idx_model` ON `stat` (`model`,`grain`,`period_start`);

627
packages/stats/core/migrations/20260522121617_common_dust/snapshot.json Normal file
View File

@@ -0,0 +1,627 @@
{
"version": "6",
"dialect": "mysql",
"id": "72655266-65da-408e-bfd8-9f3a4ad817a5",
"prevIds": [
"00000000-0000-0000-0000-000000000000"
],
"ddl": [
{
"name": "stat",
"entityType": "tables"
},
{
"type": "bigint",
"notNull": true,
"autoIncrement": true,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "id",
"entityType": "columns",
"table": "stat"
},
{
"type": "varchar(16)",
"notNull": true,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "grain",
"entityType": "columns",
"table": "stat"
},
{
"type": "datetime",
"notNull": true,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "period_start",
"entityType": "columns",
"table": "stat"
},
{
"type": "datetime",
"notNull": true,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "period_end",
"entityType": "columns",
"table": "stat"
},
{
"type": "varchar(64)",
"notNull": true,
"autoIncrement": false,
"default": "'all'",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "dataset",
"entityType": "columns",
"table": "stat"
},
{
"type": "varchar(64)",
"notNull": true,
"autoIncrement": false,
"default": "'all'",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "tier",
"entityType": "columns",
"table": "stat"
},
{
"type": "varchar(64)",
"notNull": true,
"autoIncrement": false,
"default": "'all'",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "client",
"entityType": "columns",
"table": "stat"
},
{
"type": "varchar(64)",
"notNull": true,
"autoIncrement": false,
"default": "'all'",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "source",
"entityType": "columns",
"table": "stat"
},
{
"type": "varchar(128)",
"notNull": true,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "provider",
"entityType": "columns",
"table": "stat"
},
{
"type": "varchar(256)",
"notNull": true,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "model",
"entityType": "columns",
"table": "stat"
},
{
"type": "varchar(256)",
"notNull": true,
"autoIncrement": false,
"default": "''",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "provider_model",
"entityType": "columns",
"table": "stat"
},
{
"type": "bigint",
"notNull": true,
"autoIncrement": false,
"default": "0",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "sessions",
"entityType": "columns",
"table": "stat"
},
{
"type": "bigint",
"notNull": true,
"autoIncrement": false,
"default": "0",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "requests",
"entityType": "columns",
"table": "stat"
},
{
"type": "bigint",
"notNull": true,
"autoIncrement": false,
"default": "0",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "input_tokens",
"entityType": "columns",
"table": "stat"
},
{
"type": "bigint",
"notNull": true,
"autoIncrement": false,
"default": "0",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "output_tokens",
"entityType": "columns",
"table": "stat"
},
{
"type": "bigint",
"notNull": true,
"autoIncrement": false,
"default": "0",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "reasoning_tokens",
"entityType": "columns",
"table": "stat"
},
{
"type": "bigint",
"notNull": true,
"autoIncrement": false,
"default": "0",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "cache_read_tokens",
"entityType": "columns",
"table": "stat"
},
{
"type": "bigint",
"notNull": true,
"autoIncrement": false,
"default": "0",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "total_tokens",
"entityType": "columns",
"table": "stat"
},
{
"type": "bigint",
"notNull": true,
"autoIncrement": false,
"default": "0",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "input_cost_microcents",
"entityType": "columns",
"table": "stat"
},
{
"type": "bigint",
"notNull": true,
"autoIncrement": false,
"default": "0",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "output_cost_microcents",
"entityType": "columns",
"table": "stat"
},
{
"type": "bigint",
"notNull": true,
"autoIncrement": false,
"default": "0",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "total_cost_microcents",
"entityType": "columns",
"table": "stat"
},
{
"type": "decimal(12,2)",
"notNull": false,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "avg_duration_ms",
"entityType": "columns",
"table": "stat"
},
{
"type": "int",
"notNull": false,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "p50_duration_ms",
"entityType": "columns",
"table": "stat"
},
{
"type": "int",
"notNull": false,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "p95_duration_ms",
"entityType": "columns",
"table": "stat"
},
{
"type": "decimal(12,2)",
"notNull": false,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "avg_ttfb_ms",
"entityType": "columns",
"table": "stat"
},
{
"type": "int",
"notNull": false,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "p50_ttfb_ms",
"entityType": "columns",
"table": "stat"
},
{
"type": "int",
"notNull": false,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "p95_ttfb_ms",
"entityType": "columns",
"table": "stat"
},
{
"type": "decimal(12,4)",
"notNull": false,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "avg_output_tps",
"entityType": "columns",
"table": "stat"
},
{
"type": "bigint",
"notNull": true,
"autoIncrement": false,
"default": "0",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "success_count",
"entityType": "columns",
"table": "stat"
},
{
"type": "bigint",
"notNull": true,
"autoIncrement": false,
"default": "0",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "error_count",
"entityType": "columns",
"table": "stat"
},
{
"type": "bigint",
"notNull": true,
"autoIncrement": false,
"default": "0",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "sample_count",
"entityType": "columns",
"table": "stat"
},
{
"type": "int",
"notNull": false,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "rank_by_tokens",
"entityType": "columns",
"table": "stat"
},
{
"type": "int",
"notNull": false,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "rank_by_requests",
"entityType": "columns",
"table": "stat"
},
{
"type": "int",
"notNull": false,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "rank_by_cost",
"entityType": "columns",
"table": "stat"
},
{
"type": "datetime",
"notNull": true,
"autoIncrement": false,
"default": "(now())",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "created_at",
"entityType": "columns",
"table": "stat"
},
{
"type": "datetime",
"notNull": true,
"autoIncrement": false,
"default": "(now())",
"onUpdateNow": true,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "updated_at",
"entityType": "columns",
"table": "stat"
},
{
"columns": [
"id"
],
"name": "PRIMARY",
"table": "stat",
"entityType": "pks"
},
{
"columns": [
{
"value": "grain",
"isExpression": false
},
{
"value": "period_start",
"isExpression": false
},
{
"value": "dataset",
"isExpression": false
},
{
"value": "tier",
"isExpression": false
},
{
"value": "client",
"isExpression": false
},
{
"value": "source",
"isExpression": false
},
{
"value": "provider",
"isExpression": false
},
{
"value": "model",
"isExpression": false
}
],
"isUnique": true,
"using": null,
"algorithm": null,
"lock": null,
"nameExplicit": true,
"name": "uniq_model_period",
"entityType": "indexes",
"table": "stat"
},
{
"columns": [
{
"value": "grain",
"isExpression": false
},
{
"value": "period_start",
"isExpression": false
},
{
"value": "dataset",
"isExpression": false
},
{
"value": "tier",
"isExpression": false
},
{
"value": "total_tokens",
"isExpression": false
}
],
"isUnique": false,
"using": null,
"algorithm": null,
"lock": null,
"nameExplicit": true,
"name": "idx_leaderboard_tokens",
"entityType": "indexes",
"table": "stat"
},
{
"columns": [
{
"value": "model",
"isExpression": false
},
{
"value": "grain",
"isExpression": false
},
{
"value": "period_start",
"isExpression": false
}
],
"isUnique": false,
"using": null,
"algorithm": null,
"lock": null,
"nameExplicit": true,
"name": "idx_model",
"entityType": "indexes",
"table": "stat"
}
],
"renames": []
}

94
packages/stats/core/migrations/20260523110335_cool_vin_gonzales/migration.sql Normal file
View File

@@ -0,0 +1,94 @@
CREATE TABLE `geo_stat` (
`id` bigint AUTO_INCREMENT PRIMARY KEY,
`grain` varchar(16) NOT NULL,
`period_start` datetime NOT NULL,
`period_end` datetime NOT NULL,
`dataset` varchar(64) NOT NULL DEFAULT 'all',
`tier` varchar(64) NOT NULL DEFAULT 'all',
`client` varchar(64) NOT NULL DEFAULT 'all',
`source` varchar(64) NOT NULL DEFAULT 'all',
`country` char(2) NOT NULL,
`continent` varchar(8) NOT NULL DEFAULT '',
`sessions` bigint NOT NULL DEFAULT 0,
`requests` bigint NOT NULL DEFAULT 0,
`input_tokens` bigint NOT NULL DEFAULT 0,
`output_tokens` bigint NOT NULL DEFAULT 0,
`reasoning_tokens` bigint NOT NULL DEFAULT 0,
`cache_read_tokens` bigint NOT NULL DEFAULT 0,
`total_tokens` bigint NOT NULL DEFAULT 0,
`input_cost_microcents` bigint NOT NULL DEFAULT 0,
`output_cost_microcents` bigint NOT NULL DEFAULT 0,
`total_cost_microcents` bigint NOT NULL DEFAULT 0,
`avg_duration_ms` decimal(12,2),
`p50_duration_ms` int,
`p95_duration_ms` int,
`avg_ttfb_ms` decimal(12,2),
`p50_ttfb_ms` int,
`p95_ttfb_ms` int,
`avg_output_tps` decimal(12,4),
`success_count` bigint NOT NULL DEFAULT 0,
`error_count` bigint NOT NULL DEFAULT 0,
`sample_count` bigint NOT NULL DEFAULT 0,
`market_share_tokens` decimal(10,6),
`market_share_requests` decimal(10,6),
`market_share_sessions` decimal(10,6),
`rank_by_tokens` int,
`rank_by_requests` int,
`rank_by_sessions` int,
`rank_by_cost` int,
`created_at` datetime NOT NULL DEFAULT (now()),
`updated_at` datetime NOT NULL DEFAULT (now()) ON UPDATE CURRENT_TIMESTAMP,
CONSTRAINT `uniq_country_period` UNIQUE INDEX(`grain`,`period_start`,`dataset`,`tier`,`client`,`source`,`country`)
);
--> statement-breakpoint
CREATE TABLE `provider_stat` (
`id` bigint AUTO_INCREMENT PRIMARY KEY,
`grain` varchar(16) NOT NULL,
`period_start` datetime NOT NULL,
`period_end` datetime NOT NULL,
`dataset` varchar(64) NOT NULL DEFAULT 'all',
`tier` varchar(64) NOT NULL DEFAULT 'all',
`client` varchar(64) NOT NULL DEFAULT 'all',
`source` varchar(64) NOT NULL DEFAULT 'all',
`provider` varchar(128) NOT NULL,
`sessions` bigint NOT NULL DEFAULT 0,
`requests` bigint NOT NULL DEFAULT 0,
`input_tokens` bigint NOT NULL DEFAULT 0,
`output_tokens` bigint NOT NULL DEFAULT 0,
`reasoning_tokens` bigint NOT NULL DEFAULT 0,
`cache_read_tokens` bigint NOT NULL DEFAULT 0,
`total_tokens` bigint NOT NULL DEFAULT 0,
`input_cost_microcents` bigint NOT NULL DEFAULT 0,
`output_cost_microcents` bigint NOT NULL DEFAULT 0,
`total_cost_microcents` bigint NOT NULL DEFAULT 0,
`avg_duration_ms` decimal(12,2),
`p50_duration_ms` int,
`p95_duration_ms` int,
`avg_ttfb_ms` decimal(12,2),
`p50_ttfb_ms` int,
`p95_ttfb_ms` int,
`avg_output_tps` decimal(12,4),
`success_count` bigint NOT NULL DEFAULT 0,
`error_count` bigint NOT NULL DEFAULT 0,
`sample_count` bigint NOT NULL DEFAULT 0,
`market_share_tokens` decimal(10,6),
`market_share_requests` decimal(10,6),
`market_share_sessions` decimal(10,6),
`rank_by_tokens` int,
`rank_by_requests` int,
`rank_by_sessions` int,
`rank_by_cost` int,
`created_at` datetime NOT NULL DEFAULT (now()),
`updated_at` datetime NOT NULL DEFAULT (now()) ON UPDATE CURRENT_TIMESTAMP,
CONSTRAINT `uniq_provider_period` UNIQUE INDEX(`grain`,`period_start`,`dataset`,`tier`,`client`,`source`,`provider`)
);
--> statement-breakpoint
RENAME TABLE `stat` TO `model_stat`;--> statement-breakpoint
CREATE INDEX `idx_country_map_tokens` ON `geo_stat` (`grain`,`period_start`,`dataset`,`tier`,`total_tokens`);--> statement-breakpoint
CREATE INDEX `idx_country_rank` ON `geo_stat` (`grain`,`period_start`,`dataset`,`tier`,`rank_by_tokens`);--> statement-breakpoint
CREATE INDEX `idx_country` ON `geo_stat` (`country`,`grain`,`period_start`);--> statement-breakpoint
CREATE INDEX `idx_continent` ON `geo_stat` (`continent`,`grain`,`period_start`);--> statement-breakpoint
CREATE INDEX `idx_provider_leaderboard_tokens` ON `provider_stat` (`grain`,`period_start`,`dataset`,`tier`,`total_tokens`);--> statement-breakpoint
CREATE INDEX `idx_provider_market_share` ON `provider_stat` (`grain`,`period_start`,`dataset`,`tier`,`market_share_tokens`);--> statement-breakpoint
CREATE INDEX `idx_provider_rank` ON `provider_stat` (`grain`,`period_start`,`dataset`,`tier`,`rank_by_tokens`);--> statement-breakpoint
CREATE INDEX `idx_provider` ON `provider_stat` (`provider`,`grain`,`period_start`);

2041
packages/stats/core/migrations/20260523110335_cool_vin_gonzales/snapshot.json Normal file
View File

File diff suppressed because it is too large Load Diff