feat: initial datalake and stats site (#28666)
This commit is contained in:
Adam
@@ -30,7 +30,7 @@ export const api = new sst.cloudflare.Worker("Api", {
|
||||
transform: {
|
||||
worker: (args) => {
|
||||
args.logpush = true
|
||||
if ($app.stage === "vimtor") return
|
||||
if ($app.stage === "vimtor" || $app.stage === "adam") return
|
||||
args.bindings = $resolve(args.bindings).apply((bindings) => [
|
||||
...bindings,
|
||||
{
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import { domain } from "./stage"
|
||||
import { deployAws, domain } from "./stage"
|
||||
import { EMAILOCTOPUS_API_KEY } from "./app"
|
||||
import { SECRET } from "./secret"
|
||||
|
||||
const lake = deployAws ? await import("./lake") : undefined
|
||||
|
||||
////////////////
|
||||
// DATABASE
|
||||
////////////////
|
||||
@@ -240,7 +242,7 @@ const SALESFORCE_INSTANCE_URL = new sst.Secret("SALESFORCE_INSTANCE_URL")
|
||||
|
||||
const logProcessor = new sst.cloudflare.Worker("LogProcessor", {
|
||||
handler: "packages/console/function/src/log-processor.ts",
|
||||
link: [new sst.Secret("HONEYCOMB_API_KEY")],
|
||||
link: [SECRET.HoneycombApiKey, ...(lake?.lakeIngest ? [lake.lakeIngest] : [])],
|
||||
})
|
||||
|
||||
new sst.cloudflare.x.SolidStart("Console", {
|
||||
|
||||
340
infra/lake.ts
Normal file
340
infra/lake.ts
Normal file
@@ -0,0 +1,340 @@
|
||||
import { domain } from "./stage"
|
||||
|
||||
const current = aws.getCallerIdentityOutput({})
|
||||
const partition = aws.getPartitionOutput({})
|
||||
const region = aws.getRegionOutput({})
|
||||
|
||||
const tableBucketName = `opencode-${$app.stage}-lake`
|
||||
const glueCatalogName = "s3tablescatalog"
|
||||
const glueCatalogArn = $interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:catalog`
|
||||
const glueS3TablesCatalogArn = $interpolate`${glueCatalogArn}/${glueCatalogName}`
|
||||
const glueS3TablesChildCatalogArn = $interpolate`${glueS3TablesCatalogArn}/${tableBucketName}`
|
||||
const glueS3TablesDatabaseWildcardArn = $interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:database/${glueCatalogName}/${tableBucketName}/*`
|
||||
const glueS3TablesTableWildcardArn = $interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:table/${glueCatalogName}/${tableBucketName}/*/*`
|
||||
const s3TablesBucketWildcardArn = $interpolate`arn:${partition.partition}:s3tables:${region.region}:${current.accountId}:bucket/*`
|
||||
|
||||
export const tableBucket = new aws.s3tables.TableBucket(
|
||||
"LakeTableBucket",
|
||||
{
|
||||
name: tableBucketName,
|
||||
forceDestroy: $app.stage !== "production",
|
||||
},
|
||||
)
|
||||
|
||||
const s3TablesCatalog = new aws.cloudcontrol.Resource(
|
||||
"LakeS3TablesCatalog",
|
||||
{
|
||||
typeName: "AWS::Glue::Catalog",
|
||||
desiredState: $jsonStringify({
|
||||
Name: glueCatalogName,
|
||||
Description: "Federated catalog for S3 Tables",
|
||||
FederatedCatalog: {
|
||||
Identifier: s3TablesBucketWildcardArn,
|
||||
ConnectionName: "aws:s3tables",
|
||||
},
|
||||
CreateDatabaseDefaultPermissions: [
|
||||
{
|
||||
Principal: {
|
||||
DataLakePrincipalIdentifier: "IAM_ALLOWED_PRINCIPALS",
|
||||
},
|
||||
Permissions: ["ALL"],
|
||||
},
|
||||
],
|
||||
CreateTableDefaultPermissions: [
|
||||
{
|
||||
Principal: {
|
||||
DataLakePrincipalIdentifier: "IAM_ALLOWED_PRINCIPALS",
|
||||
},
|
||||
Permissions: ["ALL"],
|
||||
},
|
||||
],
|
||||
AllowFullTableExternalDataAccess: "True",
|
||||
}),
|
||||
},
|
||||
{ dependsOn: [tableBucket] },
|
||||
)
|
||||
|
||||
const athenaResultsBucket = new aws.s3.Bucket(
|
||||
"LakeAthenaResults",
|
||||
{
|
||||
bucket: `opencode-${$app.stage}-lake-athena-results`,
|
||||
forceDestroy: $app.stage !== "production",
|
||||
},
|
||||
)
|
||||
|
||||
const firehoseErrorBucket = new aws.s3.Bucket(
|
||||
"LakeFirehoseErrors",
|
||||
{
|
||||
bucket: `opencode-${$app.stage}-lake-firehose-errors`,
|
||||
forceDestroy: $app.stage !== "production",
|
||||
},
|
||||
)
|
||||
|
||||
const athenaWorkgroup = new aws.athena.Workgroup(
|
||||
"LakeAthenaWorkgroup",
|
||||
{
|
||||
name: `opencode-${$app.stage}-lake-workgroup`,
|
||||
forceDestroy: $app.stage !== "production",
|
||||
configuration: {
|
||||
enforceWorkgroupConfiguration: true,
|
||||
publishCloudwatchMetricsEnabled: true,
|
||||
resultConfiguration: {
|
||||
outputLocation: $interpolate`s3://${athenaResultsBucket.bucket}/`,
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
const firehoseRole = new aws.iam.Role(
|
||||
"LakeFirehoseRole",
|
||||
{
|
||||
assumeRolePolicy: aws.iam.getPolicyDocumentOutput({
|
||||
statements: [
|
||||
{
|
||||
effect: "Allow",
|
||||
actions: ["sts:AssumeRole"],
|
||||
principals: [
|
||||
{
|
||||
type: "Service",
|
||||
identifiers: ["firehose.amazonaws.com"],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}).json,
|
||||
},
|
||||
)
|
||||
|
||||
const firehosePolicy = new aws.iam.RolePolicy(
|
||||
"LakeFirehosePolicy",
|
||||
{
|
||||
role: firehoseRole.id,
|
||||
policy: aws.iam.getPolicyDocumentOutput({
|
||||
statements: [
|
||||
{
|
||||
effect: "Allow",
|
||||
actions: [
|
||||
"s3tables:ListTableBuckets",
|
||||
"s3tables:GetTableBucket",
|
||||
"s3tables:GetNamespace",
|
||||
"s3tables:GetTable",
|
||||
"s3tables:GetTableData",
|
||||
"s3tables:GetTableMetadataLocation",
|
||||
"s3tables:ListNamespaces",
|
||||
"s3tables:ListTables",
|
||||
"s3tables:PutTableData",
|
||||
"s3tables:UpdateTableMetadataLocation",
|
||||
],
|
||||
resources: ["*"],
|
||||
},
|
||||
{
|
||||
effect: "Allow",
|
||||
actions: [
|
||||
"glue:GetCatalog",
|
||||
"glue:GetCatalogs",
|
||||
"glue:GetDatabase",
|
||||
"glue:GetDatabases",
|
||||
"glue:GetTable",
|
||||
"glue:GetTables",
|
||||
"glue:UpdateTable",
|
||||
],
|
||||
resources: [
|
||||
glueCatalogArn,
|
||||
glueS3TablesCatalogArn,
|
||||
$interpolate`${glueS3TablesCatalogArn}/*`,
|
||||
glueS3TablesDatabaseWildcardArn,
|
||||
glueS3TablesTableWildcardArn,
|
||||
$interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:database/*`,
|
||||
$interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:table/*/*`,
|
||||
$interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:table/${glueCatalogName}/*`,
|
||||
],
|
||||
},
|
||||
{
|
||||
effect: "Allow",
|
||||
actions: [
|
||||
"s3:AbortMultipartUpload",
|
||||
"s3:GetBucketLocation",
|
||||
"s3:GetObject",
|
||||
"s3:ListBucket",
|
||||
"s3:ListBucketMultipartUploads",
|
||||
"s3:PutObject",
|
||||
],
|
||||
resources: [firehoseErrorBucket.arn, $interpolate`${firehoseErrorBucket.arn}/*`],
|
||||
},
|
||||
{
|
||||
effect: "Allow",
|
||||
actions: ["lakeformation:GetDataAccess"],
|
||||
resources: ["*"],
|
||||
},
|
||||
],
|
||||
}).json,
|
||||
},
|
||||
)
|
||||
|
||||
const firehose = new aws.kinesis.FirehoseDeliveryStream(
|
||||
"LakeFirehose",
|
||||
{
|
||||
name: `opencode-${$app.stage}-lake-ingest`,
|
||||
destination: "iceberg",
|
||||
icebergConfiguration: {
|
||||
appendOnly: true,
|
||||
bufferingInterval: 60,
|
||||
bufferingSize: 1,
|
||||
catalogArn: glueS3TablesChildCatalogArn,
|
||||
processingConfiguration: {
|
||||
enabled: true,
|
||||
processors: [
|
||||
{
|
||||
type: "MetadataExtraction",
|
||||
parameters: [
|
||||
{ parameterName: "JsonParsingEngine", parameterValue: "JQ-1.6" },
|
||||
{
|
||||
parameterName: "MetadataExtractionQuery",
|
||||
parameterValue:
|
||||
'{destinationDatabaseName:._lake_database,destinationTableName:._lake_table,operation:(._lake_operation // "insert")}',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
roleArn: firehoseRole.arn,
|
||||
s3BackupMode: "FailedDataOnly",
|
||||
s3Configuration: {
|
||||
roleArn: firehoseRole.arn,
|
||||
bucketArn: firehoseErrorBucket.arn,
|
||||
errorOutputPrefix: "errors/!{firehose:error-output-type}/",
|
||||
},
|
||||
},
|
||||
},
|
||||
{ dependsOn: [s3TablesCatalog, firehosePolicy] },
|
||||
)
|
||||
|
||||
export const lakeVpc = new sst.aws.Vpc("LakeVpc")
|
||||
export const lakeCluster = new sst.aws.Cluster("LakeCluster", { vpc: lakeVpc })
|
||||
export const lakeRegion = region.region
|
||||
export const lakeCatalog = $interpolate`${glueCatalogName}/${tableBucket.name}`
|
||||
export const lakeAthenaWorkgroup = athenaWorkgroup
|
||||
|
||||
const ingestSecret = new random.RandomPassword("LakeIngestSecret", { length: 32 })
|
||||
|
||||
const ingestConfig = new sst.Linkable("LakeIngestConfig", {
|
||||
properties: {
|
||||
streamName: firehose.name,
|
||||
secret: ingestSecret.result,
|
||||
},
|
||||
})
|
||||
|
||||
const ingestService = new sst.aws.Service("LakeIngestService", {
|
||||
cluster: lakeCluster,
|
||||
architecture: "arm64",
|
||||
cpu: "0.5 vCPU",
|
||||
memory: "1 GB",
|
||||
image: {
|
||||
context: ".",
|
||||
dockerfile: "packages/stats/server/Dockerfile",
|
||||
},
|
||||
link: [ingestConfig],
|
||||
permissions: [
|
||||
{
|
||||
actions: ["firehose:PutRecord", "firehose:PutRecordBatch"],
|
||||
resources: [firehose.arn],
|
||||
},
|
||||
],
|
||||
scaling: {
|
||||
min: $app.stage === "production" ? 2 : 1,
|
||||
max: $app.stage === "production" ? 32 : 4,
|
||||
cpuUtilization: 60,
|
||||
memoryUtilization: 70,
|
||||
},
|
||||
loadBalancer: {
|
||||
domain: {
|
||||
name: `lake.${domain}`,
|
||||
dns: sst.cloudflare.dns(),
|
||||
},
|
||||
rules: [
|
||||
{ listen: "80/http", redirect: "443/https" },
|
||||
{ listen: "443/https", forward: "3000/http" },
|
||||
],
|
||||
health: {
|
||||
"3000/http": {
|
||||
path: "/ready",
|
||||
successCodes: "200-299",
|
||||
},
|
||||
},
|
||||
},
|
||||
health: {
|
||||
command: [
|
||||
"CMD-SHELL",
|
||||
"bun --eval \"fetch('http://localhost:3000/health').then((r) => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))\"",
|
||||
],
|
||||
interval: "30 seconds",
|
||||
retries: 3,
|
||||
startPeriod: "30 seconds",
|
||||
timeout: "5 seconds",
|
||||
},
|
||||
dev: {
|
||||
command: "bun run start",
|
||||
directory: "packages/stats/server",
|
||||
url: "http://localhost:3000",
|
||||
},
|
||||
wait: $app.stage === "production",
|
||||
})
|
||||
|
||||
export const lakeIngest = new sst.Linkable("LakeIngest", {
|
||||
properties: {
|
||||
url: ingestService.url,
|
||||
secret: ingestSecret.result,
|
||||
},
|
||||
})
|
||||
|
||||
export const lakeQueryPermissions = [
|
||||
{
|
||||
actions: ["athena:StartQueryExecution", "athena:GetQueryExecution", "athena:GetQueryResults"],
|
||||
resources: [athenaWorkgroup.arn],
|
||||
},
|
||||
{
|
||||
actions: [
|
||||
"glue:GetCatalog",
|
||||
"glue:GetCatalogs",
|
||||
"glue:GetDatabase",
|
||||
"glue:GetDatabases",
|
||||
"glue:GetTable",
|
||||
"glue:GetTables",
|
||||
"glue:GetPartitions",
|
||||
],
|
||||
resources: [
|
||||
glueCatalogArn,
|
||||
glueS3TablesCatalogArn,
|
||||
$interpolate`${glueS3TablesCatalogArn}/*`,
|
||||
glueS3TablesDatabaseWildcardArn,
|
||||
glueS3TablesTableWildcardArn,
|
||||
$interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:database/*`,
|
||||
$interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:table/*/*`,
|
||||
$interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:table/${glueCatalogName}/*`,
|
||||
],
|
||||
},
|
||||
{
|
||||
actions: ["s3:GetBucketLocation", "s3:ListBucket"],
|
||||
resources: [athenaResultsBucket.arn],
|
||||
},
|
||||
{
|
||||
actions: ["s3:GetObject", "s3:PutObject", "s3:AbortMultipartUpload", "s3:ListBucketMultipartUploads"],
|
||||
resources: [$interpolate`${athenaResultsBucket.arn}/*`],
|
||||
},
|
||||
{
|
||||
actions: [
|
||||
"s3tables:GetTableBucket",
|
||||
"s3tables:GetNamespace",
|
||||
"s3tables:GetTable",
|
||||
"s3tables:GetTableData",
|
||||
"s3tables:GetTableMetadataLocation",
|
||||
"s3tables:ListNamespaces",
|
||||
"s3tables:ListTables",
|
||||
],
|
||||
resources: ["*"],
|
||||
},
|
||||
{
|
||||
actions: ["lakeformation:GetDataAccess"],
|
||||
resources: ["*"],
|
||||
},
|
||||
]
|
||||
@@ -7,6 +7,7 @@ sst.Linkable.wrap(random.RandomPassword, (resource) => ({
|
||||
export const SECRET = {
|
||||
R2AccessKey: new sst.Secret("R2AccessKey", "unknown"),
|
||||
R2SecretKey: new sst.Secret("R2SecretKey", "unknown"),
|
||||
HoneycombApiKey: new sst.Secret("HONEYCOMB_API_KEY"),
|
||||
HoneycombWebhookSecret: new random.RandomPassword("HoneycombWebhookSecret", { length: 24 }),
|
||||
UpstashRedisRestUrl: new sst.Secret("UpstashRedisRestUrl"),
|
||||
UpstashRedisRestToken: new sst.Secret("UpstashRedisRestToken"),
|
||||
|
||||
@@ -5,6 +5,50 @@ export const domain = (() => {
|
||||
})()
|
||||
|
||||
export const zoneID = "430ba34c138cfb5360826c4909f99be8"
|
||||
export const deployAws = $app.stage === "production" || $app.stage === "dev" || $app.stage === "adam"
|
||||
|
||||
const githubActionsDeployRole = (() => {
|
||||
if ($app.stage !== "dev" && $app.stage !== "production") return
|
||||
|
||||
const provider = new aws.iam.OpenIdConnectProvider("GithubActionsOidcProvider", {
|
||||
url: "https://token.actions.githubusercontent.com",
|
||||
clientIdLists: ["sts.amazonaws.com"],
|
||||
})
|
||||
const role = new aws.iam.Role("GithubActionsDeployRole", {
|
||||
name: `opencode-${$app.stage}-github-actions-deploy`,
|
||||
maxSessionDuration: 3600,
|
||||
assumeRolePolicy: aws.iam.getPolicyDocumentOutput({
|
||||
statements: [
|
||||
{
|
||||
effect: "Allow",
|
||||
actions: ["sts:AssumeRoleWithWebIdentity"],
|
||||
principals: [{ type: "Federated", identifiers: [provider.arn] }],
|
||||
conditions: [
|
||||
{
|
||||
test: "StringEquals",
|
||||
variable: "token.actions.githubusercontent.com:aud",
|
||||
values: ["sts.amazonaws.com"],
|
||||
},
|
||||
{
|
||||
test: "StringEquals",
|
||||
variable: "token.actions.githubusercontent.com:sub",
|
||||
values: [`repo:anomalyco/opencode:environment:${$app.stage}`],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}).json,
|
||||
})
|
||||
|
||||
new aws.iam.RolePolicyAttachment("GithubActionsDeployRoleAdmin", {
|
||||
role: role.name,
|
||||
policyArn: "arn:aws:iam::aws:policy/AdministratorAccess",
|
||||
})
|
||||
|
||||
return role
|
||||
})()
|
||||
|
||||
export const githubActionsDeployRoleArn = githubActionsDeployRole?.arn
|
||||
|
||||
new cloudflare.RegionalHostname("RegionalHostname", {
|
||||
hostname: domain,
|
||||
|
||||
207
infra/stats.ts
Normal file
207
infra/stats.ts
Normal file
@@ -0,0 +1,207 @@
|
||||
import { lakeAthenaWorkgroup, lakeCatalog, lakeCluster, lakeQueryPermissions, lakeRegion, tableBucket } from "./lake"
|
||||
|
||||
const domain = (() => {
|
||||
if ($app.stage === "production") return "stats.opencode.ai"
|
||||
if ($app.stage === "dev") return "stats.dev.opencode.ai"
|
||||
return `stats.${$app.stage}.dev.opencode.ai`
|
||||
})()
|
||||
|
||||
////////////////
|
||||
// LAKE
|
||||
////////////////
|
||||
|
||||
const inferenceNamespace = new aws.s3tables.Namespace("LakeInferenceNamespace", {
|
||||
namespace: "inference",
|
||||
tableBucketArn: tableBucket.arn,
|
||||
})
|
||||
|
||||
const inferenceEventTable = new aws.s3tables.Table(
|
||||
"LakeInferenceEventTable",
|
||||
{
|
||||
name: "event",
|
||||
namespace: inferenceNamespace.namespace,
|
||||
tableBucketArn: inferenceNamespace.tableBucketArn,
|
||||
format: "ICEBERG",
|
||||
metadata: {
|
||||
iceberg: {
|
||||
schema: {
|
||||
fields: [
|
||||
{ name: "event_timestamp", type: "string", required: false },
|
||||
{ name: "event_date", type: "string", required: false },
|
||||
{ name: "event_type", type: "string", required: false },
|
||||
{ name: "dataset", type: "string", required: false },
|
||||
{ name: "cf_continent", type: "string", required: false },
|
||||
{ name: "cf_country", type: "string", required: false },
|
||||
{ name: "cf_city", type: "string", required: false },
|
||||
{ name: "cf_region", type: "string", required: false },
|
||||
{ name: "cf_latitude", type: "double", required: false },
|
||||
{ name: "cf_longitude", type: "double", required: false },
|
||||
{ name: "cf_timezone", type: "string", required: false },
|
||||
{ name: "duration", type: "double", required: false },
|
||||
{ name: "request_length", type: "long", required: false },
|
||||
{ name: "status", type: "int", required: false },
|
||||
{ name: "ip", type: "string", required: false },
|
||||
{ name: "is_stream", type: "boolean", required: false },
|
||||
{ name: "session", type: "string", required: false },
|
||||
{ name: "request", type: "string", required: false },
|
||||
{ name: "client", type: "string", required: false },
|
||||
{ name: "user_agent", type: "string", required: false },
|
||||
{ name: "model_variant", type: "string", required: false },
|
||||
{ name: "source", type: "string", required: false },
|
||||
{ name: "provider", type: "string", required: false },
|
||||
{ name: "provider_model", type: "string", required: false },
|
||||
{ name: "model", type: "string", required: false },
|
||||
{ name: "llm_error_code", type: "int", required: false },
|
||||
{ name: "llm_error_message", type: "string", required: false },
|
||||
{ name: "error_response", type: "string", required: false },
|
||||
{ name: "error_type", type: "string", required: false },
|
||||
{ name: "error_message", type: "string", required: false },
|
||||
{ name: "error_cause", type: "string", required: false },
|
||||
{ name: "error_cause2", type: "string", required: false },
|
||||
{ name: "api_key", type: "string", required: false },
|
||||
{ name: "workspace", type: "string", required: false },
|
||||
{ name: "is_subscription", type: "boolean", required: false },
|
||||
{ name: "subscription", type: "string", required: false },
|
||||
{ name: "response_length", type: "long", required: false },
|
||||
{ name: "time_to_first_byte", type: "long", required: false },
|
||||
{ name: "timestamp_first_byte", type: "long", required: false },
|
||||
{ name: "timestamp_last_byte", type: "long", required: false },
|
||||
{ name: "tokens_input", type: "long", required: false },
|
||||
{ name: "tokens_output", type: "long", required: false },
|
||||
{ name: "tokens_reasoning", type: "long", required: false },
|
||||
{ name: "tokens_cache_read", type: "long", required: false },
|
||||
{ name: "tokens_cache_write_5m", type: "long", required: false },
|
||||
{ name: "tokens_cache_write_1h", type: "long", required: false },
|
||||
{ name: "cost_input_microcents", type: "long", required: false },
|
||||
{ name: "cost_output_microcents", type: "long", required: false },
|
||||
{ name: "cost_cache_read_microcents", type: "long", required: false },
|
||||
{ name: "cost_cache_write_microcents", type: "long", required: false },
|
||||
{ name: "cost_total_microcents", type: "long", required: false },
|
||||
{ name: "cost_input", type: "long", required: false },
|
||||
{ name: "cost_output", type: "long", required: false },
|
||||
{ name: "cost_cache_read", type: "long", required: false },
|
||||
{ name: "cost_cache_write_5m", type: "long", required: false },
|
||||
{ name: "cost_cache_write_1h", type: "long", required: false },
|
||||
{ name: "cost_total", type: "long", required: false },
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{ deleteBeforeReplace: $app.stage !== "production" },
|
||||
)
|
||||
|
||||
export const inferenceEvent = new sst.Linkable("InferenceEvent", {
|
||||
properties: {
|
||||
region: lakeRegion,
|
||||
catalog: lakeCatalog,
|
||||
database: inferenceNamespace.namespace,
|
||||
table: inferenceEventTable.name,
|
||||
tableBucket: tableBucket.name,
|
||||
workgroup: lakeAthenaWorkgroup.name,
|
||||
},
|
||||
})
|
||||
|
||||
////////////////
|
||||
// DATABASE
|
||||
////////////////
|
||||
|
||||
const cluster = planetscale.getDatabaseOutput({
|
||||
name: "opencode-stats",
|
||||
organization: "anomalyco",
|
||||
})
|
||||
|
||||
const branch =
|
||||
$app.stage === "production"
|
||||
? planetscale.getBranchOutput({
|
||||
name: "production",
|
||||
organization: cluster.organization,
|
||||
database: cluster.name,
|
||||
})
|
||||
: new planetscale.Branch("StatsDatabaseBranch", {
|
||||
database: cluster.name,
|
||||
organization: cluster.organization,
|
||||
name: $app.stage,
|
||||
parentBranch: "production",
|
||||
})
|
||||
|
||||
const password = new planetscale.Password("StatsDatabasePassword", {
|
||||
name: $app.stage,
|
||||
database: cluster.name,
|
||||
organization: cluster.organization,
|
||||
branch: branch.name,
|
||||
})
|
||||
|
||||
const databaseUrl = $interpolate`mysql://${password.username.apply(encodeURIComponent)}:${password.plaintext.apply(
|
||||
encodeURIComponent,
|
||||
)}@${password.accessHostUrl}/${cluster.name}`
|
||||
|
||||
export const database = new sst.Linkable("StatsDatabase", {
|
||||
properties: {
|
||||
host: password.accessHostUrl,
|
||||
database: cluster.name,
|
||||
username: password.username,
|
||||
password: password.plaintext,
|
||||
port: 3306,
|
||||
url: databaseUrl,
|
||||
},
|
||||
})
|
||||
|
||||
new sst.x.DevCommand("StatsStudio", {
|
||||
link: [database],
|
||||
environment: {
|
||||
DATABASE_URL: databaseUrl,
|
||||
},
|
||||
dev: {
|
||||
command: "bun db:studio",
|
||||
directory: "packages/stats/core",
|
||||
autostart: false,
|
||||
},
|
||||
})
|
||||
|
||||
////////////////
|
||||
// APP
|
||||
////////////////
|
||||
|
||||
// export const app = new sst.cloudflare.x.SolidStart("Stats", {
|
||||
// path: "packages/stats/app",
|
||||
// buildCommand: "bun run build",
|
||||
// domain,
|
||||
// link: [database],
|
||||
// environment: {
|
||||
// PUBLIC_URL: `https://${domain}`,
|
||||
// },
|
||||
// })
|
||||
|
||||
////////////////
|
||||
// SERVICES
|
||||
////////////////
|
||||
|
||||
const statsSyncConfig = new sst.Linkable("StatsSyncConfig", {
|
||||
properties: {
|
||||
dataset: "zen",
|
||||
},
|
||||
})
|
||||
|
||||
export const statSync = new sst.aws.Service("StatsSyncService", {
|
||||
cluster: lakeCluster,
|
||||
architecture: "arm64",
|
||||
cpu: "0.25 vCPU",
|
||||
memory: "0.5 GB",
|
||||
image: {
|
||||
context: ".",
|
||||
dockerfile: "packages/stats/server/Dockerfile",
|
||||
},
|
||||
command: ["bun", "src/stat-sync.ts"],
|
||||
link: [database, inferenceEvent, statsSyncConfig],
|
||||
permissions: lakeQueryPermissions,
|
||||
scaling: {
|
||||
min: 1,
|
||||
max: 1,
|
||||
},
|
||||
dev: {
|
||||
command: "bun src/stat-sync.ts",
|
||||
directory: "packages/stats/server",
|
||||
autostart: false,
|
||||
},
|
||||
})
|
||||
Reference in New Issue
Block a user