feat: initial datalake and stats site (#28666)
This commit is contained in:
Adam
340
infra/lake.ts
Normal file
340
infra/lake.ts
Normal file
@@ -0,0 +1,340 @@
|
||||
import { domain } from "./stage"
|
||||
|
||||
const current = aws.getCallerIdentityOutput({})
|
||||
const partition = aws.getPartitionOutput({})
|
||||
const region = aws.getRegionOutput({})
|
||||
|
||||
const tableBucketName = `opencode-${$app.stage}-lake`
|
||||
const glueCatalogName = "s3tablescatalog"
|
||||
const glueCatalogArn = $interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:catalog`
|
||||
const glueS3TablesCatalogArn = $interpolate`${glueCatalogArn}/${glueCatalogName}`
|
||||
const glueS3TablesChildCatalogArn = $interpolate`${glueS3TablesCatalogArn}/${tableBucketName}`
|
||||
const glueS3TablesDatabaseWildcardArn = $interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:database/${glueCatalogName}/${tableBucketName}/*`
|
||||
const glueS3TablesTableWildcardArn = $interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:table/${glueCatalogName}/${tableBucketName}/*/*`
|
||||
const s3TablesBucketWildcardArn = $interpolate`arn:${partition.partition}:s3tables:${region.region}:${current.accountId}:bucket/*`
|
||||
|
||||
export const tableBucket = new aws.s3tables.TableBucket(
|
||||
"LakeTableBucket",
|
||||
{
|
||||
name: tableBucketName,
|
||||
forceDestroy: $app.stage !== "production",
|
||||
},
|
||||
)
|
||||
|
||||
const s3TablesCatalog = new aws.cloudcontrol.Resource(
|
||||
"LakeS3TablesCatalog",
|
||||
{
|
||||
typeName: "AWS::Glue::Catalog",
|
||||
desiredState: $jsonStringify({
|
||||
Name: glueCatalogName,
|
||||
Description: "Federated catalog for S3 Tables",
|
||||
FederatedCatalog: {
|
||||
Identifier: s3TablesBucketWildcardArn,
|
||||
ConnectionName: "aws:s3tables",
|
||||
},
|
||||
CreateDatabaseDefaultPermissions: [
|
||||
{
|
||||
Principal: {
|
||||
DataLakePrincipalIdentifier: "IAM_ALLOWED_PRINCIPALS",
|
||||
},
|
||||
Permissions: ["ALL"],
|
||||
},
|
||||
],
|
||||
CreateTableDefaultPermissions: [
|
||||
{
|
||||
Principal: {
|
||||
DataLakePrincipalIdentifier: "IAM_ALLOWED_PRINCIPALS",
|
||||
},
|
||||
Permissions: ["ALL"],
|
||||
},
|
||||
],
|
||||
AllowFullTableExternalDataAccess: "True",
|
||||
}),
|
||||
},
|
||||
{ dependsOn: [tableBucket] },
|
||||
)
|
||||
|
||||
const athenaResultsBucket = new aws.s3.Bucket(
|
||||
"LakeAthenaResults",
|
||||
{
|
||||
bucket: `opencode-${$app.stage}-lake-athena-results`,
|
||||
forceDestroy: $app.stage !== "production",
|
||||
},
|
||||
)
|
||||
|
||||
const firehoseErrorBucket = new aws.s3.Bucket(
|
||||
"LakeFirehoseErrors",
|
||||
{
|
||||
bucket: `opencode-${$app.stage}-lake-firehose-errors`,
|
||||
forceDestroy: $app.stage !== "production",
|
||||
},
|
||||
)
|
||||
|
||||
const athenaWorkgroup = new aws.athena.Workgroup(
|
||||
"LakeAthenaWorkgroup",
|
||||
{
|
||||
name: `opencode-${$app.stage}-lake-workgroup`,
|
||||
forceDestroy: $app.stage !== "production",
|
||||
configuration: {
|
||||
enforceWorkgroupConfiguration: true,
|
||||
publishCloudwatchMetricsEnabled: true,
|
||||
resultConfiguration: {
|
||||
outputLocation: $interpolate`s3://${athenaResultsBucket.bucket}/`,
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
const firehoseRole = new aws.iam.Role(
|
||||
"LakeFirehoseRole",
|
||||
{
|
||||
assumeRolePolicy: aws.iam.getPolicyDocumentOutput({
|
||||
statements: [
|
||||
{
|
||||
effect: "Allow",
|
||||
actions: ["sts:AssumeRole"],
|
||||
principals: [
|
||||
{
|
||||
type: "Service",
|
||||
identifiers: ["firehose.amazonaws.com"],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}).json,
|
||||
},
|
||||
)
|
||||
|
||||
const firehosePolicy = new aws.iam.RolePolicy(
|
||||
"LakeFirehosePolicy",
|
||||
{
|
||||
role: firehoseRole.id,
|
||||
policy: aws.iam.getPolicyDocumentOutput({
|
||||
statements: [
|
||||
{
|
||||
effect: "Allow",
|
||||
actions: [
|
||||
"s3tables:ListTableBuckets",
|
||||
"s3tables:GetTableBucket",
|
||||
"s3tables:GetNamespace",
|
||||
"s3tables:GetTable",
|
||||
"s3tables:GetTableData",
|
||||
"s3tables:GetTableMetadataLocation",
|
||||
"s3tables:ListNamespaces",
|
||||
"s3tables:ListTables",
|
||||
"s3tables:PutTableData",
|
||||
"s3tables:UpdateTableMetadataLocation",
|
||||
],
|
||||
resources: ["*"],
|
||||
},
|
||||
{
|
||||
effect: "Allow",
|
||||
actions: [
|
||||
"glue:GetCatalog",
|
||||
"glue:GetCatalogs",
|
||||
"glue:GetDatabase",
|
||||
"glue:GetDatabases",
|
||||
"glue:GetTable",
|
||||
"glue:GetTables",
|
||||
"glue:UpdateTable",
|
||||
],
|
||||
resources: [
|
||||
glueCatalogArn,
|
||||
glueS3TablesCatalogArn,
|
||||
$interpolate`${glueS3TablesCatalogArn}/*`,
|
||||
glueS3TablesDatabaseWildcardArn,
|
||||
glueS3TablesTableWildcardArn,
|
||||
$interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:database/*`,
|
||||
$interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:table/*/*`,
|
||||
$interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:table/${glueCatalogName}/*`,
|
||||
],
|
||||
},
|
||||
{
|
||||
effect: "Allow",
|
||||
actions: [
|
||||
"s3:AbortMultipartUpload",
|
||||
"s3:GetBucketLocation",
|
||||
"s3:GetObject",
|
||||
"s3:ListBucket",
|
||||
"s3:ListBucketMultipartUploads",
|
||||
"s3:PutObject",
|
||||
],
|
||||
resources: [firehoseErrorBucket.arn, $interpolate`${firehoseErrorBucket.arn}/*`],
|
||||
},
|
||||
{
|
||||
effect: "Allow",
|
||||
actions: ["lakeformation:GetDataAccess"],
|
||||
resources: ["*"],
|
||||
},
|
||||
],
|
||||
}).json,
|
||||
},
|
||||
)
|
||||
|
||||
const firehose = new aws.kinesis.FirehoseDeliveryStream(
|
||||
"LakeFirehose",
|
||||
{
|
||||
name: `opencode-${$app.stage}-lake-ingest`,
|
||||
destination: "iceberg",
|
||||
icebergConfiguration: {
|
||||
appendOnly: true,
|
||||
bufferingInterval: 60,
|
||||
bufferingSize: 1,
|
||||
catalogArn: glueS3TablesChildCatalogArn,
|
||||
processingConfiguration: {
|
||||
enabled: true,
|
||||
processors: [
|
||||
{
|
||||
type: "MetadataExtraction",
|
||||
parameters: [
|
||||
{ parameterName: "JsonParsingEngine", parameterValue: "JQ-1.6" },
|
||||
{
|
||||
parameterName: "MetadataExtractionQuery",
|
||||
parameterValue:
|
||||
'{destinationDatabaseName:._lake_database,destinationTableName:._lake_table,operation:(._lake_operation // "insert")}',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
roleArn: firehoseRole.arn,
|
||||
s3BackupMode: "FailedDataOnly",
|
||||
s3Configuration: {
|
||||
roleArn: firehoseRole.arn,
|
||||
bucketArn: firehoseErrorBucket.arn,
|
||||
errorOutputPrefix: "errors/!{firehose:error-output-type}/",
|
||||
},
|
||||
},
|
||||
},
|
||||
{ dependsOn: [s3TablesCatalog, firehosePolicy] },
|
||||
)
|
||||
|
||||
export const lakeVpc = new sst.aws.Vpc("LakeVpc")
|
||||
export const lakeCluster = new sst.aws.Cluster("LakeCluster", { vpc: lakeVpc })
|
||||
export const lakeRegion = region.region
|
||||
export const lakeCatalog = $interpolate`${glueCatalogName}/${tableBucket.name}`
|
||||
export const lakeAthenaWorkgroup = athenaWorkgroup
|
||||
|
||||
const ingestSecret = new random.RandomPassword("LakeIngestSecret", { length: 32 })
|
||||
|
||||
const ingestConfig = new sst.Linkable("LakeIngestConfig", {
|
||||
properties: {
|
||||
streamName: firehose.name,
|
||||
secret: ingestSecret.result,
|
||||
},
|
||||
})
|
||||
|
||||
const ingestService = new sst.aws.Service("LakeIngestService", {
|
||||
cluster: lakeCluster,
|
||||
architecture: "arm64",
|
||||
cpu: "0.5 vCPU",
|
||||
memory: "1 GB",
|
||||
image: {
|
||||
context: ".",
|
||||
dockerfile: "packages/stats/server/Dockerfile",
|
||||
},
|
||||
link: [ingestConfig],
|
||||
permissions: [
|
||||
{
|
||||
actions: ["firehose:PutRecord", "firehose:PutRecordBatch"],
|
||||
resources: [firehose.arn],
|
||||
},
|
||||
],
|
||||
scaling: {
|
||||
min: $app.stage === "production" ? 2 : 1,
|
||||
max: $app.stage === "production" ? 32 : 4,
|
||||
cpuUtilization: 60,
|
||||
memoryUtilization: 70,
|
||||
},
|
||||
loadBalancer: {
|
||||
domain: {
|
||||
name: `lake.${domain}`,
|
||||
dns: sst.cloudflare.dns(),
|
||||
},
|
||||
rules: [
|
||||
{ listen: "80/http", redirect: "443/https" },
|
||||
{ listen: "443/https", forward: "3000/http" },
|
||||
],
|
||||
health: {
|
||||
"3000/http": {
|
||||
path: "/ready",
|
||||
successCodes: "200-299",
|
||||
},
|
||||
},
|
||||
},
|
||||
health: {
|
||||
command: [
|
||||
"CMD-SHELL",
|
||||
"bun --eval \"fetch('http://localhost:3000/health').then((r) => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))\"",
|
||||
],
|
||||
interval: "30 seconds",
|
||||
retries: 3,
|
||||
startPeriod: "30 seconds",
|
||||
timeout: "5 seconds",
|
||||
},
|
||||
dev: {
|
||||
command: "bun run start",
|
||||
directory: "packages/stats/server",
|
||||
url: "http://localhost:3000",
|
||||
},
|
||||
wait: $app.stage === "production",
|
||||
})
|
||||
|
||||
export const lakeIngest = new sst.Linkable("LakeIngest", {
|
||||
properties: {
|
||||
url: ingestService.url,
|
||||
secret: ingestSecret.result,
|
||||
},
|
||||
})
|
||||
|
||||
export const lakeQueryPermissions = [
|
||||
{
|
||||
actions: ["athena:StartQueryExecution", "athena:GetQueryExecution", "athena:GetQueryResults"],
|
||||
resources: [athenaWorkgroup.arn],
|
||||
},
|
||||
{
|
||||
actions: [
|
||||
"glue:GetCatalog",
|
||||
"glue:GetCatalogs",
|
||||
"glue:GetDatabase",
|
||||
"glue:GetDatabases",
|
||||
"glue:GetTable",
|
||||
"glue:GetTables",
|
||||
"glue:GetPartitions",
|
||||
],
|
||||
resources: [
|
||||
glueCatalogArn,
|
||||
glueS3TablesCatalogArn,
|
||||
$interpolate`${glueS3TablesCatalogArn}/*`,
|
||||
glueS3TablesDatabaseWildcardArn,
|
||||
glueS3TablesTableWildcardArn,
|
||||
$interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:database/*`,
|
||||
$interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:table/*/*`,
|
||||
$interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:table/${glueCatalogName}/*`,
|
||||
],
|
||||
},
|
||||
{
|
||||
actions: ["s3:GetBucketLocation", "s3:ListBucket"],
|
||||
resources: [athenaResultsBucket.arn],
|
||||
},
|
||||
{
|
||||
actions: ["s3:GetObject", "s3:PutObject", "s3:AbortMultipartUpload", "s3:ListBucketMultipartUploads"],
|
||||
resources: [$interpolate`${athenaResultsBucket.arn}/*`],
|
||||
},
|
||||
{
|
||||
actions: [
|
||||
"s3tables:GetTableBucket",
|
||||
"s3tables:GetNamespace",
|
||||
"s3tables:GetTable",
|
||||
"s3tables:GetTableData",
|
||||
"s3tables:GetTableMetadataLocation",
|
||||
"s3tables:ListNamespaces",
|
||||
"s3tables:ListTables",
|
||||
],
|
||||
resources: ["*"],
|
||||
},
|
||||
{
|
||||
actions: ["lakeformation:GetDataAccess"],
|
||||
resources: ["*"],
|
||||
},
|
||||
]
|
||||
Reference in New Issue
Block a user