From 72c2b8d0de9ce5b16ec72e01e88f45a12b6c86c7 Mon Sep 17 00:00:00 2001 From: DavidSeptimus-Klotho Date: Thu, 22 Dec 2022 16:40:45 -0700 Subject: [PATCH 01/11] Sanitizes AWS resource names in deploylib and related IAC files --- pkg/infra/pulumi_aws/deploylib.ts | 226 ++++++++++++------ pkg/infra/pulumi_aws/files.go | 2 +- pkg/infra/pulumi_aws/iac/elasticache.ts | 10 +- pkg/infra/pulumi_aws/iac/load_balancing.ts | 33 ++- pkg/infra/pulumi_aws/iac/memorydb.ts | 8 +- .../iac/sanitization/aws/app_runner.ts | 11 + .../iac/sanitization/aws/cloud_watch.ts | 17 ++ .../pulumi_aws/iac/sanitization/aws/common.ts | 33 +++ .../iac/sanitization/aws/dynamodb.ts | 13 + .../pulumi_aws/iac/sanitization/aws/ec2.ts | 40 ++++ .../pulumi_aws/iac/sanitization/aws/ecs.ts | 31 +++ .../pulumi_aws/iac/sanitization/aws/eks.ts | 20 ++ .../iac/sanitization/aws/elasticache.ts | 50 ++++ .../pulumi_aws/iac/sanitization/aws/elb.ts | 58 +++++ .../iac/sanitization/aws/eventbridge.ts | 11 + .../pulumi_aws/iac/sanitization/aws/iam.ts | 21 ++ .../pulumi_aws/iac/sanitization/aws/index.ts | 39 +++ .../pulumi_aws/iac/sanitization/aws/lambda.ts | 11 + .../iac/sanitization/aws/memorydb.ts | 52 ++++ .../pulumi_aws/iac/sanitization/aws/rds.ts | 79 ++++++ .../pulumi_aws/iac/sanitization/aws/s3.ts | 42 ++++ .../iac/sanitization/aws/secrets_manager.ts | 11 + .../iac/sanitization/aws/service_discovery.ts | 34 +++ .../pulumi_aws/iac/sanitization/aws/sns.ts | 33 +++ .../pulumi_aws/iac/sanitization/sanitizer.ts | 197 +++++++++++++++ pkg/infra/pulumi_aws/index.ts.tmpl | 2 +- pkg/infra/pulumi_aws/package-lock.json | 24 ++ pkg/infra/pulumi_aws/package.json | 6 +- pkg/infra/pulumi_aws/plugin_iac.go | 46 ++++ 29 files changed, 1060 insertions(+), 100 deletions(-) create mode 100644 pkg/infra/pulumi_aws/iac/sanitization/aws/app_runner.ts create mode 100644 pkg/infra/pulumi_aws/iac/sanitization/aws/cloud_watch.ts create mode 100644 pkg/infra/pulumi_aws/iac/sanitization/aws/common.ts create mode 100644 pkg/infra/pulumi_aws/iac/sanitization/aws/dynamodb.ts create mode 100644 pkg/infra/pulumi_aws/iac/sanitization/aws/ec2.ts create mode 100644 pkg/infra/pulumi_aws/iac/sanitization/aws/ecs.ts create mode 100644 pkg/infra/pulumi_aws/iac/sanitization/aws/eks.ts create mode 100644 pkg/infra/pulumi_aws/iac/sanitization/aws/elasticache.ts create mode 100644 pkg/infra/pulumi_aws/iac/sanitization/aws/elb.ts create mode 100644 pkg/infra/pulumi_aws/iac/sanitization/aws/eventbridge.ts create mode 100644 pkg/infra/pulumi_aws/iac/sanitization/aws/iam.ts create mode 100644 pkg/infra/pulumi_aws/iac/sanitization/aws/index.ts create mode 100644 pkg/infra/pulumi_aws/iac/sanitization/aws/lambda.ts create mode 100644 pkg/infra/pulumi_aws/iac/sanitization/aws/memorydb.ts create mode 100644 pkg/infra/pulumi_aws/iac/sanitization/aws/rds.ts create mode 100644 pkg/infra/pulumi_aws/iac/sanitization/aws/s3.ts create mode 100644 pkg/infra/pulumi_aws/iac/sanitization/aws/secrets_manager.ts create mode 100644 pkg/infra/pulumi_aws/iac/sanitization/aws/service_discovery.ts create mode 100644 pkg/infra/pulumi_aws/iac/sanitization/aws/sns.ts create mode 100644 pkg/infra/pulumi_aws/iac/sanitization/sanitizer.ts diff --git a/pkg/infra/pulumi_aws/deploylib.ts b/pkg/infra/pulumi_aws/deploylib.ts index 4fc0e3720..dd897c4df 100755 --- a/pkg/infra/pulumi_aws/deploylib.ts +++ b/pkg/infra/pulumi_aws/deploylib.ts @@ -1,7 +1,6 @@ -import { Region } from '@pulumi/aws' import * as aws from '@pulumi/aws' +import { Region } from '@pulumi/aws' import * as awsx from '@pulumi/awsx' -import * as k8s from '@pulumi/kubernetes' import * as pulumi from '@pulumi/pulumi' import * as sha256 from 'simple-sha256' @@ -11,9 +10,11 @@ import * as crypto from 'crypto' import { setupElasticacheCluster } from './iac/elasticache' import * as analytics from './iac/analytics' +import { h, sanitized, validate } from './iac/sanitization/sanitizer' import { LoadBalancerPlugin } from './iac/load_balancing' import { DefaultEksClusterOptions, Eks, EksExecUnit, HelmChart } from './iac/eks' import { setupMemoryDbCluster } from './iac/memorydb' +import AwsSanitizer from './iac/sanitization/aws' export enum Resource { exec_unit = 'exec_unit', @@ -112,7 +113,12 @@ export class CloudCCLib { if (this.createVPC) { this.getVpcSgSubnets() } - const resolvedBucketName = pulumi.interpolate`${this.account.accountId}${physicalPayloadsBucketName}` + const resolvedBucketName = this.account.accountId.apply( + (accountId) => + sanitized( + AwsSanitizer.S3.bucket.nameValidation() + )`${accountId}${physicalPayloadsBucketName}` + ) this.createBuckets([resolvedBucketName], true) this.addSharedPolicyStatement({ Effect: 'Allow', @@ -140,7 +146,6 @@ export class CloudCCLib { this.createVpcEndpoints() return } - if ( klothoVPC.id == undefined || klothoVPC.sgId == undefined || @@ -176,8 +181,11 @@ export class CloudCCLib { this.publicSubnetIds = this.klothoVPC.publicSubnetIds this.privateSubnetIds = this.klothoVPC.privateSubnetIds - const klothoSG = new aws.ec2.SecurityGroup(this.name, { - name: this.name, + const sgName = sanitized(AwsSanitizer.EC2.vpc.securityGroup.nameValidation())`${h( + this.name + )}` + const klothoSG = new aws.ec2.SecurityGroup(sgName, { + name: sgName, vpcId: this.klothoVPC.id, egress: [ { @@ -319,8 +327,11 @@ export class CloudCCLib { .forEach((item) => combinedPolicyStatements.add(item)) } if (combinedPolicyStatements.size > 0) { + const policyName = sanitized(AwsSanitizer.IAM.policy.nameValidation())`${h( + this.name + )}-${h(physicalName)}-exec` const policy = new aws.iam.Policy( - `${this.name}-${physicalName}-exec`, + policyName, { policy: { Version: '2012-10-17', @@ -330,7 +341,7 @@ export class CloudCCLib { { parent: role } ) new aws.iam.RolePolicyAttachment( - `${this.name}-${physicalName}-exec`, + policyName, { role: role, policyArn: policy.arn, @@ -355,7 +366,10 @@ export class CloudCCLib { Resource: ['*'], }) - const accessRole = new aws.iam.Role(`${execUnitName}-ar-access-role`, { + const roleName = sanitized(AwsSanitizer.IAM.role.nameValidation())`${h(this.name)}-${h( + execUnitName + )}-ar-access-role` + const accessRole = new aws.iam.Role(roleName, { assumeRolePolicy: { Version: '2012-10-17', Statement: [ @@ -371,7 +385,9 @@ export class CloudCCLib { }) const policy = new aws.iam.Policy( - `${execUnitName}-ar-access-policy`, + sanitized(AwsSanitizer.IAM.policy.nameValidation())`${h(this.name)}-${h( + execUnitName + )}-ar-access-policy`, { description: 'Role to grant AppRunner service access to ECR', policy: { @@ -414,9 +430,11 @@ export class CloudCCLib { const additionalEnvVars: { [key: string]: pulumi.Input } = this.generateExecUnitEnvVars(execUnitName, envVars) - const logGroupName = `/aws/apprunner/${this.name}-${execUnitName}-apprunner` + const logGroupName = sanitized( + AwsSanitizer.CloudWatch.logGroup.nameValidation() + )`/aws/apprunner/${h(this.name)}-${h(execUnitName)}-apprunner` let cloudwatchGroup = new aws.cloudwatch.LogGroup(`${this.name}-${execUnitName}-lg`, { - name: `${logGroupName}`, + name: logGroupName, retentionInDays: 1, }) @@ -429,8 +447,9 @@ export class CloudCCLib { } }) - const serviceName = `${this.name}-${execUnitName}-apprunner` - + const serviceName = sanitized(AwsSanitizer.AppRunner.service.nameValidation())`${h( + this.name + )}-${h(execUnitName)}-apprunner` const service = new aws.apprunner.Service(serviceName, { serviceName: serviceName, sourceConfiguration: { @@ -478,12 +497,16 @@ export class CloudCCLib { network_placement === 'public' ? this.publicSubnetIds : this.privateSubnetIds const lambdaRole = this.createRoleForName(execUnitName) + const lambdaName = sanitized(AwsSanitizer.Lambda.lambdaFunction.nameValidation())`${h( + this.name + )}-${h(execUnitName)}` + const lambdaConfig: aws.lambda.FunctionArgs = { ...baseArgs, packageType: 'Image', imageUri: image, role: lambdaRole.arn, - name: `${this.name}-${execUnitName}`, + name: lambdaName, tags: { env: 'production', service: execUnitName, @@ -501,8 +524,11 @@ export class CloudCCLib { } } + const logGroupName = sanitized( + AwsSanitizer.CloudWatch.logGroup.nameValidation() + )`/aws/lambda/${lambdaName}-function-api-lg` let cloudwatchGroup = new aws.cloudwatch.LogGroup(`${execUnitName}-function-api-lg`, { - name: pulumi.interpolate`/aws/lambda/${lambdaConfig.name}`, + name: logGroupName, retentionInDays: 1, }) @@ -644,6 +670,8 @@ export class CloudCCLib { subscribers: string[] ): aws.sns.Topic { let topic = `${this.name}_${name}_${event}` + // validate rather than sanitize since the PubSub runtime depends on a specific topic format + validate(topic, AwsSanitizer.SNS.topic.nameValidation()) if (topic.length > 256) { const hash = crypto.createHash('sha256') hash.update(topic) @@ -693,8 +721,9 @@ export class CloudCCLib { } setupKV(): aws.dynamodb.Table { + const tableName = sanitized(AwsSanitizer.DynamoDB.table.nameValidation())`${h(this.name)}` const db = new aws.dynamodb.Table( - `KV_${this.name}`, + `KV_${tableName}`, { attributes: [ { name: 'pk', type: 'S' }, @@ -709,7 +738,7 @@ export class CloudCCLib { attributeName: 'expiration', enabled: true, }, - name: this.name, + name: tableName, }, { protect: this.protect } ) @@ -789,38 +818,38 @@ export class CloudCCLib { } private createExecutionRole(execUnitPhysicalName: string) { - const lambdaExecRole = new aws.iam.Role( - `${this.name}_${this.generateHashFromPhysicalName(execUnitPhysicalName)}_LambdaExec`, - { - assumeRolePolicy: { - Version: '2012-10-17', - Statement: [ - { - Action: 'sts:AssumeRole', - Principal: { - Service: 'lambda.amazonaws.com', - }, - Effect: 'Allow', - Sid: '', + const roleName = sanitized(AwsSanitizer.IAM.role.nameValidation())`${h( + this.name + )}_${this.generateHashFromPhysicalName(execUnitPhysicalName)}_LambdaExec` + const lambdaExecRole = new aws.iam.Role(roleName, { + assumeRolePolicy: { + Version: '2012-10-17', + Statement: [ + { + Action: 'sts:AssumeRole', + Principal: { + Service: 'lambda.amazonaws.com', }, - { - Action: 'sts:AssumeRole', - Principal: { - Service: 'ecs-tasks.amazonaws.com', - }, - Effect: 'Allow', + Effect: 'Allow', + Sid: '', + }, + { + Action: 'sts:AssumeRole', + Principal: { + Service: 'ecs-tasks.amazonaws.com', }, - { - Action: 'sts:AssumeRole', - Principal: { - Service: 'tasks.apprunner.amazonaws.com', - }, - Effect: 'Allow', + Effect: 'Allow', + }, + { + Action: 'sts:AssumeRole', + Principal: { + Service: 'tasks.apprunner.amazonaws.com', }, - ], - }, - } - ) + Effect: 'Allow', + }, + ], + }, + }) // https://docs.aws.amazon.com/lambda/latest/dg/monitoring-cloudwatchlogs.html#monitoring-cloudwatchlogs-prereqs new aws.iam.RolePolicyAttachment(`${this.name}-${execUnitPhysicalName}-lambdabasic`, { role: lambdaExecRole, @@ -836,9 +865,11 @@ export class CloudCCLib { const lambdaNames = execUnitNames.map((n) => `${this.name}-${n}`) const warmerRole = this.createRoleForName(name) - + const warmerFuncName = sanitized(AwsSanitizer.Lambda.lambdaFunction.nameValidation())`${h( + this.name + )}-lambdawarmer` let warmerLambda = new aws.lambda.CallbackFunction(name, { - name: `${this.name}-lambdaWarmer`, + name: warmerFuncName, memorySize: 128 /*MB*/, timeout: 60, runtime: 'nodejs14.x', @@ -886,13 +917,18 @@ export class CloudCCLib { aws.cloudwatch.onSchedule('warmUpLambda', 'cron(0/5 * * * ? *)', warmUpLambda) } - public scheduleFunction(execGroupName, moduleName, functionName, cronExpression) { + public scheduleFunction(execUnitName, moduleName, functionName, cronExpression) { + const execGroupName = `${this.name}/${execUnitName}` const key = sha256.sync(cronExpression).slice(0, 5) const name = `${execGroupName}.${functionName}:${key}` const scheduleRole = this.createRoleForName(name) + const schedulerFuncName = sanitized( + AwsSanitizer.Lambda.lambdaFunction.nameValidation() + )`${h(this.name)}/${h(execUnitName)}_${h(functionName)}-${key}` + let lambdaScheduler = new aws.lambda.CallbackFunction(name, { - name: `${this.name}-${execGroupName}_${functionName}-${key}`, + name: schedulerFuncName, memorySize: 128 /*MB*/, timeout: 300, runtime: 'nodejs14.x', @@ -929,15 +965,22 @@ export class CloudCCLib { }) } - let cloudwatchLogs = new aws.cloudwatch.LogGroup(`${name}-function-api-lg`, { - name: pulumi.interpolate`/aws/lambda/${lambdaScheduler.id}`, + let cloudwatchLogs = new aws.cloudwatch.LogGroup(`${name}`, { + name: lambdaScheduler.id.apply( + (id) => + sanitized(AwsSanitizer.CloudWatch.logGroup.nameValidation())`/aws/lambda/${h( + name + )}-function-api-lg` + ), retentionInDays: 1, }) const schedulerLambda: aws.cloudwatch.EventRuleEventHandler = lambdaScheduler const warmUpLambdaSchedule: aws.cloudwatch.EventRuleEventSubscription = aws.cloudwatch.onSchedule( - `${execGroupName}_${functionName}_act`, + sanitized(AwsSanitizer.EventBridge.rule.nameValidation())`${h(execGroupName)}_${h( + functionName + )}_act`, `cron(${cronExpression})`, schedulerLambda ) @@ -945,6 +988,8 @@ export class CloudCCLib { public setupSecrets(secrets: string[]) { for (const secret of secrets) { + const secretName = `${this.name}-${secret}` + validate(secretName, AwsSanitizer.SecretsManager.secret.nameValidation()) let awsSecret: aws.secretsmanager.Secret if (this.secrets.has(secret)) { awsSecret = this.secrets.get(secret)! @@ -952,7 +997,7 @@ export class CloudCCLib { awsSecret = new aws.secretsmanager.Secret( `${secret}`, { - name: `${this.name}-${secret}`, + name: secretName, recoveryWindowInDays: 0, }, { protect: this.protect } @@ -990,7 +1035,10 @@ export class CloudCCLib { public setupRDS(orm: string, args: Partial) { if (!this.subnetGroup) { - this.subnetGroup = new aws.rds.SubnetGroup(this.name, { + const subnetGroupName = sanitized(AwsSanitizer.RDS.dbSubnetGroup.nameValidation())`${h( + this.name + )}` + this.subnetGroup = new aws.rds.SubnetGroup(subnetGroupName, { subnetIds: this.privateSubnetIds, tags: { Name: 'Klotho DB subnet group', @@ -998,12 +1046,15 @@ export class CloudCCLib { }) } - const dbName = orm.toLowerCase() + const dbName = sanitized( + AwsSanitizer.RDS.engine.pg.database.nameValidation() + )`${orm.toLowerCase()}` const config = new pulumi.Config() const username = config.require(`${dbName}_username`) const password = config.requireSecret(`${dbName}_password`) // create the db resources + validate(dbName, AwsSanitizer.RDS.instance.nameValidation()) const rds = new aws.rds.Instance( dbName, { @@ -1022,9 +1073,10 @@ export class CloudCCLib { // setup secrets for the proxy const secretName = `${dbName}_secret` - + const ssmSecretName = `${this.name}-${secretName}` + validate(ssmSecretName, AwsSanitizer.SecretsManager.secret.nameValidation()) let rdsSecret = new aws.secretsmanager.Secret(`${secretName}`, { - name: `${this.name}-${secretName}`, + name: ssmSecretName, recoveryWindowInDays: 0, }) @@ -1061,8 +1113,11 @@ export class CloudCCLib { } }) + // prettier-ignore + const ormRoleName = sanitized(AwsSanitizer.IAM.role.nameValidation())`${h(dbName)}-ormsecretrole` //setup role for proxy const role = new aws.iam.Role(`${dbName}-ormsecretrole`, { + name: ormRoleName, assumeRolePolicy: { Version: '2012-10-17', Statement: [ @@ -1077,7 +1132,9 @@ export class CloudCCLib { }, }) - const policy = new aws.iam.Policy(`${dbName}-ormsecretpolicy`, { + // prettier-ignore + const ormPolicyName = sanitized(AwsSanitizer.IAM.policy.nameValidation())`${h(dbName)}-ormsecretpolicy` + const policy = new aws.iam.Policy(ormPolicyName, { description: 'klotho orm secret policy', policy: { Version: '2012-10-17', @@ -1097,7 +1154,8 @@ export class CloudCCLib { }) // setup the rds proxy - const proxy = new aws.rds.Proxy(`${dbName}`, { + const proxyName = sanitized(AwsSanitizer.RDS.dbProxy.nameValidation())`${h(dbName)}` + const proxy = new aws.rds.Proxy(proxyName, { debugLogging: false, engineFamily: 'POSTGRESQL', idleClientTimeout: 1800, @@ -1268,8 +1326,9 @@ export class CloudCCLib { } createRoleForName(name: string): aws.iam.Role { - const role: aws.iam.Role = this.createExecutionRole(name) - this.execUnitToRole.set(name, role) + const roleName = sanitized(AwsSanitizer.IAM.role.nameValidation())`${name}` + const role: aws.iam.Role = this.createExecutionRole(roleName) + this.execUnitToRole.set(roleName, role) return role } @@ -1294,17 +1353,22 @@ export class CloudCCLib { this.privateDnsNamespace = new aws.servicediscovery.PrivateDnsNamespace( `${this.name}-privateDns`, { - name: `${this.name}-privateDns`, + name: sanitized( + AwsSanitizer.ServiceDiscovery.privateDnsNamespace.nameValidation() + )`${h(this.name)}-privateDns`, description: 'Used for service discovery', vpc: this.klothoVPC.id, } ) - this.cluster = new awsx.ecs.Cluster(`${this.name}-cluster`, { - vpc: this.klothoVPC, - cluster: providedClustername, - securityGroups: [], // otherwise, awsx creates a default one with 0.0.0.0/0. See #314 - }) + this.cluster = new awsx.ecs.Cluster( + sanitized(AwsSanitizer.ECS.cluster.nameValidation())`${h(this.name)}-cluster`, + { + vpc: this.klothoVPC, + cluster: providedClustername, + securityGroups: [], // otherwise, awsx creates a default one with 0.0.0.0/0. See #314 + } + ) } createEksResources = async ( @@ -1312,7 +1376,9 @@ export class CloudCCLib { charts?: HelmChart[], lbPlugin?: LoadBalancerPlugin ) => { - let clusterName = `${this.name}-eks-cluster` + let clusterName = sanitized(AwsSanitizer.EKS.cluster.nameValidation())`${h( + this.name + )}-eks-cluster` const providedClustername = kloConfig.get('eks-cluster') const existingCluster = undefined for (const execUnit of execUnits) { @@ -1404,7 +1470,10 @@ export class CloudCCLib { this.execUnitToVpcLink.set(execUnitName, vpcLink) } - const logGroupName = `/aws/fargate/${this.name}-${execUnitName}-task` + const logGroupName = sanitized( + AwsSanitizer.CloudWatch.logGroup.nameValidation() + )`/aws/fargate/${h(this.name)}-${h(execUnitName)}-task` + let cloudwatchGroup = new aws.cloudwatch.LogGroup(`${this.name}-${execUnitName}-lg`, { name: `${logGroupName}`, retentionInDays: 1, @@ -1436,7 +1505,9 @@ export class CloudCCLib { const task = new awsx.ecs.FargateTaskDefinition(`${execUnitName}-task`, { logGroup: cloudwatchGroup, - family: `${execUnitName}-family`, + family: sanitized(AwsSanitizer.ECS.taskDefinition.familyValidation())`${h( + execUnitName + )}-family`, executionRole: role, taskRole: role, container: { @@ -1478,6 +1549,9 @@ export class CloudCCLib { const service = new awsx.ecs.FargateService( `${execUnitName}-service`, { + name: sanitized(AwsSanitizer.ECS.service.nameValidation())`${h( + execUnitName + )}-service}`, cluster: this.cluster, taskDefinition: task, desiredCount: 1, @@ -1500,7 +1574,9 @@ export class CloudCCLib { ) => { if (type === 'elasticache') { const subnetGroup = new aws.elasticache.SubnetGroup( - `${this.name}-${name}-subnetgroup`.replace('_', '-').toLocaleLowerCase(), + sanitized( + AwsSanitizer.Elasticache.cacheSubnetGroup.cacheSubnetGroupNameValidation() + )`${h(this.name)}-${h(name)}-subnetgroup`, { subnetIds: this.privateSubnetIds, tags: { @@ -1549,7 +1625,9 @@ export class CloudCCLib { } const subnetGroup = new aws.memorydb.SubnetGroup( - `${this.name}-${name}-subnetgroup`.replace('_', '-').toLocaleLowerCase(), + sanitized(AwsSanitizer.MemoryDB.subnetGroup.subnetGroupNameValidation())`${ + this.name + }-${h(name)}-subnetgroup`, { subnetIds: subnets, tags: { diff --git a/pkg/infra/pulumi_aws/files.go b/pkg/infra/pulumi_aws/files.go index 0303c97aa..12f104163 100644 --- a/pkg/infra/pulumi_aws/files.go +++ b/pkg/infra/pulumi_aws/files.go @@ -6,7 +6,7 @@ import ( "github.com/klothoplatform/klotho/pkg/templateutils" ) -//go:embed *.tmpl *.ts *.json iac/*.ts iac/k8s/* +//go:embed *.tmpl *.ts *.json iac/*.ts iac/k8s/* iac/sanitization/* var files embed.FS var index = templateutils.MustTemplate(files, "index.ts.tmpl") diff --git a/pkg/infra/pulumi_aws/iac/elasticache.ts b/pkg/infra/pulumi_aws/iac/elasticache.ts index 26136e4c8..c12af1952 100644 --- a/pkg/infra/pulumi_aws/iac/elasticache.ts +++ b/pkg/infra/pulumi_aws/iac/elasticache.ts @@ -1,7 +1,8 @@ import * as aws from '@pulumi/aws' import * as pulumi from '@pulumi/pulumi' -import { PulumiFn } from '@pulumi/pulumi/automation' -import { Resource, CloudCCLib } from '../deploylib' +import { Resource } from '../deploylib' +import * as validators from './sanitization/aws/elasticache' +import { sanitized } from './sanitization/sanitizer' const ELASTICACHE_ENGINE = 'redis' @@ -46,7 +47,10 @@ export const setupElasticacheCluster = ( retentionInDays: 0, }) - const clusterName = sanitizeClusterName(appName, dbName) + // TODO: look into removing sanitizeClusterName when making other breaking changes to resource names + const clusterName = sanitized( + validators.cacheCluster.cacheClusterIdValidation() + )`${sanitizeClusterName(appName, dbName)}` // create the db resources const redis = new aws.elasticache.Cluster( clusterName, diff --git a/pkg/infra/pulumi_aws/iac/load_balancing.ts b/pkg/infra/pulumi_aws/iac/load_balancing.ts index 6eb59dc38..f0ecdd25e 100644 --- a/pkg/infra/pulumi_aws/iac/load_balancing.ts +++ b/pkg/infra/pulumi_aws/iac/load_balancing.ts @@ -1,22 +1,13 @@ -import { Region } from '@pulumi/aws' import * as aws from '@pulumi/aws' -import * as awsx from '@pulumi/awsx' -import * as k8s from '@pulumi/kubernetes' - -import * as pulumi from '@pulumi/pulumi' -import * as sha256 from 'simple-sha256' -import * as fs from 'fs' -import * as requestRetry from 'requestretry' -import * as crypto from 'crypto' - -import * as eks from '@pulumi/eks' +import * as validators from './sanitization/aws/elb' import { ListenerArgs, - TargetGroupArgs, LoadBalancerArgs, + TargetGroupArgs, TargetGroupAttachmentArgs, } from '@pulumi/aws/lb' import { ListenerRuleArgs } from '@pulumi/aws/alb' +import { h, sanitized } from './sanitization/sanitizer' import { CloudCCLib } from '../deploylib' export interface Route { @@ -176,9 +167,12 @@ export class LoadBalancerPlugin { params: LoadBalancerArgs ): aws.lb.LoadBalancer => { let lb: aws.lb.LoadBalancer + let lbName = sanitized(validators.loadBalancer.nameValidation())`${h(appName)}-${h( + resourceId + )}` switch (params.loadBalancerType) { case 'application': - lb = new aws.lb.LoadBalancer(`${appName}-${resourceId}`, { + lb = new aws.lb.LoadBalancer(lbName, { internal: params.internal || false, loadBalancerType: 'application', securityGroups: params.securityGroups, @@ -188,7 +182,7 @@ export class LoadBalancerPlugin { }) break case 'network': - lb = new aws.lb.LoadBalancer(`${appName}-${resourceId}`, { + lb = new aws.lb.LoadBalancer(lbName, { internal: params.internal || true, loadBalancerType: 'network', subnets: params.subnets, @@ -237,12 +231,15 @@ export class LoadBalancerPlugin { params: TargetGroupArgs ): aws.lb.TargetGroup => { let targetGroup: aws.lb.TargetGroup + let tgName = sanitized(validators.targetGroup.nameValidation())`${h(appName)}-${h( + execUnitName + )}` if (params.targetType != 'lambda' && !(params.port && params.protocol)) { throw new Error('Port and Protocol must be specified for non lambda target types') } switch (params.targetType) { case 'ip': - targetGroup = new aws.lb.TargetGroup(`${appName}-${execUnitName}`, { + targetGroup = new aws.lb.TargetGroup(tgName, { port: params.port, protocol: params.protocol, targetType: 'ip', @@ -251,7 +248,7 @@ export class LoadBalancerPlugin { }) break case 'instance': - targetGroup = new aws.lb.TargetGroup(`${appName}-${execUnitName}`, { + targetGroup = new aws.lb.TargetGroup(tgName, { port: params.port, protocol: params.protocol, vpcId: params.vpcId, @@ -259,7 +256,7 @@ export class LoadBalancerPlugin { }) break case 'alb': - targetGroup = new aws.lb.TargetGroup(`${appName}-${execUnitName}`, { + targetGroup = new aws.lb.TargetGroup(tgName, { targetType: 'alb', port: params.port, protocol: params.protocol, @@ -269,7 +266,7 @@ export class LoadBalancerPlugin { }) break case 'lambda': - targetGroup = new aws.lb.TargetGroup(`${appName}-${execUnitName}`, { + targetGroup = new aws.lb.TargetGroup(tgName, { targetType: 'lambda', tags: params.tags, }) diff --git a/pkg/infra/pulumi_aws/iac/memorydb.ts b/pkg/infra/pulumi_aws/iac/memorydb.ts index a7193bf3e..df32ac6d8 100644 --- a/pkg/infra/pulumi_aws/iac/memorydb.ts +++ b/pkg/infra/pulumi_aws/iac/memorydb.ts @@ -1,6 +1,8 @@ import * as aws from '@pulumi/aws' import * as pulumi from '@pulumi/pulumi' import { Resource } from '../deploylib' +import { sanitized } from './sanitization/sanitizer' +import { cacheCluster } from './sanitization/aws/memorydb' const sanitizeClusterName = (appName: string, dbName: string): string => { let cluster = `${appName}-${dbName}` @@ -37,7 +39,11 @@ export const setupMemoryDbCluster = ( securityGroupIds: pulumi.Output[], appName: string ) => { - const clusterName = sanitizeClusterName(appName, dbName) + // TODO: look into removing sanitizeClusterName when making other breaking changes to resource names + const clusterName = sanitized(cacheCluster.clusterNameValidation())`${sanitizeClusterName( + appName, + dbName + )}` const memdbCluster = new aws.memorydb.Cluster( clusterName, { diff --git a/pkg/infra/pulumi_aws/iac/sanitization/aws/app_runner.ts b/pkg/infra/pulumi_aws/iac/sanitization/aws/app_runner.ts new file mode 100644 index 000000000..ecc65411b --- /dev/null +++ b/pkg/infra/pulumi_aws/iac/sanitization/aws/app_runner.ts @@ -0,0 +1,11 @@ +import { regexpMatch } from '../sanitizer' + +export const service = { + nameValidation() { + return { + minLength: 4, + maxLength: 40, + rules: [regexpMatch('', /^[\w-]+$/, (n) => n.replace(/[^\w-]/g, '-'))], + } + }, +} diff --git a/pkg/infra/pulumi_aws/iac/sanitization/aws/cloud_watch.ts b/pkg/infra/pulumi_aws/iac/sanitization/aws/cloud_watch.ts new file mode 100644 index 000000000..1ad150a15 --- /dev/null +++ b/pkg/infra/pulumi_aws/iac/sanitization/aws/cloud_watch.ts @@ -0,0 +1,17 @@ +import { regexpMatch } from '../sanitizer' + +export const logGroup = { + nameValidation() { + return { + minLength: 1, + maxLength: 512, + rules: [ + regexpMatch( + "Log group names consist of the following characters: a-z, A-Z, 0-9, '_' (underscore), '-' (hyphen), '/' (forward slash), '.' (period), and '#' (number sign).", + /^[-._/#A-Za-z\d]+$/, + (n) => n.replace(/[^-._/#A-Za-z\d]/g, '_') + ), + ], + } + }, +} diff --git a/pkg/infra/pulumi_aws/iac/sanitization/aws/common.ts b/pkg/infra/pulumi_aws/iac/sanitization/aws/common.ts new file mode 100644 index 000000000..21d6b7853 --- /dev/null +++ b/pkg/infra/pulumi_aws/iac/sanitization/aws/common.ts @@ -0,0 +1,33 @@ +import { regexpMatch } from '../sanitizer' + +export const tag = { + keyValidation() { + return { + minLength: 1, + maxLength: 128, + rules: [ + regexpMatch('', /^[\p{L}\p{Z}\p{N}_.:/=+\-@]+$/u, (n) => + n.replace(/[\p{L}\p{Z}\p{N}_.:/=+\-@]/gu, '_') + ), + ], + } + }, + + valueValidation() { + return { + minLength: 0, + maxLength: 256, + rules: [ + regexpMatch('', /^[\p{L}\p{Z}\p{N}_.:/=+\-@]+$/u, (n) => + n.replace(/[\p{L}\p{Z}\p{N}_.:/=+\-@]/gu, '_') + ), + { + description: + "The aws: prefix is prohibited for tags; it's reserved for AWS use.", + apply: (v) => !v.startsWith('aws:'), + fix: (v) => v.replace(/^aws:/, ''), + }, + ], + } + }, +} diff --git a/pkg/infra/pulumi_aws/iac/sanitization/aws/dynamodb.ts b/pkg/infra/pulumi_aws/iac/sanitization/aws/dynamodb.ts new file mode 100644 index 000000000..3e7f7b5d0 --- /dev/null +++ b/pkg/infra/pulumi_aws/iac/sanitization/aws/dynamodb.ts @@ -0,0 +1,13 @@ +import { regexpMatch } from '../sanitizer' + +export const table = { + nameValidation() { + return { + minLength: 3, + maxLength: 255, + rules: [ + regexpMatch('', /^[a-zA-Z0-9_.-]+$/, (n) => n.replace(/[^a-zA-Z0-9_.-]/g, '_')), + ], + } + }, +} diff --git a/pkg/infra/pulumi_aws/iac/sanitization/aws/ec2.ts b/pkg/infra/pulumi_aws/iac/sanitization/aws/ec2.ts new file mode 100644 index 000000000..c12714859 --- /dev/null +++ b/pkg/infra/pulumi_aws/iac/sanitization/aws/ec2.ts @@ -0,0 +1,40 @@ +import { regexpMatch } from '../sanitizer' + +export const vpc = { + securityGroup: { + nameValidation() { + return { + minLength: 1, + maxLength: 255, + rules: [ + regexpMatch( + 'a-z, A-Z, 0-9, spaces, and ._-:/()#,@[]+=&;{}!$*', + /^[\w -.:/()#,@\[\]+=&;{}!$*]+$/, + (s) => s.replace(/[^\w -.:/()#,@\[\]+=&;{}!$*]/g, '_') + ), + ], + } + }, + }, +} + +export const classic = { + securityGroup: { + nameValidation() { + return { + minLength: 1, + maxLength: 255, + rules: [ + regexpMatch('Must only containASCII characters', /^[[:ascii:]]+$/, (s) => + s.replace(/[^[:ascii:]]/g, '_') + ), + { + description: "Cannot start with 'sg-'", + validate: (s) => !s.startsWith('sg-'), + fix: (s) => s.substring(3), + }, + ], + } + }, + }, +} diff --git a/pkg/infra/pulumi_aws/iac/sanitization/aws/ecs.ts b/pkg/infra/pulumi_aws/iac/sanitization/aws/ecs.ts new file mode 100644 index 000000000..506f759a9 --- /dev/null +++ b/pkg/infra/pulumi_aws/iac/sanitization/aws/ecs.ts @@ -0,0 +1,31 @@ +import { regexpMatch } from '../sanitizer' + +export const cluster = { + nameValidation() { + return { + minLength: 1, + maxLength: 255, + rules: [regexpMatch('', /^[\w-]+$/, (s) => s.replace(/[^\w-]/g, '_'))], + } + }, +} + +export const taskDefinition = { + familyValidation() { + return { + minLength: 1, + maxLength: 255, + rules: [regexpMatch('', /^[\w-]+$/, (s) => s.replace(/[^\w-]/g, '_'))], + } + }, +} + +export const service = { + nameValidation() { + return { + minLength: 1, + maxLength: 255, + rules: [regexpMatch('', /^[\w-]+$/, (s) => s.replace(/[^\w-]/g, '_'))], + } + }, +} diff --git a/pkg/infra/pulumi_aws/iac/sanitization/aws/eks.ts b/pkg/infra/pulumi_aws/iac/sanitization/aws/eks.ts new file mode 100644 index 000000000..c1722fc43 --- /dev/null +++ b/pkg/infra/pulumi_aws/iac/sanitization/aws/eks.ts @@ -0,0 +1,20 @@ +import { regexpMatch } from '../sanitizer' + +export const cluster = { + nameValidation() { + return { + minLength: 1, + maxLength: 100, + rules: [ + regexpMatch( + 'The name can contain only alphanumeric characters (case-sensitive) and hyphens.', + /^[a-zA-Z\d-]+$/, + (s) => s.replace(/[^a-zA-Z\d-]/g, '_') + ), + regexpMatch('The name must start with an alphabetic character', /^[a-zA-Z]/, (s) => + s.replace(/^[^a-zA-Z]+/, '') + ), + ], + } + }, +} diff --git a/pkg/infra/pulumi_aws/iac/sanitization/aws/elasticache.ts b/pkg/infra/pulumi_aws/iac/sanitization/aws/elasticache.ts new file mode 100644 index 000000000..cb3bbafc1 --- /dev/null +++ b/pkg/infra/pulumi_aws/iac/sanitization/aws/elasticache.ts @@ -0,0 +1,50 @@ +import { regexpMatch, regexpNotMatch } from '../sanitizer' + +export const cacheCluster = { + cacheClusterIdValidation() { + return { + minLength: 1, + maxLength: 50, + rules: [ + regexpMatch('', /[a-zA-Z0-9-]/, (s) => s.replace(/[^a-zA-Z0-9-]/g, '-')), + { + description: 'Identifier must not end with a hyphen', + validate: (s) => !s.endsWith('-'), + fix: (s) => s.replace(/-+$/, ''), + }, + regexpNotMatch('Identifier must not contain consecutive hyphens', /--/, (s) => + s.replace(/--+/g, '-') + ), + regexpMatch('Identifier must start with a letter', /^[a-zA-Z]/, (s) => + s.replace(/^[^a-zA-Z]+/, '') + ), + ], + } + }, +} +export const cacheSubnetGroup = { + cacheSubnetGroupNameValidation() { + return { + minLength: 1, + maxLength: 255, + rules: [ + regexpMatch( + '', + /^[a-z\d-]+$/, // uppercase is technically valid, but AWS will convert the value to lowercase + (s) => s.toLocaleLowerCase().replace(/[^a-z\d-]/g, '-') + ), + { + description: 'Identifier must not end with a hyphen', + validate: (s) => !s.endsWith('-'), + fix: (s) => s.replace(/-+$/, ''), + }, + regexpNotMatch('Identifier must not contain consecutive hyphens', /--/, (s) => + s.replace(/--+/g, '-') + ), + regexpMatch('Identifier must start with a letter', /^[a-zA-Z]/, (s) => + s.replace(/^[^a-zA-Z]+/, '') + ), + ], + } + }, +} diff --git a/pkg/infra/pulumi_aws/iac/sanitization/aws/elb.ts b/pkg/infra/pulumi_aws/iac/sanitization/aws/elb.ts new file mode 100644 index 000000000..38bae2cb3 --- /dev/null +++ b/pkg/infra/pulumi_aws/iac/sanitization/aws/elb.ts @@ -0,0 +1,58 @@ +import { regexpMatch } from '../sanitizer' + +export const loadBalancer = { + nameValidation() { + return { + minLength: 1, + maxLength: 32, + rules: [ + regexpMatch( + 'The name can contain only alphanumeric characters and hyphens.', + /^[a-zA-Z\d-]+$/, + (s) => s.replace(/[^a-zA-Z\d-]/g, '_') + ), + { + description: 'The name must not begin with a hyphen', + validate: (s) => !s.startsWith('-'), + fix: (s) => s.replace(/^-/, ''), + }, + { + description: 'The name must not end with a hyphen', + validate: (s) => !s.endsWith('-'), + fix: (s) => s.replace(/-$/, ''), + }, + { + description: "The name must start with 'internal-'", + validate: (s) => !s.startsWith('internal-'), + fix: (s) => s.replace(/^internal-/, ''), + }, + ], + } + }, +} + +export const targetGroup = { + nameValidation() { + return { + minLength: 1, + maxLength: 32, + rules: [ + regexpMatch( + 'The name can contain only alphanumeric characters and hyphens.', + /^[a-zA-Z\d-]+$/, + (s) => s.replace(/[^a-zA-Z\d-]/g, '_') + ), + { + description: 'The name must not begin with a hyphen', + validate: (s) => !s.startsWith('-'), + fix: (s) => s.replace(/^-/, ''), + }, + { + description: 'The name must not end with a hyphen', + validate: (s) => !s.endsWith('-'), + fix: (s) => s.replace(/-$/, ''), + }, + ], + } + }, +} diff --git a/pkg/infra/pulumi_aws/iac/sanitization/aws/eventbridge.ts b/pkg/infra/pulumi_aws/iac/sanitization/aws/eventbridge.ts new file mode 100644 index 000000000..ac6e8847e --- /dev/null +++ b/pkg/infra/pulumi_aws/iac/sanitization/aws/eventbridge.ts @@ -0,0 +1,11 @@ +import { regexpMatch } from '../sanitizer' + +export const rule = { + nameValidation() { + return { + minLength: 1, + maxLength: 64, + rules: [regexpMatch('', /^[\w-.]+$/, (n) => n.replace(/[^\w-.]/g, '_'))], + } + }, +} diff --git a/pkg/infra/pulumi_aws/iac/sanitization/aws/iam.ts b/pkg/infra/pulumi_aws/iac/sanitization/aws/iam.ts new file mode 100644 index 000000000..7359e9f3c --- /dev/null +++ b/pkg/infra/pulumi_aws/iac/sanitization/aws/iam.ts @@ -0,0 +1,21 @@ +import { regexpMatch } from '../sanitizer' + +export const role = { + nameValidation() { + return { + minLength: 1, + maxLength: 64, + rules: [regexpMatch('', /^[\w+=,.@-]+$/, (s) => s.replace(/[^\w+=,.@-]/g, '_'))], + } + }, +} + +export const policy = { + nameValidation() { + return { + minLength: 1, + maxLength: 128, + rules: [regexpMatch('', /^[\w+=,.@-]+$/, (s) => s.replace(/[^\w+=,.@-]/g, '_'))], + } + }, +} diff --git a/pkg/infra/pulumi_aws/iac/sanitization/aws/index.ts b/pkg/infra/pulumi_aws/iac/sanitization/aws/index.ts new file mode 100644 index 000000000..7cc492015 --- /dev/null +++ b/pkg/infra/pulumi_aws/iac/sanitization/aws/index.ts @@ -0,0 +1,39 @@ +import * as AppRunner from './app_runner' +import * as CloudWatch from './cloud_watch' +import * as Common from './common' +import * as DynamoDB from './dynamodb' +import * as EC2 from './ec2' +import * as ECS from './ecs' +import * as EKS from './eks' +import * as Elasticache from './elasticache' +import * as ELB from './elb' +import * as EventBridge from './eventbridge' +import * as IAM from './iam' +import * as Lambda from './lambda' +import * as MemoryDB from './memorydb' +import * as RDS from './rds' +import * as S3 from './s3' +import * as SecretsManager from './secrets_manager' +import * as ServiceDiscovery from './service_discovery' +import * as SNS from './sns' + +export default { + AppRunner, + CloudWatch, + Common, + DynamoDB, + EC2, + ECS, + EKS, + Elasticache, + ELB, + EventBridge, + IAM, + Lambda, + MemoryDB, + RDS, + S3, + SecretsManager, + ServiceDiscovery, + SNS, +} diff --git a/pkg/infra/pulumi_aws/iac/sanitization/aws/lambda.ts b/pkg/infra/pulumi_aws/iac/sanitization/aws/lambda.ts new file mode 100644 index 000000000..ec93f19b6 --- /dev/null +++ b/pkg/infra/pulumi_aws/iac/sanitization/aws/lambda.ts @@ -0,0 +1,11 @@ +import { regexpMatch } from '../sanitizer' + +export const lambdaFunction = { + nameValidation() { + return { + minLength: 1, + maxLength: 64, + rules: [regexpMatch('', /^[\w-]+$/, (s) => s.replace(/[^\w-]/g, '_'))], + } + }, +} diff --git a/pkg/infra/pulumi_aws/iac/sanitization/aws/memorydb.ts b/pkg/infra/pulumi_aws/iac/sanitization/aws/memorydb.ts new file mode 100644 index 000000000..f82c4e898 --- /dev/null +++ b/pkg/infra/pulumi_aws/iac/sanitization/aws/memorydb.ts @@ -0,0 +1,52 @@ +import { regexpMatch, regexpNotMatch } from '../sanitizer' + +export const cacheCluster = { + clusterNameValidation() { + return { + minLength: 1, + maxLength: 40, + rules: [ + regexpMatch('', /[a-zA-Z0-9-]/, (s) => s.replace(/[^a-zA-Z0-9-]/g, '-')), + { + description: 'Identifier must not end with a hyphen', + validate: (s) => !s.endsWith('-'), + fix: (s) => s.replace(/-+$/, ''), + }, + regexpNotMatch('Identifier must not contain consecutive hyphens', /--/, (s) => + s.replace(/--+/g, '-') + ), + regexpMatch('Identifier must start with a letter', /^[a-zA-Z]/, (s) => + s.replace(/^[^a-zA-Z]+/, '') + ), + ], + } + }, +} + +// subnetGroupNameValidation are not documented and are inferred from Elasticache's CacheSubnetGroupName +export const subnetGroup = { + subnetGroupNameValidation() { + return { + minLength: 1, + maxLength: 255, + rules: [ + regexpMatch( + '', + /^[a-z\d-]+$/, // uppercase is technically valid, but AWS will convert the value to lowercase + (s) => s.toLocaleLowerCase().replace(/[^a-z\d-]/g, '-') + ), + { + description: 'Identifier must not end with a hyphen', + validate: (s) => !s.endsWith('-'), + fix: (s) => s.replace(/-+$/, ''), + }, + regexpNotMatch('Identifier must not contain consecutive hyphens', /--/, (s) => + s.replace(/--+/g, '-') + ), + regexpMatch('Identifier must start with a letter', /^[a-zA-Z]/, (s) => + s.replace(/^[^a-zA-Z]+/, '') + ), + ], + } + }, +} diff --git a/pkg/infra/pulumi_aws/iac/sanitization/aws/rds.ts b/pkg/infra/pulumi_aws/iac/sanitization/aws/rds.ts new file mode 100644 index 000000000..3898b6ea1 --- /dev/null +++ b/pkg/infra/pulumi_aws/iac/sanitization/aws/rds.ts @@ -0,0 +1,79 @@ +import { regexpMatch } from '../sanitizer' + +export const dbSubnetGroup = { + nameValidation() { + return { + minLength: 1, + maxLength: 255, + rules: [ + regexpMatch('', /^[\w -.]+$/, (s) => s.replace(/[^\w -.]/g, '_')), + regexpMatch('Name must start with a letter', /^[a-zA-Z]/, (s) => + s.replace(/^[^a-zA-Z]+/, '') + ), + { + description: "Name must not be 'default'", + validate: (s) => s.toLowerCase() !== 'default', + }, + ], + } + }, +} + +export const engine = { + pg: { + database: { + nameValidation() { + return { + minLength: 1, + maxLength: 63, + rules: [], + } + }, + }, + }, +} + +export const dbProxy = { + nameValidation() { + return { + minLength: 1, + rules: [ + regexpMatch('', /^[\da-zA-Z-]+$/, (s) => s.replace(/[^\da-zA-Z-]/g, '-')), + regexpMatch('Identifier must start with a letter', /^[a-zA-Z]/, (s) => + s.replace(/^[^a-zA-Z]+/, '') + ), + { + description: 'Identifier must not end with a hyphen', + validate: (s) => !s.endsWith('-'), + fix: (s) => s.replace(/-+$/, ''), + }, + regexpMatch('Identifier must not contain consecutive hyphens', /--/, (s) => + s.replace(/--+/g, '-') + ), + ], + } + }, +} + +export const instance = { + nameValidation() { + return { + minLength: 1, + maxLength: 63, + rules: [ + regexpMatch('', /^[\da-zA-Z-]+$/, (s) => s.replace(/[^\da-zA-Z-]/g, '-')), + regexpMatch('Identifier must start with a letter', /^[a-zA-Z]/, (s) => + s.replace(/^[^a-zA-Z]+/, '') + ), + { + description: 'Identifier must not end with a hyphen', + validate: (s) => !s.endsWith('-'), + fix: (s) => s.replace(/-+$/, ''), + }, + regexpMatch('Identifier must not contain consecutive hyphens', /--/, (s) => + s.replace(/--+/g, '-') + ), + ], + } + }, +} diff --git a/pkg/infra/pulumi_aws/iac/sanitization/aws/s3.ts b/pkg/infra/pulumi_aws/iac/sanitization/aws/s3.ts new file mode 100644 index 000000000..2c70a854b --- /dev/null +++ b/pkg/infra/pulumi_aws/iac/sanitization/aws/s3.ts @@ -0,0 +1,42 @@ +import { regexpMatch, regexpNotMatch, SanitizationOptions } from '../sanitizer' + +export const bucket = { + nameValidation(): SanitizationOptions { + return { + minLength: 3, + maxLength: 63, + rules: [ + regexpMatch( + 'Bucket names can consist only of lowercase letters, numbers, dots (.), and hyphens (-).', + /^[a-z\d.-]+$/, + (n) => n.toLowerCase().replace(/[^a-z\d-.]+/g, '-') + ), + { + description: 'Bucket names must not start with the prefix "xn--".', + validate: (n) => !n.startsWith('xn--'), + fix: (n) => n.replace(/^xn--/, ''), + }, + { + description: 'Bucket names must not end with the suffix "-s3alias".', + validate: (n) => !n.endsWith('-s3alias'), + fix: (n) => n.replace(/-s3alias$/, ''), + }, + regexpMatch( + 'Bucket names must begin and end with a letter or number.', + /^[a-z\d].+[a-z\d]$/, + (n) => n.replace(/^[^a-zA-Z\d]+/, '').replace(/[^a-zA-Z\d]+$/g, '') + ), + { + description: 'Bucket names must not contain two adjacent periods.', + validate: (n) => !n.includes('..'), + fix: (n) => n.replaceAll('..', '.'), + }, + regexpNotMatch( + 'Bucket names must not be formatted as an IP address.', + /^(?:\d{1,3}\.){3}\d{1,3}$/, + (n) => n.replaceAll('.', '-') + ), + ], + } + }, +} diff --git a/pkg/infra/pulumi_aws/iac/sanitization/aws/secrets_manager.ts b/pkg/infra/pulumi_aws/iac/sanitization/aws/secrets_manager.ts new file mode 100644 index 000000000..47324c4af --- /dev/null +++ b/pkg/infra/pulumi_aws/iac/sanitization/aws/secrets_manager.ts @@ -0,0 +1,11 @@ +import { regexpMatch } from '../sanitizer' + +export const secret = { + nameValidation() { + return { + minLength: 1, + maxLength: 512, + rules: [regexpMatch('', /^[\w/+=.@-]+$/, (n) => n.replace(/[^\w/+=.@-]/g, '_'))], + } + }, +} diff --git a/pkg/infra/pulumi_aws/iac/sanitization/aws/service_discovery.ts b/pkg/infra/pulumi_aws/iac/sanitization/aws/service_discovery.ts new file mode 100644 index 000000000..a35b30002 --- /dev/null +++ b/pkg/infra/pulumi_aws/iac/sanitization/aws/service_discovery.ts @@ -0,0 +1,34 @@ +import { regexpMatch, regexpNotMatch, SanitizationOptions } from '../sanitizer' + +export const privateDnsNamespace = { + nameValidation(): SanitizationOptions { + return { + minLength: 1, + maxLength: 253, + rules: [regexpMatch('', /^[!-~]+$/, (s) => s.replace(/[^!-~]/g, '_'))], + } + }, +} + +export const service = { + nameValidation(): SanitizationOptions { + return { + minLength: 1, + maxLength: 127, + rules: [ + regexpMatch( + '', + /((?=^.{1,127}$)^([a-zA-Z0-9_][a-zA-Z0-9-_]{0,61}[a-zA-Z0-9_]|[a-zA-Z0-9])(\.([a-zA-Z0-9_][a-zA-Z0-9-_]{0,61}[a-zA-Z0-9_]|[a-zA-Z0-9]))*$)|(^\.$)/ + ), + regexpMatch( + 'Name can only contain alphanumeric characters, underscores, hyphens, and periods', + /^[\w.-]$/, + (s) => s.replace(/[^\w.-]/g, '_') + ), + regexpNotMatch('Name component must not start with a hyphen', /(^-)|(\.-)/, (s) => + s.replace(/(^-+)|((?<=\.)-+)/g, '') + ), + ], + } + }, +} diff --git a/pkg/infra/pulumi_aws/iac/sanitization/aws/sns.ts b/pkg/infra/pulumi_aws/iac/sanitization/aws/sns.ts new file mode 100644 index 000000000..f2c858af9 --- /dev/null +++ b/pkg/infra/pulumi_aws/iac/sanitization/aws/sns.ts @@ -0,0 +1,33 @@ +import { regexpMatch } from '../sanitizer' + +export const topic = { + nameValidation() { + return { + minLength: 1, + maxLength: 256, + rules: [regexpMatch('', /^[\w-]+$/, (s) => s.replace(/[^\w-]/g))], + } + }, +} + +export const fifoTopic = { + nameValidation() { + return { + minLength: 6, + maxLength: 256, + rules: [ + regexpMatch( + "The FIFO topic name must be made up of only uppercase and lowercase ASCII letters, numbers, underscores, and hyphens, and must end with the '.fifo' suffix", + /^[\w-]{1,251}\.fifo$/, + (s) => { + if (!s.endsWith('.fifo')) { + s = s.substring(0, s.length - 6) + } + s = `${s.replace(/[^\w-]/g, '-')}.fifo` + return s + } + ), + ], + } + }, +} diff --git a/pkg/infra/pulumi_aws/iac/sanitization/sanitizer.ts b/pkg/infra/pulumi_aws/iac/sanitization/sanitizer.ts new file mode 100644 index 000000000..e6196367b --- /dev/null +++ b/pkg/infra/pulumi_aws/iac/sanitization/sanitizer.ts @@ -0,0 +1,197 @@ +import Multimap = require('multimap') +import * as sha256 from 'simple-sha256' + +export interface SanitizationOptions { + maxLength?: number + minLength?: number + rules: Array + maxPasses?: number +} + +export interface SanitizationResult { + result: string + violations: Array +} + +export interface SanitizationRule extends ValidationRule { + fix?: FixFunc +} + +type FixFunc = (string) => string + +export interface ValidationRule { + description: string + + validate(string): boolean +} + +export function sanitize(s: string, options: Partial): SanitizationResult { + let result = s + let failedRules = new Array() + for (let i = 0; i < (options.maxPasses || 5); i++) { + let failedRules = options.rules?.filter((r) => !r.validate(result)) + failedRules?.forEach((f) => console.debug(f)) + if (options.minLength != null && result.length < options.minLength) { + throw new Error( + `The sanitized value, "${result}", is shorten than minLength: ${options.minLength}` + ) + } + if (options.maxLength != null && result.length > options.maxLength) { + result = result.substring(0, options.maxLength) + } + if (failedRules?.length === 0) { + return { result: result, violations: [] } + } + failedRules?.forEach((r) => (result = r.fix?.apply(this, [result]) || result)) + failedRules = options.rules?.filter((r) => !r.validate(result)) + failedRules?.forEach((f) => console.debug(f)) + if (failedRules?.length === 0) { + return { result, violations: [] } + } + } + return { result, violations: failedRules.map((r) => r.description) } +} + +export function validate(input: string, options: Partial) { + const violations = doValidate(input, options) + if (violations.length > 0) { + throw new Error(`Invalid input '${input}':\n\t${violations.join('\n\t')}`) + } +} + +function doValidate(input: string, options: Partial): Array { + const violations = new Array() + if (options.minLength != null && input.length < options.minLength) { + violations.push(`Invalid input: "${input}": length < minLength (${options.minLength})`) + } + if (options.maxLength != null && input.length > options.maxLength) { + violations.push(`Invalid input: "${input}": length > maxLength (${options.maxLength})`) + } + + violations.push( + ...(options.rules + ?.filter((r) => !r.validate(input)) + .map((r) => `validation rule violated: ${r.description}`) || []) + ) + return violations +} + +function hashComponent(input: string, maxLength: number): string { + maxLength = maxLength > 5 ? 5 : maxLength + const hash = sha256.sync(input) + return hash.substring(0, maxLength <= input.length ? maxLength : input.length) +} + +export function regexpMatch( + description: string, + pattern: RegExp, + fix: FixFunc | undefined = undefined +): SanitizationRule { + return { + description: description + ? description + : `The supplied string must match the following pattern: ${pattern.source}`, + validate: (input) => pattern.test(input), + fix, + } +} + +export function regexpNotMatch( + input: string, + pattern: RegExp, + fix: FixFunc | undefined = undefined +): SanitizationRule { + return { + description: input, + validate: (input) => !pattern.test(input), + fix, + } +} + +type ShorteningStrategy = (input: string, maxLength: number) => string + +export interface Component { + content: string + priority?: number + shorteningStrategy?: ShorteningStrategy +} + +function shortenString( + strings: TemplateStringsArray, + components: Array, + options: SanitizationOptions +) { + const resolvedComponents: Array = components.map((c) => { + return typeof c === 'string' ? { content: c } : { ...c } // clone each component to avoid unintended modification to inputs + }) + let length = + arrTextLen([...strings.raw], (s) => s.length) + + arrTextLen(resolvedComponents, (a) => a.content.length) + + let componentsByPriority = new Multimap() + resolvedComponents.forEach((c) => + componentsByPriority.set(c.priority === undefined ? 100 : c.priority, c) + ) + const sorted = [...componentsByPriority.keys()].sort() + + if (options.maxLength != null) { + for (const p of sorted) { + if (length <= options.maxLength) { + break + } + for (const c of componentsByPriority.get(p)) { + if (length <= options.maxLength) { + break + } + const oldCLen = c.content.length + c.content = c.shorteningStrategy + ? c.shorteningStrategy( + c.content, + options.maxLength - (length - c.content.length) + ) + : c.content + length -= oldCLen - c.content.length + } + } + } + + let result = '' + for (let i = 0; i < strings.length; i++) { + result += (strings[i] || '') + (resolvedComponents[i]?.content || '') + } + return result +} + +export function sanitized(options: SanitizationOptions) { + return function ( + strings: TemplateStringsArray, + ...components: Array + ): string { + const shortenedString = shortenString(strings, components, options) + const { result, violations } = sanitize(shortenedString, options) + if (violations.length > 0) { + throw new Error(`sanitization failed:\n\t${violations.join('\n\t')}`) + } + return result + } +} + +function arrTextLen(arr: Array, lenFunc: (arg0: T) => number): number { + return arr.reduce((p, c, i) => p + lenFunc(c), 0) +} + +export function h(content: string, priority: number | undefined = undefined): Component { + return { + content, + priority, + shorteningStrategy: hashComponent, + } +} + +export function t(content: string, priority: number | undefined = undefined): Component { + return { + content, + priority, + shorteningStrategy: (t, m) => t.substring(0, m), + } +} diff --git a/pkg/infra/pulumi_aws/index.ts.tmpl b/pkg/infra/pulumi_aws/index.ts.tmpl index 4c4e57847..528dbdb0e 100755 --- a/pkg/infra/pulumi_aws/index.ts.tmpl +++ b/pkg/infra/pulumi_aws/index.ts.tmpl @@ -124,7 +124,7 @@ export = async () => { keepWarm.push("{{$unit.Name}}"); {{end}} {{range $idx, $s := .Schedules}} - cloudLib.scheduleFunction("{{$cfg.AppName}}/{{$unit.Name}}", "{{$s.ModulePath}}", "{{$s.FuncName}}}", "{{$s.Cron}}"); + cloudLib.scheduleFunction("{{$unit.Name}}", "{{$s.ModulePath}}", "{{$s.FuncName}}}", "{{$s.Cron}}"); {{end -}} {{end -}} diff --git a/pkg/infra/pulumi_aws/package-lock.json b/pkg/infra/pulumi_aws/package-lock.json index f1f31f039..54f40d29b 100644 --- a/pkg/infra/pulumi_aws/package-lock.json +++ b/pkg/infra/pulumi_aws/package-lock.json @@ -16,11 +16,13 @@ "@pulumi/pulumi": "^3.40.2", "axios": "^0.27.2", "cdk8s-cli": "^2.1.4", + "multimap": "^1.1.0", "patch-package": "^6.4.7", "requestretry": "^7.0.2", "simple-sha256": "^1.1.0" }, "devDependencies": { + "@types/multimap": "^1.1.2", "@types/node": "^16.11.64", "@types/uuid": "^8.3.4" } @@ -580,6 +582,12 @@ "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.5.tgz", "integrity": "sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ==" }, + "node_modules/@types/multimap": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@types/multimap/-/multimap-1.1.2.tgz", + "integrity": "sha512-PDcGED9ZnrMI+A4ZiAZ8R7VkM2Uj+THkZ8vhbjXtfjJ5lLnr2kRI6hXo378hJTMQjEKMeXiJsf2P5H58uZDpyw==", + "dev": true + }, "node_modules/@types/node": { "version": "16.11.64", "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.64.tgz", @@ -3007,6 +3015,11 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "node_modules/multimap": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/multimap/-/multimap-1.1.0.tgz", + "integrity": "sha512-0ZIR9PasPxGXmRsEF8jsDzndzHDj7tIav+JUmvIFB/WHswliFnquxECT/De7GR4yg99ky/NlRKJT82G1y271bw==" + }, "node_modules/ncp": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz", @@ -4856,6 +4869,12 @@ "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.5.tgz", "integrity": "sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ==" }, + "@types/multimap": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@types/multimap/-/multimap-1.1.2.tgz", + "integrity": "sha512-PDcGED9ZnrMI+A4ZiAZ8R7VkM2Uj+THkZ8vhbjXtfjJ5lLnr2kRI6hXo378hJTMQjEKMeXiJsf2P5H58uZDpyw==", + "dev": true + }, "@types/node": { "version": "16.11.64", "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.64.tgz", @@ -6638,6 +6657,11 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "multimap": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/multimap/-/multimap-1.1.0.tgz", + "integrity": "sha512-0ZIR9PasPxGXmRsEF8jsDzndzHDj7tIav+JUmvIFB/WHswliFnquxECT/De7GR4yg99ky/NlRKJT82G1y271bw==" + }, "ncp": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz", diff --git a/pkg/infra/pulumi_aws/package.json b/pkg/infra/pulumi_aws/package.json index d9de84740..74d7d1a77 100755 --- a/pkg/infra/pulumi_aws/package.json +++ b/pkg/infra/pulumi_aws/package.json @@ -13,10 +13,12 @@ "cdk8s-cli": "^2.1.4", "patch-package": "^6.4.7", "requestretry": "^7.0.2", - "simple-sha256": "^1.1.0" + "simple-sha256": "^1.1.0", + "multimap": "^1.1.0" }, "devDependencies": { "@types/node": "^16.11.64", - "@types/uuid": "^8.3.4" + "@types/uuid": "^8.3.4", + "@types/multimap": "^1.1.2" } } diff --git a/pkg/infra/pulumi_aws/plugin_iac.go b/pkg/infra/pulumi_aws/plugin_iac.go index feae13448..1c5c62a1f 100644 --- a/pkg/infra/pulumi_aws/plugin_iac.go +++ b/pkg/infra/pulumi_aws/plugin_iac.go @@ -4,7 +4,9 @@ import ( "bytes" "encoding/json" "fmt" + "github.com/bmatcuk/doublestar/v4" "io" + "io/fs" "os" "strings" "text/template" @@ -152,6 +154,50 @@ func (p Plugin) Transform(result *core.CompilationResult, deps *core.Dependencie addFile("iac/k8s/add_ons/external_dns/index.ts") addFile("iac/k8s/add_ons/index.ts") + addDir := func(dir string, exclusions ...string) { + var unreadEntries []string + dirContents, err := files.ReadDir(dir) + + addEntries := func(parentDir string, entries []fs.DirEntry) { + for _, entry := range entries { + dirSuffix := "" + if entry.IsDir() { + dirSuffix = "/" + } + path := strings.TrimSuffix(parentDir, "/") + "/" + entry.Name() + dirSuffix + shouldInclude := true + for _, exclusion := range exclusions { + if shouldExclude, _ := doublestar.Match(exclusion, path); shouldExclude { + shouldInclude = false + } + } + if shouldInclude { + unreadEntries = append(unreadEntries, path) + } + } + } + addEntries(dir, dirContents) + + for len(unreadEntries) > 0 { + if err != nil { + return + } + + entry := unreadEntries[0] + unreadEntries = unreadEntries[1:] + if strings.HasSuffix(entry, "/") { + var childEntries []os.DirEntry + childEntries, err = files.ReadDir(strings.TrimSuffix(entry, "/")) + addEntries(entry, childEntries) + + } else { + addFile(entry) + } + } + } + + addDir("iac/sanitization", "**/*.{test,spec}.{ts,js}") + if err != nil { return err } From 5a37027549078dfee67b8b8cdb1ac9b4d2a00814 Mon Sep 17 00:00:00 2001 From: David Septimus <110404901+DavidSeptimus-Klotho@users.noreply.github.com> Date: Thu, 19 Jan 2023 17:30:00 -0500 Subject: [PATCH 02/11] Update pkg/infra/pulumi_aws/iac/sanitization/sanitizer.ts Co-authored-by: ewucc <92469267+ewucc@users.noreply.github.com> --- pkg/infra/pulumi_aws/iac/sanitization/sanitizer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/infra/pulumi_aws/iac/sanitization/sanitizer.ts b/pkg/infra/pulumi_aws/iac/sanitization/sanitizer.ts index e6196367b..c3fb80c7a 100644 --- a/pkg/infra/pulumi_aws/iac/sanitization/sanitizer.ts +++ b/pkg/infra/pulumi_aws/iac/sanitization/sanitizer.ts @@ -33,7 +33,7 @@ export function sanitize(s: string, options: Partial): Sani failedRules?.forEach((f) => console.debug(f)) if (options.minLength != null && result.length < options.minLength) { throw new Error( - `The sanitized value, "${result}", is shorten than minLength: ${options.minLength}` + `The sanitized value, "${result}", is shorter than minLength: ${options.minLength}` ) } if (options.maxLength != null && result.length > options.maxLength) { From c56a16c94285a6bb8f60d9bcbe83d943878bf410 Mon Sep 17 00:00:00 2001 From: DavidSeptimus-Klotho Date: Mon, 30 Jan 2023 07:43:01 -0700 Subject: [PATCH 03/11] rebase fixes --- pkg/infra/pulumi_aws/deploylib.ts | 3 +-- pkg/infra/pulumi_aws/iac/load_balancing.ts | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/infra/pulumi_aws/deploylib.ts b/pkg/infra/pulumi_aws/deploylib.ts index dd897c4df..520a4cd72 100755 --- a/pkg/infra/pulumi_aws/deploylib.ts +++ b/pkg/infra/pulumi_aws/deploylib.ts @@ -1403,8 +1403,7 @@ export class CloudCCLib { this, execUnits, charts || [], - existingCluster, - lbPlugin + existingCluster ) } diff --git a/pkg/infra/pulumi_aws/iac/load_balancing.ts b/pkg/infra/pulumi_aws/iac/load_balancing.ts index f0ecdd25e..924cab181 100644 --- a/pkg/infra/pulumi_aws/iac/load_balancing.ts +++ b/pkg/infra/pulumi_aws/iac/load_balancing.ts @@ -1,3 +1,4 @@ +import * as pulumi from '@pulumi/pulumi' import * as aws from '@pulumi/aws' import * as validators from './sanitization/aws/elb' import { From c87ee49dd0d69f0e24963080ec19aa5faaf1bc49 Mon Sep 17 00:00:00 2001 From: DavidSeptimus-Klotho Date: Mon, 30 Jan 2023 08:07:43 -0700 Subject: [PATCH 04/11] Adds reserved length for pulumi suffix --- pkg/infra/pulumi_aws/deploylib.ts | 2 +- pkg/infra/pulumi_aws/iac/load_balancing.ts | 2 +- .../pulumi_aws/iac/sanitization/sanitizer.ts | 36 +++++++++---------- 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/pkg/infra/pulumi_aws/deploylib.ts b/pkg/infra/pulumi_aws/deploylib.ts index 520a4cd72..7674ea183 100755 --- a/pkg/infra/pulumi_aws/deploylib.ts +++ b/pkg/infra/pulumi_aws/deploylib.ts @@ -10,7 +10,7 @@ import * as crypto from 'crypto' import { setupElasticacheCluster } from './iac/elasticache' import * as analytics from './iac/analytics' -import { h, sanitized, validate } from './iac/sanitization/sanitizer' +import { hash as h, sanitized, validate } from './iac/sanitization/sanitizer' import { LoadBalancerPlugin } from './iac/load_balancing' import { DefaultEksClusterOptions, Eks, EksExecUnit, HelmChart } from './iac/eks' import { setupMemoryDbCluster } from './iac/memorydb' diff --git a/pkg/infra/pulumi_aws/iac/load_balancing.ts b/pkg/infra/pulumi_aws/iac/load_balancing.ts index 924cab181..a66c97216 100644 --- a/pkg/infra/pulumi_aws/iac/load_balancing.ts +++ b/pkg/infra/pulumi_aws/iac/load_balancing.ts @@ -8,7 +8,7 @@ import { TargetGroupAttachmentArgs, } from '@pulumi/aws/lb' import { ListenerRuleArgs } from '@pulumi/aws/alb' -import { h, sanitized } from './sanitization/sanitizer' +import { hash as h, sanitized } from './sanitization/sanitizer' import { CloudCCLib } from '../deploylib' export interface Route { diff --git a/pkg/infra/pulumi_aws/iac/sanitization/sanitizer.ts b/pkg/infra/pulumi_aws/iac/sanitization/sanitizer.ts index c3fb80c7a..914f774e0 100644 --- a/pkg/infra/pulumi_aws/iac/sanitization/sanitizer.ts +++ b/pkg/infra/pulumi_aws/iac/sanitization/sanitizer.ts @@ -1,6 +1,9 @@ import Multimap = require('multimap') import * as sha256 from 'simple-sha256' +// Resource name length offset required to account for IAC-added suffixes +const reservedCharCount = 8 + export interface SanitizationOptions { maxLength?: number minLength?: number @@ -36,8 +39,8 @@ export function sanitize(s: string, options: Partial): Sani `The sanitized value, "${result}", is shorter than minLength: ${options.minLength}` ) } - if (options.maxLength != null && result.length > options.maxLength) { - result = result.substring(0, options.maxLength) + if (options.maxLength != null && result.length > options.maxLength - reservedCharCount) { + result = result.substring(0, options.maxLength - reservedCharCount) } if (failedRules?.length === 0) { return { result: result, violations: [] } @@ -64,8 +67,12 @@ function doValidate(input: string, options: Partial): Array if (options.minLength != null && input.length < options.minLength) { violations.push(`Invalid input: "${input}": length < minLength (${options.minLength})`) } - if (options.maxLength != null && input.length > options.maxLength) { - violations.push(`Invalid input: "${input}": length > maxLength (${options.maxLength})`) + if (options.maxLength != null && input.length > options.maxLength - reservedCharCount) { + violations.push( + `Invalid input: "${input}": length > maxLength (${ + options.maxLength - reservedCharCount + })` + ) } violations.push( @@ -135,20 +142,19 @@ function shortenString( const sorted = [...componentsByPriority.keys()].sort() if (options.maxLength != null) { + const maxLength = + options.maxLength - reservedCharCount < 1 ? 1 : options.maxLength - reservedCharCount for (const p of sorted) { - if (length <= options.maxLength) { + if (length <= maxLength) { break } for (const c of componentsByPriority.get(p)) { - if (length <= options.maxLength) { + if (length <= maxLength) { break } const oldCLen = c.content.length c.content = c.shorteningStrategy - ? c.shorteningStrategy( - c.content, - options.maxLength - (length - c.content.length) - ) + ? c.shorteningStrategy(c.content, maxLength - (length - c.content.length)) : c.content length -= oldCLen - c.content.length } @@ -180,18 +186,10 @@ function arrTextLen(arr: Array, lenFunc: (arg0: T) => number): number { return arr.reduce((p, c, i) => p + lenFunc(c), 0) } -export function h(content: string, priority: number | undefined = undefined): Component { +export function hash(content: string, priority: number | undefined = undefined): Component { return { content, priority, shorteningStrategy: hashComponent, } } - -export function t(content: string, priority: number | undefined = undefined): Component { - return { - content, - priority, - shorteningStrategy: (t, m) => t.substring(0, m), - } -} From 70340c47c5d36981359a898bb3f66063947dafc6 Mon Sep 17 00:00:00 2001 From: DavidSeptimus-Klotho Date: Mon, 30 Jan 2023 08:43:24 -0700 Subject: [PATCH 05/11] Fixes rds instance name double-hyphen rule --- pkg/infra/pulumi_aws/iac/sanitization/aws/rds.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/infra/pulumi_aws/iac/sanitization/aws/rds.ts b/pkg/infra/pulumi_aws/iac/sanitization/aws/rds.ts index 3898b6ea1..6ae9c5381 100644 --- a/pkg/infra/pulumi_aws/iac/sanitization/aws/rds.ts +++ b/pkg/infra/pulumi_aws/iac/sanitization/aws/rds.ts @@ -1,4 +1,4 @@ -import { regexpMatch } from '../sanitizer' +import { regexpMatch, regexpNotMatch } from '../sanitizer' export const dbSubnetGroup = { nameValidation() { @@ -47,7 +47,7 @@ export const dbProxy = { validate: (s) => !s.endsWith('-'), fix: (s) => s.replace(/-+$/, ''), }, - regexpMatch('Identifier must not contain consecutive hyphens', /--/, (s) => + regexpNotMatch('Identifier must not contain consecutive hyphens', /--/, (s) => s.replace(/--+/g, '-') ), ], @@ -70,7 +70,7 @@ export const instance = { validate: (s) => !s.endsWith('-'), fix: (s) => s.replace(/-+$/, ''), }, - regexpMatch('Identifier must not contain consecutive hyphens', /--/, (s) => + regexpNotMatch('Identifier must not contain consecutive hyphens', /--/, (s) => s.replace(/--+/g, '-') ), ], From 2c43a1f0610f802edc22f20afdcad834868a9c2a Mon Sep 17 00:00:00 2001 From: DavidSeptimus-Klotho Date: Tue, 31 Jan 2023 07:51:43 -0700 Subject: [PATCH 06/11] Fixes orm role name and lambda scheduler log group name --- pkg/infra/pulumi_aws/deploylib.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pkg/infra/pulumi_aws/deploylib.ts b/pkg/infra/pulumi_aws/deploylib.ts index 7674ea183..553d8e3b8 100755 --- a/pkg/infra/pulumi_aws/deploylib.ts +++ b/pkg/infra/pulumi_aws/deploylib.ts @@ -964,13 +964,12 @@ export class CloudCCLib { Resource: [this.execUnitToFunctions.get(execGroupName)!.arn], }) } - - let cloudwatchLogs = new aws.cloudwatch.LogGroup(`${name}`, { + let cloudwatchLogs = new aws.cloudwatch.LogGroup(`${name}-function-api-lg`, { name: lambdaScheduler.id.apply( (id) => sanitized(AwsSanitizer.CloudWatch.logGroup.nameValidation())`/aws/lambda/${h( - name - )}-function-api-lg` + id + )}` ), retentionInDays: 1, }) @@ -1116,8 +1115,7 @@ export class CloudCCLib { // prettier-ignore const ormRoleName = sanitized(AwsSanitizer.IAM.role.nameValidation())`${h(dbName)}-ormsecretrole` //setup role for proxy - const role = new aws.iam.Role(`${dbName}-ormsecretrole`, { - name: ormRoleName, + const role = new aws.iam.Role(ormRoleName, { assumeRolePolicy: { Version: '2012-10-17', Statement: [ From 7d4652f0e4e31c046395cb34dcce853016abd564 Mon Sep 17 00:00:00 2001 From: David Septimus <110404901+DavidSeptimus-Klotho@users.noreply.github.com> Date: Tue, 31 Jan 2023 18:07:34 -0500 Subject: [PATCH 07/11] adds quotes to message Co-authored-by: Yuval Shavit <110620369+yuval-klotho@users.noreply.github.com> --- pkg/infra/pulumi_aws/iac/sanitization/aws/common.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/infra/pulumi_aws/iac/sanitization/aws/common.ts b/pkg/infra/pulumi_aws/iac/sanitization/aws/common.ts index 21d6b7853..0ac50493c 100644 --- a/pkg/infra/pulumi_aws/iac/sanitization/aws/common.ts +++ b/pkg/infra/pulumi_aws/iac/sanitization/aws/common.ts @@ -23,7 +23,7 @@ export const tag = { ), { description: - "The aws: prefix is prohibited for tags; it's reserved for AWS use.", + "The \"aws:\" prefix is prohibited for tags; it's reserved for AWS use.", apply: (v) => !v.startsWith('aws:'), fix: (v) => v.replace(/^aws:/, ''), }, From be91a47135dc2de29ec52f55bd11266da803d425 Mon Sep 17 00:00:00 2001 From: DavidSeptimus-Klotho Date: Wed, 1 Feb 2023 07:58:47 -0700 Subject: [PATCH 08/11] Adds back sanitization for ELB and Service Discovery post-rebase --- pkg/infra/pulumi_aws/deploylib.ts | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/pkg/infra/pulumi_aws/deploylib.ts b/pkg/infra/pulumi_aws/deploylib.ts index 553d8e3b8..d6627e22d 100755 --- a/pkg/infra/pulumi_aws/deploylib.ts +++ b/pkg/infra/pulumi_aws/deploylib.ts @@ -1444,17 +1444,25 @@ export class CloudCCLib { let nlb if (needsLoadBalancer) { - nlb = new awsx.lb.NetworkLoadBalancer(`${execUnitName}-nlb`, { - external: false, - vpc: this.klothoVPC, - subnets: this.privateSubnetIds, - }) + nlb = new awsx.lb.NetworkLoadBalancer( + sanitized(AwsSanitizer.ELB.loadBalancer.nameValidation())`${h(execUnitName)}-nlb`, + { + external: false, + vpc: this.klothoVPC, + subnets: this.privateSubnetIds, + } + ) this.execUnitToNlb.set(execUnitName, nlb) const targetGroup: awsx.elasticloadbalancingv2.NetworkTargetGroup = - nlb.createTargetGroup(`${execUnitName}-tg`, { - port: 3000, - }) + nlb.createTargetGroup( + sanitized(AwsSanitizer.ELB.targetGroup.nameValidation())`${h( + execUnitName + )})-tg`, + { + port: 3000, + } + ) const listener = targetGroup.createListener(`${execUnitName}-listener`, { port: 80, @@ -1483,8 +1491,11 @@ export class CloudCCLib { additionalEnvVars.push({ name, value }) } - const discoveryService = new aws.servicediscovery.Service(execUnitName, { - name: execUnitName, + const serviceName = sanitized(AwsSanitizer.ServiceDiscovery.service.nameValidation())`${h( + execUnitName + )}` + const discoveryService = new aws.servicediscovery.Service(serviceName, { + name: serviceName, dnsConfig: { namespaceId: this.privateDnsNamespace.id, dnsRecords: [ From ddf07cc5140e53c0fabbfca816c95e866a92aee1 Mon Sep 17 00:00:00 2001 From: DavidSeptimus-Klotho Date: Wed, 1 Feb 2023 14:55:01 -0700 Subject: [PATCH 09/11] Revises addDir function in IAC plugin --- .../pulumi_aws/iac/sanitization/aws/common.ts | 2 +- pkg/infra/pulumi_aws/plugin_iac.go | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/pkg/infra/pulumi_aws/iac/sanitization/aws/common.ts b/pkg/infra/pulumi_aws/iac/sanitization/aws/common.ts index 0ac50493c..4c2ca567a 100644 --- a/pkg/infra/pulumi_aws/iac/sanitization/aws/common.ts +++ b/pkg/infra/pulumi_aws/iac/sanitization/aws/common.ts @@ -23,7 +23,7 @@ export const tag = { ), { description: - "The \"aws:\" prefix is prohibited for tags; it's reserved for AWS use.", + 'The "aws:" prefix is prohibited for tags; it\'s reserved for AWS use.', apply: (v) => !v.startsWith('aws:'), fix: (v) => v.replace(/^aws:/, ''), }, diff --git a/pkg/infra/pulumi_aws/plugin_iac.go b/pkg/infra/pulumi_aws/plugin_iac.go index 1c5c62a1f..8319767ba 100644 --- a/pkg/infra/pulumi_aws/plugin_iac.go +++ b/pkg/infra/pulumi_aws/plugin_iac.go @@ -154,10 +154,14 @@ func (p Plugin) Transform(result *core.CompilationResult, deps *core.Dependencie addFile("iac/k8s/add_ons/external_dns/index.ts") addFile("iac/k8s/add_ons/index.ts") - addDir := func(dir string, exclusions ...string) { + addDir := func(dir string, exclusions ...string) error { var unreadEntries []string dirContents, err := files.ReadDir(dir) + if err != nil { + return err + } + addEntries := func(parentDir string, entries []fs.DirEntry) { for _, entry := range entries { dirSuffix := "" @@ -169,6 +173,7 @@ func (p Plugin) Transform(result *core.CompilationResult, deps *core.Dependencie for _, exclusion := range exclusions { if shouldExclude, _ := doublestar.Match(exclusion, path); shouldExclude { shouldInclude = false + break } } if shouldInclude { @@ -179,24 +184,26 @@ func (p Plugin) Transform(result *core.CompilationResult, deps *core.Dependencie addEntries(dir, dirContents) for len(unreadEntries) > 0 { - if err != nil { - return - } - entry := unreadEntries[0] unreadEntries = unreadEntries[1:] if strings.HasSuffix(entry, "/") { var childEntries []os.DirEntry childEntries, err = files.ReadDir(strings.TrimSuffix(entry, "/")) + + if err != nil { + return err + } + addEntries(entry, childEntries) } else { addFile(entry) } } + return nil } - addDir("iac/sanitization", "**/*.{test,spec}.{ts,js}") + err = addDir("iac/sanitization", "**/*.{test,spec}.{ts,js}") if err != nil { return err From 656034440e07a38180551521df7de2910ac9b017 Mon Sep 17 00:00:00 2001 From: DavidSeptimus-Klotho Date: Wed, 1 Feb 2023 17:09:19 -0700 Subject: [PATCH 10/11] Fixes fix functions for tag keys and values --- pkg/infra/pulumi_aws/iac/sanitization/aws/common.ts | 4 ++-- pkg/infra/pulumi_aws/iac/sanitization/sanitizer.ts | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/infra/pulumi_aws/iac/sanitization/aws/common.ts b/pkg/infra/pulumi_aws/iac/sanitization/aws/common.ts index 4c2ca567a..80d1d5c4d 100644 --- a/pkg/infra/pulumi_aws/iac/sanitization/aws/common.ts +++ b/pkg/infra/pulumi_aws/iac/sanitization/aws/common.ts @@ -7,7 +7,7 @@ export const tag = { maxLength: 128, rules: [ regexpMatch('', /^[\p{L}\p{Z}\p{N}_.:/=+\-@]+$/u, (n) => - n.replace(/[\p{L}\p{Z}\p{N}_.:/=+\-@]/gu, '_') + n.replace(/[^\p{L}\p{Z}\p{N}_.:/=+\-@]/gu, '_') ), ], } @@ -19,7 +19,7 @@ export const tag = { maxLength: 256, rules: [ regexpMatch('', /^[\p{L}\p{Z}\p{N}_.:/=+\-@]+$/u, (n) => - n.replace(/[\p{L}\p{Z}\p{N}_.:/=+\-@]/gu, '_') + n.replace(/[^\p{L}\p{Z}\p{N}_.:/=+\-@]/gu, '_') ), { description: diff --git a/pkg/infra/pulumi_aws/iac/sanitization/sanitizer.ts b/pkg/infra/pulumi_aws/iac/sanitization/sanitizer.ts index 914f774e0..3e60f2b6f 100644 --- a/pkg/infra/pulumi_aws/iac/sanitization/sanitizer.ts +++ b/pkg/infra/pulumi_aws/iac/sanitization/sanitizer.ts @@ -104,12 +104,14 @@ export function regexpMatch( } export function regexpNotMatch( - input: string, + description: string, pattern: RegExp, fix: FixFunc | undefined = undefined ): SanitizationRule { return { - description: input, + description: description + ? description + : `The supplied string must not match the following pattern: ${pattern.source}`, validate: (input) => !pattern.test(input), fix, } From 9f35f12e85169736f5d97e927efb34d6c60fa84d Mon Sep 17 00:00:00 2001 From: DavidSeptimus-Klotho Date: Thu, 2 Feb 2023 07:40:45 -0700 Subject: [PATCH 11/11] Replacement pattern improvements for ELB --- pkg/infra/pulumi_aws/iac/sanitization/aws/elb.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/infra/pulumi_aws/iac/sanitization/aws/elb.ts b/pkg/infra/pulumi_aws/iac/sanitization/aws/elb.ts index 38bae2cb3..a55cd7f2d 100644 --- a/pkg/infra/pulumi_aws/iac/sanitization/aws/elb.ts +++ b/pkg/infra/pulumi_aws/iac/sanitization/aws/elb.ts @@ -45,12 +45,12 @@ export const targetGroup = { { description: 'The name must not begin with a hyphen', validate: (s) => !s.startsWith('-'), - fix: (s) => s.replace(/^-/, ''), + fix: (s) => s.replace(/^-+/, ''), }, { description: 'The name must not end with a hyphen', validate: (s) => !s.endsWith('-'), - fix: (s) => s.replace(/-$/, ''), + fix: (s) => s.replace(/-+$/, ''), }, ], }