From 894a819f0bf09c548e4e8bc0377298651fa3f493 Mon Sep 17 00:00:00 2001 From: Cyril Fait Date: Mon, 25 Mar 2024 20:22:22 +0100 Subject: [PATCH] ssm parameters prefix --- deploy.sh | 8 +-- .../template-cicd-cfn-module.yaml | 22 +++---- .../nested-stacks/template-cicd-glue-job.yaml | 14 ++--- .../template-cicd-lambda-layer.yaml | 14 ++--- .../template-cicd-modules-pipelines.yaml | 6 +- sdlf-cicd/template-cfn-module.yaml | 2 +- sdlf-cicd/template-cicd-domain-roles.yaml | 20 +++---- sdlf-cicd/template-cicd-domain-team-role.yaml | 46 +++++++------- sdlf-cicd/template-cicd-domain.yaml | 26 ++++---- sdlf-cicd/template-cicd-prerequisites.yaml | 18 +++--- sdlf-cicd/template-cicd-sdlf-pipelines.yaml | 56 ++++++++--------- .../template-cicd-sdlf-repositories.yaml | 26 ++++---- sdlf-cicd/template-cicd-team-pipeline.yaml | 42 ++++++------- sdlf-cicd/template-cicd-team-repository.yaml | 4 +- sdlf-cicd/template-glue-job.yaml | 14 ++--- sdlf-cicd/template-lambda-layer.yaml | 4 +- .../configuration/resource_configs.py | 30 +++++----- sdlf-dataset/template.yaml | 18 +++--- .../lambda/replicate/src/lambda_function.py | 4 +- sdlf-foundations/template.yaml | 48 +++++++-------- sdlf-monitoring/template.yaml | 44 +++++++------- sdlf-pipeline/template.yaml | 28 ++++----- .../initial-check/src/lambda_function.py | 2 +- sdlf-stage-dataquality/template.yaml | 38 ++++++------ sdlf-stageA/template.yaml | 60 +++++++++---------- sdlf-stageB/template.yaml | 50 ++++++++-------- .../datasets-dynamodb/src/lambda_function.py | 2 +- .../pipelines-dynamodb/src/lambda_function.py | 2 +- sdlf-team/template.yaml | 52 ++++++++-------- .../pipeline-main.yaml | 4 +- .../workshop-examples/legislators/deploy.sh | 10 ++-- .../scripts/legislators-glue-job.yaml | 10 ++-- 32 files changed, 362 insertions(+), 362 deletions(-) diff --git a/deploy.sh b/deploy.sh index 4cc0dc00..d0a7e9bd 100755 --- a/deploy.sh +++ b/deploy.sh @@ -159,7 +159,7 @@ devops_account () { --profile "$DEVOPS_AWS_PROFILE" || exit 1 template_protection "$STACK_NAME" "$REGION" "$DEVOPS_AWS_PROFILE" - ARTIFACTS_BUCKET=$(aws --region "$REGION" --profile "$DEVOPS_AWS_PROFILE" ssm get-parameter --name /SDLF/S3/DevOpsArtifactsBucket --query "Parameter.Value" --output text) + ARTIFACTS_BUCKET=$(aws --region "$REGION" --profile "$DEVOPS_AWS_PROFILE" ssm get-parameter --name /SDLF2/S3/DevOpsArtifactsBucket --query "Parameter.Value" --output text) mkdir "$DIRNAME"/output aws cloudformation package \ --s3-bucket "$ARTIFACTS_BUCKET" --s3-prefix template-cicd-sdlf-repositories \ @@ -173,7 +173,7 @@ devops_account () { --stack-name "$STACK_NAME" \ --template-file "$DIRNAME"/output/packaged-template-cicd-sdlf-repositories.yaml \ --parameter-overrides \ - pKMSKey=/SDLF/KMS/CICDKeyId \ + pKMSKey=/SDLF2/KMS/CICDKeyId \ --tags Framework=sdlf \ --capabilities "CAPABILITY_NAMED_IAM" "CAPABILITY_AUTO_EXPAND" \ --region "$REGION" \ @@ -207,8 +207,8 @@ devops_account () { --stack-name "$STACK_NAME" \ --template-file "$DIRNAME"/output/packaged-template-cicd-sdlf-pipelines.yaml \ --parameter-overrides \ - pArtifactsBucket=/SDLF/S3/DevOpsArtifactsBucket \ - pKMSKey=/SDLF/KMS/CICDKeyId \ + pArtifactsBucket=/SDLF2/S3/DevOpsArtifactsBucket \ + pKMSKey=/SDLF2/KMS/CICDKeyId \ --tags Framework=sdlf \ --capabilities "CAPABILITY_NAMED_IAM" "CAPABILITY_AUTO_EXPAND" \ --region "$REGION" \ diff --git a/sdlf-cicd/nested-stacks/template-cicd-cfn-module.yaml b/sdlf-cicd/nested-stacks/template-cicd-cfn-module.yaml index e5e13f65..21edaefb 100644 --- a/sdlf-cicd/nested-stacks/template-cicd-cfn-module.yaml +++ b/sdlf-cicd/nested-stacks/template-cicd-cfn-module.yaml @@ -5,11 +5,11 @@ Parameters: pArtifactsBucket: Description: The artifacts bucket used by CodeBuild and CodePipeline Type: AWS::SSM::Parameter::Value - Default: /SDLF/S3/DevOpsArtifactsBucket + Default: /SDLF2/S3/DevOpsArtifactsBucket pKMSKey: Description: The KMS key used by CodeBuild and CodePipeline Type: AWS::SSM::Parameter::Value - Default: /SDLF/KMS/CICDKeyId + Default: /SDLF2/KMS/CICDKeyId pStagesRepositoriesPrefix: Description: Prefix of the repositories containing SDLF stages Type: String @@ -17,7 +17,7 @@ Parameters: pEnableVpc: Description: Deploy SDLF resources in a VPC Type: AWS::SSM::Parameter::Value - Default: /SDLF/VPC/Enabled + Default: /SDLF2/VPC/Enabled Conditions: RunInVpc: !Equals [!Ref pEnableVpc, true] @@ -93,7 +93,7 @@ Resources: Condition: ArnEqualsIfExists: "ec2:Vpc": - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}" + - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF2/VPC/VpcId}}" - !Ref "AWS::NoValue" - !If - RunInVpc @@ -107,7 +107,7 @@ Resources: "ec2:AuthorizedService": codebuild.amazonaws.com ArnEquals: "ec2:Vpc": - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}" + - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF2/VPC/VpcId}}" - !Ref "AWS::NoValue" - PolicyName: sdlf-cicd-build-stages-cfn-modules PolicyDocument: @@ -141,7 +141,7 @@ Resources: - RunInVpc - SecurityGroupIds: !Split [",", !ImportValue sdlf-cicd-prerequisites-vpc-security-groups] Subnets: !Split [",", !ImportValue sdlf-cicd-prerequisites-vpc-subnets] - VpcId: "{{resolve:ssm:/SDLF/VPC/VpcId}}" + VpcId: "{{resolve:ssm:/SDLF2/VPC/VpcId}}" - !Ref "AWS::NoValue" Environment: EnvironmentVariables: @@ -202,7 +202,7 @@ Resources: export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN # compare hashes to avoid creating a new module version when there is no change - if CURRENT_MODULE=$(aws ssm --endpoint-url "$SSM_ENDPOINT_URL" get-parameter --name "/SDLF/CFN/$DOMAIN_NAME-$TEAM_NAME-$MODULE_NAME-MODULE" --query "Parameter.Value" --output text); then + if CURRENT_MODULE=$(aws ssm --endpoint-url "$SSM_ENDPOINT_URL" get-parameter --name "/SDLF2/CFN/$DOMAIN_NAME-$TEAM_NAME-$MODULE_NAME-MODULE" --query "Parameter.Value" --output text); then echo "Current module version commit id: $CURRENT_MODULE" echo "New module version commit id: $NEW_MODULE" if [ "$NEW_MODULE" == "$CURRENT_MODULE" ]; then @@ -246,7 +246,7 @@ Resources: - RunInVpc - SecurityGroupIds: !Split [",", !ImportValue sdlf-cicd-prerequisites-vpc-security-groups] Subnets: !Split [",", !ImportValue sdlf-cicd-prerequisites-vpc-subnets] - VpcId: "{{resolve:ssm:/SDLF/VPC/VpcId}}" + VpcId: "{{resolve:ssm:/SDLF2/VPC/VpcId}}" - !Ref "AWS::NoValue" Environment: EnvironmentVariables: @@ -332,7 +332,7 @@ Resources: NEW_MODULE=$(git rev-parse --verify --end-of-options HEAD) MODULE_NAME=${!STAGE_REPOSITORY##*-} # compare hashes to avoid creating a new module version when there is no change - if CURRENT_MODULE=$(aws ssm --endpoint-url "$SSM_ENDPOINT_URL" get-parameter --name "/SDLF/CFN/$DOMAIN_NAME-$TEAM_NAME-$MODULE_NAME-MODULE" --query "Parameter.Value" --output text); then + if CURRENT_MODULE=$(aws ssm --endpoint-url "$SSM_ENDPOINT_URL" get-parameter --name "/SDLF2/CFN/$DOMAIN_NAME-$TEAM_NAME-$MODULE_NAME-MODULE" --query "Parameter.Value" --output text); then echo "Current module version commit id: $CURRENT_MODULE" echo "New module version commit id: $NEW_MODULE" if [ "$NEW_MODULE" == "$CURRENT_MODULE" ]; then @@ -369,7 +369,7 @@ Resources: rBuildCloudformationModuleStageSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/CodeBuild/BuildCloudformationModuleStage + Name: /SDLF2/CodeBuild/BuildCloudformationModuleStage Type: String Value: !Ref rBuildCloudformationModuleStage Description: Name of the CodeBuild job that build a CloudFormation module from a SAM template @@ -377,7 +377,7 @@ Resources: rBuildCloudformationStagesModulesSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/CodeBuild/BuildCloudformationStagesModules + Name: /SDLF2/CodeBuild/BuildCloudformationStagesModules Type: String Value: !Ref rBuildCloudformationStagesModules Description: Name of the CodeBuild job that build CloudFormation modules from a SAM template for SDLF stages diff --git a/sdlf-cicd/nested-stacks/template-cicd-glue-job.yaml b/sdlf-cicd/nested-stacks/template-cicd-glue-job.yaml index 8680304a..ca2e2939 100644 --- a/sdlf-cicd/nested-stacks/template-cicd-glue-job.yaml +++ b/sdlf-cicd/nested-stacks/template-cicd-glue-job.yaml @@ -5,15 +5,15 @@ Parameters: pArtifactsBucket: Description: The artifacts bucket used by CodeBuild and CodePipeline Type: AWS::SSM::Parameter::Value - Default: /SDLF/S3/DevOpsArtifactsBucket + Default: /SDLF2/S3/DevOpsArtifactsBucket pKMSKey: Description: The KMS key used by CodeBuild and CodePipeline Type: AWS::SSM::Parameter::Value - Default: /SDLF/KMS/CICDKeyId + Default: /SDLF2/KMS/CICDKeyId pEnableVpc: Description: Deploy SDLF resources in a VPC Type: AWS::SSM::Parameter::Value - Default: /SDLF/VPC/Enabled + Default: /SDLF2/VPC/Enabled Conditions: RunInVpc: !Equals [!Ref pEnableVpc, true] @@ -86,7 +86,7 @@ Resources: Condition: ArnEqualsIfExists: "ec2:Vpc": - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}" + - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF2/VPC/VpcId}}" - !Ref "AWS::NoValue" - !If - RunInVpc @@ -100,7 +100,7 @@ Resources: "ec2:AuthorizedService": codebuild.amazonaws.com ArnEquals: "ec2:Vpc": - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}" + - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF2/VPC/VpcId}}" - !Ref "AWS::NoValue" rGlueJobPackage: @@ -115,7 +115,7 @@ Resources: - RunInVpc - SecurityGroupIds: !Split [",", !ImportValue sdlf-cicd-prerequisites-vpc-security-groups] Subnets: !Split [",", !ImportValue sdlf-cicd-prerequisites-vpc-subnets] - VpcId: "{{resolve:ssm:/SDLF/VPC/VpcId}}" + VpcId: "{{resolve:ssm:/SDLF2/VPC/VpcId}}" - !Ref "AWS::NoValue" Environment: EnvironmentVariables: @@ -188,7 +188,7 @@ Resources: rGlueJobPackageSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/CodeBuild/PrepareGlueJobPackage + Name: /SDLF2/CodeBuild/PrepareGlueJobPackage Type: String Value: !Ref rGlueJobPackage Description: Name of the CodeBuild job that prepares Glue jobs diff --git a/sdlf-cicd/nested-stacks/template-cicd-lambda-layer.yaml b/sdlf-cicd/nested-stacks/template-cicd-lambda-layer.yaml index 6fc1b989..0dd3f6d4 100644 --- a/sdlf-cicd/nested-stacks/template-cicd-lambda-layer.yaml +++ b/sdlf-cicd/nested-stacks/template-cicd-lambda-layer.yaml @@ -5,15 +5,15 @@ Parameters: pArtifactsBucket: Description: The artifacts bucket used by CodeBuild and CodePipeline Type: AWS::SSM::Parameter::Value - Default: /SDLF/S3/DevOpsArtifactsBucket + Default: /SDLF2/S3/DevOpsArtifactsBucket pKMSKey: Description: The KMS key used by CodeBuild and CodePipeline Type: AWS::SSM::Parameter::Value - Default: /SDLF/KMS/CICDKeyId + Default: /SDLF2/KMS/CICDKeyId pEnableVpc: Description: Deploy SDLF resources in a VPC Type: AWS::SSM::Parameter::Value - Default: /SDLF/VPC/Enabled + Default: /SDLF2/VPC/Enabled Conditions: RunInVpc: !Equals [!Ref pEnableVpc, true] @@ -86,7 +86,7 @@ Resources: Condition: ArnEqualsIfExists: "ec2:Vpc": - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}" + - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF2/VPC/VpcId}}" - !Ref "AWS::NoValue" - !If - RunInVpc @@ -100,7 +100,7 @@ Resources: "ec2:AuthorizedService": codebuild.amazonaws.com ArnEquals: "ec2:Vpc": - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}" + - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF2/VPC/VpcId}}" - !Ref "AWS::NoValue" rBuildLambdaLayersPackage: @@ -115,7 +115,7 @@ Resources: - RunInVpc - SecurityGroupIds: !Split [",", !ImportValue sdlf-cicd-prerequisites-vpc-security-groups] Subnets: !Split [",", !ImportValue sdlf-cicd-prerequisites-vpc-subnets] - VpcId: "{{resolve:ssm:/SDLF/VPC/VpcId}}" + VpcId: "{{resolve:ssm:/SDLF2/VPC/VpcId}}" - !Ref "AWS::NoValue" Environment: EnvironmentVariables: @@ -229,7 +229,7 @@ Resources: rBuildLambdaLayersPackageSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/CodeBuild/BuildLambdaLayersPackage + Name: /SDLF2/CodeBuild/BuildLambdaLayersPackage Type: String Value: !Ref rBuildLambdaLayersPackage Description: Name of the CodeBuild job that creates a Lambda Layer containing the libraries and version numbers listed in the requirements.txt file in the repository provided diff --git a/sdlf-cicd/nested-stacks/template-cicd-modules-pipelines.yaml b/sdlf-cicd/nested-stacks/template-cicd-modules-pipelines.yaml index b437117e..06790820 100644 --- a/sdlf-cicd/nested-stacks/template-cicd-modules-pipelines.yaml +++ b/sdlf-cicd/nested-stacks/template-cicd-modules-pipelines.yaml @@ -5,11 +5,11 @@ Parameters: pArtifactsBucket: Description: The artifacts bucket used by CodeBuild and CodePipeline Type: AWS::SSM::Parameter::Value - Default: /SDLF/S3/DevOpsArtifactsBucket + Default: /SDLF2/S3/DevOpsArtifactsBucket pKMSKey: Description: The KMS key used by CodeBuild and CodePipeline Type: AWS::SSM::Parameter::Value - Default: /SDLF/KMS/CICDKeyId + Default: /SDLF2/KMS/CICDKeyId pCicdRepository: Type: String pMainRepository: @@ -255,7 +255,7 @@ Resources: rMainRepositoryPipelineSsm: Type: AWS::SSM::Parameter Properties: - Name: !Sub /SDLF/CodePipeline/MainRepository${pEnvironment}Pipeline + Name: !Sub /SDLF2/CodePipeline/MainRepository${pEnvironment}Pipeline Type: String Value: !Ref rMainRepositoryPipeline Description: !Sub CodePipeline that triggers on sdlf-main ${pEnvironment} changes diff --git a/sdlf-cicd/template-cfn-module.yaml b/sdlf-cicd/template-cfn-module.yaml index 6b693343..44a5a925 100644 --- a/sdlf-cicd/template-cfn-module.yaml +++ b/sdlf-cicd/template-cfn-module.yaml @@ -38,7 +38,7 @@ Resources: Type: AWS::SSM::Parameter DependsOn: rCloudFormationModuleDefaultVersion Properties: - Name: !Sub /SDLF/CFN/${pDomain}-${pTeamName}-${pModuleName}-MODULE + Name: !Sub /SDLF2/CFN/${pDomain}-${pTeamName}-${pModuleName}-MODULE Type: String Value: !Ref pModuleGitRef Description: Git reference (commit id) with the sources of this module version diff --git a/sdlf-cicd/template-cicd-domain-roles.yaml b/sdlf-cicd/template-cicd-domain-roles.yaml index eeeab33a..911f5a20 100644 --- a/sdlf-cicd/template-cicd-domain-roles.yaml +++ b/sdlf-cicd/template-cicd-domain-roles.yaml @@ -38,7 +38,7 @@ Resources: Type: AWS::SSM::Parameter Condition: MultiAccountSetup Properties: - Name: /SDLF/VPC/Enabled + Name: /SDLF2/VPC/Enabled Type: String Value: !Ref pEnableVpc Description: Deploy SDLF resources in a VPC @@ -93,7 +93,7 @@ Resources: Action: - ssm:GetParameter - ssm:GetParameters - Resource: !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/CFN/* + Resource: !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/CFN/* - Effect: Allow Action: - cloudformation:CreateChangeSet @@ -205,7 +205,7 @@ Resources: - ssm:RemoveTagsFromResource - ssm:DeleteParameter Resource: - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/CFN/* + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/CFN/* - PolicyName: sdlf-lambdalayers-codepipeline PolicyDocument: Version: "2012-10-17" @@ -232,7 +232,7 @@ Resources: - ssm:RemoveTagsFromResource - ssm:DeleteParameter Resource: - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/Lambda/LatestDatalakeLibraryLayer + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/Lambda/LatestDatalakeLibraryLayer rDomainCloudFormationRole: Type: AWS::IAM::Role @@ -396,7 +396,7 @@ Resources: - RunInVpc - StringEquals: "lambda:VpcIds": - - "{{resolve:ssm:/SDLF/VPC/VpcId}}" + - "{{resolve:ssm:/SDLF2/VPC/VpcId}}" - !Ref "AWS::NoValue" - Effect: Allow Action: @@ -559,7 +559,7 @@ Resources: - ssm:ListTagsForResource - ssm:PutParameter - ssm:RemoveTagsFromResource - Resource: !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/* + Resource: !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/* - Effect: Allow Action: - lakeformation:DeregisterResource # W11 exception @@ -706,8 +706,8 @@ Resources: - ssm:GetParameters - ssm:GetParametersByPath Resource: - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/Misc/* - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/VPC/* + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/Misc/* + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/VPC/* - Effect: Allow Action: - iam:GetRole @@ -750,7 +750,7 @@ Outputs: Description: List of security group ids that will be attached to Lambda functions and CodeBuild projects Value: !If - RunInVpc - - "{{resolve:ssm:/SDLF/VPC/SecurityGroupIds}}" + - "{{resolve:ssm:/SDLF2/VPC/SecurityGroupIds}}" - "-" Export: Name: !Join ["-", [!Ref "AWS::StackName", "vpc-security-groups"]] @@ -759,7 +759,7 @@ Outputs: Description: List of subnet ids that will be attached to Lambda functions and CodeBuild projects Value: !If - RunInVpc - - "{{resolve:ssm:/SDLF/VPC/SubnetIds}}" + - "{{resolve:ssm:/SDLF2/VPC/SubnetIds}}" - "-" Export: Name: !Join ["-", [!Ref "AWS::StackName", "vpc-subnets"]] diff --git a/sdlf-cicd/template-cicd-domain-team-role.yaml b/sdlf-cicd/template-cicd-domain-team-role.yaml index 51665a4c..f440fde7 100644 --- a/sdlf-cicd/template-cicd-domain-team-role.yaml +++ b/sdlf-cicd/template-cicd-domain-team-role.yaml @@ -27,7 +27,7 @@ Parameters: pEnableVpc: Description: Deploy SDLF resources in a VPC Type: AWS::SSM::Parameter::Value - Default: /SDLF/VPC/Enabled + Default: /SDLF2/VPC/Enabled Conditions: EnableLambdaLayerBuilder: !Equals [!Ref pEnableLambdaLayerBuilder, true] @@ -148,12 +148,12 @@ Resources: - ssm:DeleteParameters - ssm:PutParameter Resource: - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/IAM/${pTeamName}/TeamPermissionsBoundary - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/KMS/${pTeamName}/InfraKeyId - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/KMS/${pTeamName}/DataKeyId - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/SNS/${pTeamName}/Notifications - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/EventBridge/${pTeamName}/ScheduleGroupName - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/IAM/${pTeamName}/CrawlerRoleArn + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/IAM/${pTeamName}/TeamPermissionsBoundary + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/KMS/${pTeamName}/InfraKeyId + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/KMS/${pTeamName}/DataKeyId + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/SNS/${pTeamName}/Notifications + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/EventBridge/${pTeamName}/ScheduleGroupName + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/IAM/${pTeamName}/CrawlerRoleArn - Sid: ReadOnlyAccessSsm Effect: Allow Action: @@ -164,9 +164,9 @@ Resources: - ssm:GetParametersByPath - ssm:ListTagsForResource Resource: - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/Misc/* - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/S3/* - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/Lambda/LatestDatalakeLibraryLayer + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/Misc/* + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/S3/* + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/Lambda/LatestDatalakeLibraryLayer - Effect: Allow Action: - ssm:AddTagsToResource @@ -181,16 +181,16 @@ Resources: - ssm:DeleteParameters - ssm:PutParameter Resource: - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/Pipelines/${pTeamName}/* - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/Datasets/${pTeamName}/* - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/Lambda/${pTeamName}/* - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/SQS/${pTeamName}/* - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/SNS/${pTeamName}/* - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/KMS/${pTeamName}/* - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/EventBridge/${pTeamName}/* - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/SM/${pTeamName}/* - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/Glue/${pTeamName}/* - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/IAM/${pTeamName}/* + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/Pipelines/${pTeamName}/* + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/Datasets/${pTeamName}/* + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/Lambda/${pTeamName}/* + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/SQS/${pTeamName}/* + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/SNS/${pTeamName}/* + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/KMS/${pTeamName}/* + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/EventBridge/${pTeamName}/* + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/SM/${pTeamName}/* + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/Glue/${pTeamName}/* + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/IAM/${pTeamName}/* - Effect: Allow Action: - kms:CreateGrant @@ -292,7 +292,7 @@ Resources: - RunInVpc - StringEquals: "lambda:VpcIds": - - "{{resolve:ssm:/SDLF/VPC/VpcId}}" + - "{{resolve:ssm:/SDLF2/VPC/VpcId}}" - !Ref "AWS::NoValue" - Effect: Allow Action: @@ -494,8 +494,8 @@ Resources: - ssm:GetParameters - ssm:GetParametersByPath Resource: - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/KMS/KeyArn - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/VPC/* + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/KMS/KeyArn + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/VPC/* - Effect: Allow Action: - iam:AttachRolePolicy diff --git a/sdlf-cicd/template-cicd-domain.yaml b/sdlf-cicd/template-cicd-domain.yaml index 198fcd89..9fc140ec 100644 --- a/sdlf-cicd/template-cicd-domain.yaml +++ b/sdlf-cicd/template-cicd-domain.yaml @@ -5,14 +5,14 @@ Parameters: pArtifactsBucket: Description: The artifacts bucket used by CodeBuild and CodePipeline Type: AWS::SSM::Parameter::Value - Default: /SDLF/S3/DevOpsArtifactsBucket + Default: /SDLF2/S3/DevOpsArtifactsBucket pKMSKey: Description: The KMS key used by CodeBuild and CodePipeline Type: AWS::SSM::Parameter::Value - Default: /SDLF/KMS/CICDKeyId + Default: /SDLF2/KMS/CICDKeyId pMainRepository: Type: AWS::SSM::Parameter::Value - Default: /SDLF/CodeCommit/MainCodeCommit + Default: /SDLF2/CodeCommit/MainCodeCommit pChildAccountId: Description: Child AWS account ID Type: String @@ -29,24 +29,24 @@ Parameters: pBuildCloudformationModuleStage: Description: CodeBuild job that build a CloudFormation module from a SAM template Type: AWS::SSM::Parameter::Value - Default: /SDLF/CodeBuild/BuildCloudformationModuleStage + Default: /SDLF2/CodeBuild/BuildCloudformationModuleStage pBuildCloudFormationPackage: Description: CodeBuild job that packages a CloudFormation template Type: AWS::SSM::Parameter::Value - Default: /SDLF/CodeBuild/BuildCloudFormationPackage + Default: /SDLF2/CodeBuild/BuildCloudFormationPackage pEnableMonitoring: Description: Build sdlf-monitoring cloudformation module as part of domain pipelines Type: AWS::SSM::Parameter::Value - Default: /SDLF/Monitoring/Enabled + Default: /SDLF2/Monitoring/Enabled pCicdRepository: Type: AWS::SSM::Parameter::Value - Default: /SDLF/CodeCommit/CicdCodeCommit + Default: /SDLF2/CodeCommit/CicdCodeCommit pFoundationsRepository: Type: AWS::SSM::Parameter::Value - Default: /SDLF/CodeCommit/FoundationsCodeCommit + Default: /SDLF2/CodeCommit/FoundationsCodeCommit pTeamRepository: Type: AWS::SSM::Parameter::Value - Default: /SDLF/CodeCommit/TeamCodeCommit + Default: /SDLF2/CodeCommit/TeamCodeCommit pSdlfModuleDomain: Type: String Default: awslabs @@ -103,7 +103,7 @@ Resources: - !Sub arn:${AWS::Partition}:codecommit:${AWS::Region}:${AWS::AccountId}:${pMainRepository} - !If - EnableMonitoring - - !Sub "arn:${AWS::Partition}:codecommit:${AWS::Region}:${AWS::AccountId}:{{resolve:ssm:/SDLF/CodeCommit/MonitoringCodeCommit}}" + - !Sub "arn:${AWS::Partition}:codecommit:${AWS::Region}:${AWS::AccountId}:{{resolve:ssm:/SDLF2/CodeCommit/MonitoringCodeCommit}}" - !Ref AWS::NoValue - Effect: Allow Action: @@ -212,7 +212,7 @@ Resources: OutputArtifacts: - Name: SourceMonitoringArtifact Configuration: - RepositoryName: "{{resolve:ssm:/SDLF/CodeCommit/MonitoringCodeCommit}}" + RepositoryName: "{{resolve:ssm:/SDLF2/CodeCommit/MonitoringCodeCommit}}" BranchName: !FindInMap [pCodeCommitBranch, !Ref pEnvironment, branch] PollForSourceChanges: false RunOrder: 1 @@ -358,7 +358,7 @@ Resources: detail-type: - CodePipeline Pipeline Execution State Change resources: - - !Sub "arn:${AWS::Partition}:codepipeline:${AWS::Region}:${AWS::AccountId}:{{resolve:ssm:/SDLF/CodePipeline/MainRepository${pEnvironment}Pipeline}}" + - !Sub "arn:${AWS::Partition}:codepipeline:${AWS::Region}:${AWS::AccountId}:{{resolve:ssm:/SDLF2/CodePipeline/MainRepository${pEnvironment}Pipeline}}" detail: state: - SUCCEEDED @@ -370,7 +370,7 @@ Resources: rChildAccountIdSsm: Type: AWS::SSM::Parameter Properties: - Name: !Sub /SDLF/Misc/Domains/${pDomain}/${pEnvironment}/AccountId + Name: !Sub /SDLF2/Misc/Domains/${pDomain}/${pEnvironment}/AccountId Type: String Value: !Ref pChildAccountId Description: !Sub ${pDomain} ${pEnvironment} environment Child Account Id diff --git a/sdlf-cicd/template-cicd-prerequisites.yaml b/sdlf-cicd/template-cicd-prerequisites.yaml index 0bd1c078..f0b8d35e 100644 --- a/sdlf-cicd/template-cicd-prerequisites.yaml +++ b/sdlf-cicd/template-cicd-prerequisites.yaml @@ -23,12 +23,12 @@ Conditions: Resources: ######## OPTIONAL SDLF FEATURES ######### - # when enabling VPC support, /SDLF/VPC/VpcId, /SDLF/VPC/SecurityGroupIds and /SDLF/VPC/SubnetIds are required too (see Outputs section) + # when enabling VPC support, /SDLF2/VPC/VpcId, /SDLF2/VPC/SecurityGroupIds and /SDLF2/VPC/SubnetIds are required too (see Outputs section) # both for the devops account and child accounts. rVpcFeatureSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/VPC/Enabled + Name: /SDLF2/VPC/Enabled Type: String Value: !Ref pEnableVpc Description: Deploy SDLF resources in a VPC @@ -36,7 +36,7 @@ Resources: rBuildLambdaLayersFeatureSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/LambdaLayerBuilder/Enabled + Name: /SDLF2/LambdaLayerBuilder/Enabled Type: String Value: false Description: Add Lambda layer builder infrastructure and pipeline stages @@ -44,7 +44,7 @@ Resources: rGlueJobDeployerFeatureSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/GlueJobDeployer/Enabled + Name: /SDLF2/GlueJobDeployer/Enabled Type: String Value: false Description: Add Glue job deployer infrastructure and pipeline stages @@ -52,7 +52,7 @@ Resources: rMonitoringFeatureSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/Monitoring/Enabled + Name: /SDLF2/Monitoring/Enabled Type: String Value: false Description: Build sdlf-monitoring cloudformation module as part of domain pipelines @@ -207,7 +207,7 @@ Resources: rKMSKeySsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/KMS/CICDKeyId + Name: /SDLF2/KMS/CICDKeyId Type: String Value: !GetAtt rKMSKey.Arn Description: CICD KMS key @@ -483,7 +483,7 @@ Resources: rArtifactsBucketSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/S3/DevOpsArtifactsBucket + Name: /SDLF2/S3/DevOpsArtifactsBucket Type: String Value: !Ref rArtifactsBucket Description: S3 DevOps Artifacts Bucket @@ -622,7 +622,7 @@ Outputs: Description: List of security group ids that will be attached to Lambda functions and CodeBuild projects Value: !If - RunInVpc - - "{{resolve:ssm:/SDLF/VPC/SecurityGroupIds}}" + - "{{resolve:ssm:/SDLF2/VPC/SecurityGroupIds}}" - "-" Export: Name: !Join ["-", [!Ref "AWS::StackName", "vpc-security-groups"]] @@ -631,7 +631,7 @@ Outputs: Description: List of subnet ids that will be attached to Lambda functions and CodeBuild projects Value: !If - RunInVpc - - "{{resolve:ssm:/SDLF/VPC/SubnetIds}}" + - "{{resolve:ssm:/SDLF2/VPC/SubnetIds}}" - "-" Export: Name: !Join ["-", [!Ref "AWS::StackName", "vpc-subnets"]] diff --git a/sdlf-cicd/template-cicd-sdlf-pipelines.yaml b/sdlf-cicd/template-cicd-sdlf-pipelines.yaml index 7ea5adbf..c8a08ec6 100644 --- a/sdlf-cicd/template-cicd-sdlf-pipelines.yaml +++ b/sdlf-cicd/template-cicd-sdlf-pipelines.yaml @@ -5,17 +5,17 @@ Parameters: pArtifactsBucket: Description: The artifacts bucket used by CodeBuild and CodePipeline Type: AWS::SSM::Parameter::Value - Default: /SDLF/S3/DevOpsArtifactsBucket + Default: /SDLF2/S3/DevOpsArtifactsBucket pKMSKey: Description: The KMS key used by CodeBuild and CodePipeline Type: AWS::SSM::Parameter::Value - Default: /SDLF/KMS/CICDKeyId + Default: /SDLF2/KMS/CICDKeyId pCicdRepository: Type: AWS::SSM::Parameter::Value - Default: /SDLF/CodeCommit/CicdCodeCommit + Default: /SDLF2/CodeCommit/CicdCodeCommit pMainRepository: Type: AWS::SSM::Parameter::Value - Default: /SDLF/CodeCommit/MainCodeCommit + Default: /SDLF2/CodeCommit/MainCodeCommit pMainRepositoriesPrefix: Type: String Default: sdlf2-main- @@ -30,15 +30,15 @@ Parameters: pEnableVpc: Description: Deploy SDLF resources in a VPC Type: AWS::SSM::Parameter::Value - Default: /SDLF/VPC/Enabled + Default: /SDLF2/VPC/Enabled pEnableLambdaLayerBuilder: Description: Add Lambda layer builder infrastructure and pipeline stages Type: AWS::SSM::Parameter::Value - Default: /SDLF/LambdaLayerBuilder/Enabled + Default: /SDLF2/LambdaLayerBuilder/Enabled pEnableGlueJobDeployer: Description: Add Glue job deployer infrastructure and pipeline stages Type: AWS::SSM::Parameter::Value - Default: /SDLF/GlueJobDeployer/Enabled + Default: /SDLF2/GlueJobDeployer/Enabled Conditions: EnableLambdaLayerBuilder: !Equals [!Ref pEnableLambdaLayerBuilder, true] @@ -185,9 +185,9 @@ Resources: Action: - ssm:GetParameters Resource: - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/S3/DevOpsArtifactsBucket - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/KMS/CICDKeyId - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/CodeBuild/BuildCloudformationModuleStage + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/S3/DevOpsArtifactsBucket + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/KMS/CICDKeyId + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/CodeBuild/BuildCloudformationModuleStage - Effect: Allow Action: - iam:GetRole @@ -330,7 +330,7 @@ Resources: - ssm:GetParameters - ssm:GetParametersByPath Resource: - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/Misc/* + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/Misc/* - Effect: Allow Action: - s3:PutObject @@ -488,15 +488,15 @@ Resources: - ssm:AddTagsToResource - ssm:DeleteParameter Resource: - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/Misc/* - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/S3/DevOpsArtifactsBucket - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/KMS/CICDKeyId - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/CodePipeline/* - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/CodeBuild/* - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/LambdaLayerBuilder/Enabled - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/GlueJobDeployer/Enabled - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/Monitoring/Enabled - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/CodeCommit/* + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/Misc/* + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/S3/DevOpsArtifactsBucket + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/KMS/CICDKeyId + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/CodePipeline/* + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/CodeBuild/* + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/LambdaLayerBuilder/Enabled + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/GlueJobDeployer/Enabled + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/Monitoring/Enabled + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/CodeCommit/* - Effect: Allow Action: - codepipeline:GetPipelineState @@ -643,7 +643,7 @@ Resources: Condition: ArnEqualsIfExists: "ec2:Vpc": - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}" + - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF2/VPC/VpcId}}" - !Ref "AWS::NoValue" - !If - RunInVpc @@ -657,7 +657,7 @@ Resources: "ec2:AuthorizedService": codebuild.amazonaws.com ArnEquals: "ec2:Vpc": - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}" + - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF2/VPC/VpcId}}" - !Ref "AWS::NoValue" rCloudFormationPackageCodeBuildProject: @@ -671,7 +671,7 @@ Resources: - RunInVpc - SecurityGroupIds: !Split [",", !ImportValue sdlf-cicd-prerequisites-vpc-security-groups] Subnets: !Split [",", !ImportValue sdlf-cicd-prerequisites-vpc-subnets] - VpcId: "{{resolve:ssm:/SDLF/VPC/VpcId}}" + VpcId: "{{resolve:ssm:/SDLF2/VPC/VpcId}}" - !Ref "AWS::NoValue" Environment: ComputeType: BUILD_GENERAL1_SMALL @@ -771,7 +771,7 @@ Resources: Condition: ArnEqualsIfExists: "ec2:Vpc": - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}" + - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF2/VPC/VpcId}}" - !Ref "AWS::NoValue" - !If - RunInVpc @@ -785,7 +785,7 @@ Resources: "ec2:AuthorizedService": codebuild.amazonaws.com ArnEquals: "ec2:Vpc": - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}" + - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF2/VPC/VpcId}}" - !Ref "AWS::NoValue" @@ -801,7 +801,7 @@ Resources: - RunInVpc - SecurityGroupIds: !Split [",", !ImportValue sdlf-cicd-prerequisites-vpc-security-groups] Subnets: !Split [",", !ImportValue sdlf-cicd-prerequisites-vpc-subnets] - VpcId: "{{resolve:ssm:/SDLF/VPC/VpcId}}" + VpcId: "{{resolve:ssm:/SDLF2/VPC/VpcId}}" - !Ref "AWS::NoValue" Environment: EnvironmentVariables: @@ -862,14 +862,14 @@ Resources: rBuildDeployDatalakeLibraryLayerSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/CodeBuild/BuildDeployDatalakeLibraryLayer + Name: /SDLF2/CodeBuild/BuildDeployDatalakeLibraryLayer Type: String Value: !Ref rBuildDeployDatalakeLibraryLayer Description: Name of the CodeBuild job that packages the Datalake Libs into a Lambda Layer rBuildCloudFormationPackageSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/CodeBuild/BuildCloudFormationPackage + Name: /SDLF2/CodeBuild/BuildCloudFormationPackage Type: String Value: !Ref rCloudFormationPackageCodeBuildProject Description: CodeBuild job that packages a CloudFormation template diff --git a/sdlf-cicd/template-cicd-sdlf-repositories.yaml b/sdlf-cicd/template-cicd-sdlf-repositories.yaml index 93e99fd2..8fe1fd9d 100644 --- a/sdlf-cicd/template-cicd-sdlf-repositories.yaml +++ b/sdlf-cicd/template-cicd-sdlf-repositories.yaml @@ -5,7 +5,7 @@ Parameters: pKMSKey: Description: The KMS key used by CodeBuild and CodePipeline Type: AWS::SSM::Parameter::Value - Default: /SDLF/KMS/CICDKeyId + Default: /SDLF2/KMS/CICDKeyId pCicdRepository: Type: String Default: sdlf2-cicd @@ -42,7 +42,7 @@ Parameters: pEnableMonitoring: Description: Build sdlf-monitoring cloudformation module as part of domain pipelines Type: AWS::SSM::Parameter::Value - Default: /SDLF/Monitoring/Enabled + Default: /SDLF2/Monitoring/Enabled Conditions: EnableMonitoring: !Equals [!Ref pEnableMonitoring, true] @@ -203,7 +203,7 @@ Resources: rCicdCodeCommitSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/CodeCommit/CicdCodeCommit + Name: /SDLF2/CodeCommit/CicdCodeCommit Type: String Value: !GetAtt rCicdCodeCommit.Name Description: Name of the Cicd repository @@ -211,7 +211,7 @@ Resources: rFoundationsCodeCommitSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/CodeCommit/FoundationsCodeCommit + Name: /SDLF2/CodeCommit/FoundationsCodeCommit Type: String Value: !GetAtt rFoundationsCodeCommit.Name Description: Name of the Foundations repository @@ -219,7 +219,7 @@ Resources: rTeamCodeCommitSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/CodeCommit/TeamCodeCommit + Name: /SDLF2/CodeCommit/TeamCodeCommit Type: String Value: !GetAtt rTeamCodeCommit.Name Description: Name of the Team repository @@ -227,7 +227,7 @@ Resources: rPipelineCodeCommitSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/CodeCommit/PipelineCodeCommit + Name: /SDLF2/CodeCommit/PipelineCodeCommit Type: String Value: !GetAtt rPipelineCodeCommit.Name Description: Name of the Pipeline repository @@ -235,7 +235,7 @@ Resources: rDatasetCodeCommitSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/CodeCommit/DatasetCodeCommit + Name: /SDLF2/CodeCommit/DatasetCodeCommit Type: String Value: !GetAtt rDatasetCodeCommit.Name Description: Name of the Dataset repository @@ -243,7 +243,7 @@ Resources: rStageACodeCommitSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/CodeCommit/StageACodeCommit + Name: /SDLF2/CodeCommit/StageACodeCommit Type: String Value: !GetAtt rStageACodeCommit.Name Description: Name of the StageA repository @@ -251,7 +251,7 @@ Resources: rStageBCodeCommitSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/CodeCommit/StageBCodeCommit + Name: /SDLF2/CodeCommit/StageBCodeCommit Type: String Value: !GetAtt rStageBCodeCommit.Name Description: Name of the StageB repository @@ -259,7 +259,7 @@ Resources: rDatalakeLibraryCodeCommitSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/CodeCommit/DatalakeLibraryCodeCommit + Name: /SDLF2/CodeCommit/DatalakeLibraryCodeCommit Type: String Value: !GetAtt rDatalakeLibraryCodeCommit.Name Description: Name of the DatalakeLibrary repository @@ -267,7 +267,7 @@ Resources: rUtilsCodeCommitSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/CodeCommit/UtilsCodeCommit + Name: /SDLF2/CodeCommit/UtilsCodeCommit Type: String Value: !Ref pUtilsRepository Description: Name of the Utils repository @@ -275,7 +275,7 @@ Resources: rMainCodeCommitSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/CodeCommit/MainCodeCommit + Name: /SDLF2/CodeCommit/MainCodeCommit Type: String Value: !GetAtt rMainCodeCommit.Name Description: Name of the main repository @@ -284,7 +284,7 @@ Resources: Type: AWS::SSM::Parameter Condition: EnableMonitoring Properties: - Name: /SDLF/CodeCommit/MonitoringCodeCommit + Name: /SDLF2/CodeCommit/MonitoringCodeCommit Type: String Value: !GetAtt rMonitoringCodeCommit.Name Description: Name of the monitoring repository diff --git a/sdlf-cicd/template-cicd-team-pipeline.yaml b/sdlf-cicd/template-cicd-team-pipeline.yaml index dd74caf0..c193a77c 100644 --- a/sdlf-cicd/template-cicd-team-pipeline.yaml +++ b/sdlf-cicd/template-cicd-team-pipeline.yaml @@ -5,11 +5,11 @@ Parameters: pArtifactsBucket: Description: The artifacts bucket used by CodeBuild and CodePipeline Type: AWS::SSM::Parameter::Value - Default: /SDLF/S3/DevOpsArtifactsBucket + Default: /SDLF2/S3/DevOpsArtifactsBucket pKMSKey: Description: The KMS key used by CodeBuild and CodePipeline Type: AWS::SSM::Parameter::Value - Default: /SDLF/KMS/CICDKeyId + Default: /SDLF2/KMS/CICDKeyId pDomain: Description: Name of the data domain (all lowercase, no symbols or spaces) Type: String @@ -33,46 +33,46 @@ Parameters: pBuildDeployDatalakeLibraryLayer: Description: Name of the CodeBuild job that packages the Datalake Libs into a Lambda Layer Type: AWS::SSM::Parameter::Value - Default: /SDLF/CodeBuild/BuildDeployDatalakeLibraryLayer + Default: /SDLF2/CodeBuild/BuildDeployDatalakeLibraryLayer pBuildCloudformationModuleStage: Description: CodeBuild job that build a CloudFormation module from a SAM template Type: AWS::SSM::Parameter::Value - Default: /SDLF/CodeBuild/BuildCloudformationModuleStage + Default: /SDLF2/CodeBuild/BuildCloudformationModuleStage pBuildCloudformationStagesModules: Description: CodeBuild job that build CloudFormation modules from a SAM template for SDLF stages Type: AWS::SSM::Parameter::Value - Default: /SDLF/CodeBuild/BuildCloudformationStagesModules + Default: /SDLF2/CodeBuild/BuildCloudformationStagesModules pBuildCloudFormationPackage: Description: CodeBuild job that packages a CloudFormation template Type: AWS::SSM::Parameter::Value - Default: /SDLF/CodeBuild/BuildCloudFormationPackage + Default: /SDLF2/CodeBuild/BuildCloudFormationPackage pEnableLambdaLayerBuilder: Description: Add Lambda layer builder infrastructure and pipeline stages Type: AWS::SSM::Parameter::Value - Default: /SDLF/LambdaLayerBuilder/Enabled + Default: /SDLF2/LambdaLayerBuilder/Enabled pEnableGlueJobDeployer: Description: Add Glue job deployer infrastructure and pipeline stages Type: AWS::SSM::Parameter::Value - Default: /SDLF/GlueJobDeployer/Enabled + Default: /SDLF2/GlueJobDeployer/Enabled pCicdRepository: Description: Name of the Cicd repository Type: AWS::SSM::Parameter::Value - Default: /SDLF/CodeCommit/CicdCodeCommit + Default: /SDLF2/CodeCommit/CicdCodeCommit pDatalakeLibraryRepository: Type: AWS::SSM::Parameter::Value - Default: /SDLF/CodeCommit/DatalakeLibraryCodeCommit + Default: /SDLF2/CodeCommit/DatalakeLibraryCodeCommit pPipelineRepository: Type: AWS::SSM::Parameter::Value - Default: /SDLF/CodeCommit/PipelineCodeCommit + Default: /SDLF2/CodeCommit/PipelineCodeCommit pStageARepository: Type: AWS::SSM::Parameter::Value - Default: /SDLF/CodeCommit/StageACodeCommit + Default: /SDLF2/CodeCommit/StageACodeCommit pStageBRepository: Type: AWS::SSM::Parameter::Value - Default: /SDLF/CodeCommit/StageBCodeCommit + Default: /SDLF2/CodeCommit/StageBCodeCommit pDatasetRepository: Type: AWS::SSM::Parameter::Value - Default: /SDLF/CodeCommit/DatasetCodeCommit + Default: /SDLF2/CodeCommit/DatasetCodeCommit pSdlfModuleDomain: Type: String Default: awslabs @@ -126,7 +126,7 @@ Resources: - codecommit:GetUploadArchiveStatus - codecommit:CancelUploadArchive Resource: - - !Sub "arn:${AWS::Partition}:codecommit:${AWS::Region}:${AWS::AccountId}:{{resolve:ssm:/SDLF/CodeCommit/${pTeamName}/MainCodeCommit}}" + - !Sub "arn:${AWS::Partition}:codecommit:${AWS::Region}:${AWS::AccountId}:{{resolve:ssm:/SDLF2/CodeCommit/${pTeamName}/MainCodeCommit}}" - !Sub arn:${AWS::Partition}:codecommit:${AWS::Region}:${AWS::AccountId}:${pCicdRepository} - !Sub arn:${AWS::Partition}:codecommit:${AWS::Region}:${AWS::AccountId}:${pDatalakeLibraryRepository} - !Sub arn:${AWS::Partition}:codecommit:${AWS::Region}:${AWS::AccountId}:${pPipelineRepository} @@ -160,11 +160,11 @@ Resources: - !Sub "arn:${AWS::Partition}:codebuild:${AWS::Region}:${AWS::AccountId}:project/${pBuildDeployDatalakeLibraryLayer}" - !If - EnableLambdaLayerBuilder - - !Sub "arn:${AWS::Partition}:codebuild:${AWS::Region}:${AWS::AccountId}:project/{{resolve:ssm:/SDLF/CodeBuild/BuildLambdaLayersPackage}}" + - !Sub "arn:${AWS::Partition}:codebuild:${AWS::Region}:${AWS::AccountId}:project/{{resolve:ssm:/SDLF2/CodeBuild/BuildLambdaLayersPackage}}" - !Ref AWS::NoValue - !If - EnableGlueJobDeployer - - !Sub "arn:${AWS::Partition}:codebuild:${AWS::Region}:${AWS::AccountId}:project/{{resolve:ssm:/SDLF/CodeBuild/PrepareGlueJobPackage}}" + - !Sub "arn:${AWS::Partition}:codebuild:${AWS::Region}:${AWS::AccountId}:project/{{resolve:ssm:/SDLF2/CodeBuild/PrepareGlueJobPackage}}" - !Ref AWS::NoValue - PolicyName: sdlf-cicd-cloudformation PolicyDocument: @@ -191,7 +191,7 @@ Resources: OutputArtifacts: - Name: TemplateSource Configuration: - RepositoryName: !Sub "{{resolve:ssm:/SDLF/CodeCommit/${pTeamName}/MainCodeCommit}}" + RepositoryName: !Sub "{{resolve:ssm:/SDLF2/CodeCommit/${pTeamName}/MainCodeCommit}}" BranchName: !FindInMap [pCodeCommitBranch, !Ref pEnvironment, branch] PollForSourceChanges: false @@ -296,7 +296,7 @@ Resources: Version: "1" Provider: CodeBuild Configuration: - ProjectName: "{{resolve:ssm:/SDLF/CodeBuild/BuildLambdaLayersPackage}}" + ProjectName: "{{resolve:ssm:/SDLF2/CodeBuild/BuildLambdaLayersPackage}}" EnvironmentVariables: !Sub >- [{"name":"ENVIRONMENT", "value":"${pEnvironment}", "type":"PLAINTEXT"}, {"name":"DOMAIN", "value":"${pDomain}", "type":"PLAINTEXT"}, @@ -315,7 +315,7 @@ Resources: Version: "1" Provider: CodeBuild Configuration: - ProjectName: "{{resolve:ssm:/SDLF/CodeBuild/PrepareGlueJobPackage}}" + ProjectName: "{{resolve:ssm:/SDLF2/CodeBuild/PrepareGlueJobPackage}}" EnvironmentVariables: !Sub >- [{"name":"ENVIRONMENT", "value":"${pEnvironment}", "type":"PLAINTEXT"}, {"name":"DOMAIN", "value":"${pDomain}", "type":"PLAINTEXT"}, @@ -611,7 +611,7 @@ Resources: detail-type: - CodeCommit Repository State Change resources: - - !Sub "arn:${AWS::Partition}:codecommit:${AWS::Region}:${AWS::AccountId}:{{resolve:ssm:/SDLF/CodeCommit/${pTeamName}/MainCodeCommit}}" + - !Sub "arn:${AWS::Partition}:codecommit:${AWS::Region}:${AWS::AccountId}:{{resolve:ssm:/SDLF2/CodeCommit/${pTeamName}/MainCodeCommit}}" detail: event: - referenceCreated diff --git a/sdlf-cicd/template-cicd-team-repository.yaml b/sdlf-cicd/template-cicd-team-repository.yaml index 9bf1577f..301258a7 100644 --- a/sdlf-cicd/template-cicd-team-repository.yaml +++ b/sdlf-cicd/template-cicd-team-repository.yaml @@ -5,7 +5,7 @@ Parameters: pKMSKey: Description: The KMS key used by CodeBuild and CodePipeline Type: AWS::SSM::Parameter::Value - Default: /SDLF/KMS/CICDKeyId + Default: /SDLF2/KMS/CICDKeyId pDomain: Description: Name of the data domain (all lowercase, no symbols or spaces) Type: String @@ -36,7 +36,7 @@ Resources: rTeamMainCodeCommitSsm: Type: AWS::SSM::Parameter Properties: - Name: !Sub /SDLF/CodeCommit/${pTeamName}/MainCodeCommit + Name: !Sub /SDLF2/CodeCommit/${pTeamName}/MainCodeCommit Type: String Value: !GetAtt rTeamMainCodeCommit.Name Description: !Sub Name of the ${pDomain} ${pTeamName} main repository diff --git a/sdlf-cicd/template-glue-job.yaml b/sdlf-cicd/template-glue-job.yaml index b7661a4a..5459b8d0 100644 --- a/sdlf-cicd/template-glue-job.yaml +++ b/sdlf-cicd/template-glue-job.yaml @@ -9,7 +9,7 @@ Parameters: pDomain: Description: Name of the data domain (all lowercase, no symbols or spaces) Type: AWS::SSM::Parameter::Value - Default: /SDLF/Misc/pDomain + Default: /SDLF2/Misc/pDomain pEnvironment: Description: Environment name Type: String @@ -27,7 +27,7 @@ Parameters: pEnableVpc: Description: Deploy SDLF resources in a VPC Type: AWS::SSM::Parameter::Value - Default: /SDLF/VPC/Enabled + Default: /SDLF2/VPC/Enabled Conditions: GlueJobsNotEmpty: !Not @@ -41,7 +41,7 @@ Resources: Type: AWS::IAM::Role Properties: Path: !Sub /sdlf-${pTeamName}/ - PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF/IAM/${pTeamName}/TeamPermissionsBoundary}}" + PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF2/IAM/${pTeamName}/TeamPermissionsBoundary}}" AssumeRolePolicyDocument: Version: 2012-10-17 Statement: @@ -69,9 +69,9 @@ Resources: - kms:GenerateDataKey* - kms:ReEncrypt* Resource: - - !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/InfraKeyId}}" - - !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/DataKeyId}}" - - "{{resolve:ssm:/SDLF/KMS/KeyArn}}" + - !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/InfraKeyId}}" + - !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/DataKeyId}}" + - "{{resolve:ssm:/SDLF2/KMS/KeyArn}}" rGlueConnection: Type: AWS::Glue::Connection @@ -127,7 +127,7 @@ Resources: MaxCapacity: 2.0 GlueVersion: "4.0" Name: !Sub sdlf-${pTeamName}-${GlueJobName} - SecurityConfiguration: !Sub "{{resolve:ssm:/SDLF/Glue/${pTeamName}/SecurityConfigurationId}}" + SecurityConfiguration: !Sub "{{resolve:ssm:/SDLF2/Glue/${pTeamName}/SecurityConfigurationId}}" Role: !Ref rGlueRole Connections: !If - RunInVpc diff --git a/sdlf-cicd/template-lambda-layer.yaml b/sdlf-cicd/template-lambda-layer.yaml index 43e1c784..0dd39085 100644 --- a/sdlf-cicd/template-lambda-layer.yaml +++ b/sdlf-cicd/template-lambda-layer.yaml @@ -53,8 +53,8 @@ Resources: Name: !If [ DatalakeLibraryLayer, - !Sub "/SDLF/Lambda/Latest${LayerName}Layer", - !Sub "/SDLF/Lambda/${pTeamName}/Latest${LayerName}Layer" + !Sub "/SDLF2/Lambda/Latest${LayerName}Layer", + !Sub "/SDLF2/Lambda/${pTeamName}/Latest${LayerName}Layer" ] Type: String Value: !Ref diff --git a/sdlf-datalakeLibrary/python/datalake_library/configuration/resource_configs.py b/sdlf-datalakeLibrary/python/datalake_library/configuration/resource_configs.py index 15368b51..2726648c 100755 --- a/sdlf-datalakeLibrary/python/datalake_library/configuration/resource_configs.py +++ b/sdlf-datalakeLibrary/python/datalake_library/configuration/resource_configs.py @@ -46,43 +46,43 @@ def destination_encryption_key(self): @property def artifacts_bucket(self): if not self._artifacts_bucket: - self._artifacts_bucket = self._get_ssm_param("/SDLF/S3/ArtifactsBucket") + self._artifacts_bucket = self._get_ssm_param("/SDLF2/S3/ArtifactsBucket") return self._artifacts_bucket @property def raw_bucket(self): if not self._raw_bucket: - self._raw_bucket = self._get_ssm_param("/SDLF/S3/CentralBucket") + self._raw_bucket = self._get_ssm_param("/SDLF2/S3/CentralBucket") return self._raw_bucket @property def raw_bucket_kms_key(self): if not self._raw_bucket_kms_key: - self._raw_bucket_kms_key = self._get_ssm_param("/SDLF/KMS/CentralBucket") + self._raw_bucket_kms_key = self._get_ssm_param("/SDLF2/KMS/CentralBucket") return self._raw_bucket_kms_key @property def stage_bucket(self): if not self._stage_bucket: - self._stage_bucket = self._get_ssm_param("/SDLF/S3/StageBucket") + self._stage_bucket = self._get_ssm_param("/SDLF2/S3/StageBucket") return self._stage_bucket @property def stage_bucket_kms_key(self): if not self._stage_bucket_kms_key: - self._stage_bucket_kms_key = self._get_ssm_param("/SDLF/KMS/StageBucket") + self._stage_bucket_kms_key = self._get_ssm_param("/SDLF2/KMS/StageBucket") return self._stage_bucket_kms_key @property def analytics_bucket(self): if not self._analytics_bucket: - self._analytics_bucket = self._get_ssm_param("/SDLF/S3/AnalyticsBucket").split(":")[-1] + self._analytics_bucket = self._get_ssm_param("/SDLF2/S3/AnalyticsBucket").split(":")[-1] return self._analytics_bucket @property def analytics_bucket_kms_key(self): if not self._analytics_bucket_kms_key: - self._analytics_bucket_kms_key = self._get_ssm_param("/SDLF/KMS/AnalyticsBucket") + self._analytics_bucket_kms_key = self._get_ssm_param("/SDLF2/KMS/AnalyticsBucket") return self._analytics_bucket_kms_key @@ -110,25 +110,25 @@ def _fetch_from_ssm(self): @property def object_metadata_table(self): if not self._object_metadata_table: - self._object_metadata_table = self._get_ssm_param("/SDLF/Dynamo/ObjectCatalog") + self._object_metadata_table = self._get_ssm_param("/SDLF2/Dynamo/ObjectCatalog") return self._object_metadata_table @property def transform_mapping_table(self): if not self._transform_mapping_table: - self._transform_mapping_table = self._get_ssm_param("/SDLF/Dynamo/TransformMapping") + self._transform_mapping_table = self._get_ssm_param("/SDLF2/Dynamo/TransformMapping") return self._transform_mapping_table @property def pipelines_table(self): if not self._pipelines_table: - self._pipelines_table = self._get_ssm_param("/SDLF/Dynamo/Pipelines") + self._pipelines_table = self._get_ssm_param("/SDLF2/Dynamo/Pipelines") return self._pipelines_table @property def manifests_control_table(self): if not self._manifests_control_table: - self._manifests_control_table = self._get_ssm_param("/SDLF/Dynamo/Manifests") + self._manifests_control_table = self._get_ssm_param("/SDLF2/Dynamo/Manifests") return self._manifests_control_table @@ -158,7 +158,7 @@ def _fetch_from_ssm(self): def get_stage_queue_name(self): if not self._stage_queue_name: self._stage_queue_name = self._get_ssm_param( - "/SDLF/SQS/{}/{}{}Queue".format(self._team, self._prefix, self._stage) + "/SDLF2/SQS/{}/{}{}Queue".format(self._team, self._prefix, self._stage) ) return self._stage_queue_name @@ -166,7 +166,7 @@ def get_stage_queue_name(self): def get_stage_dlq_name(self): if not self._stage_dlq_name: self._stage_dlq_name = self._get_ssm_param( - "/SDLF/SQS/{}/{}{}DLQ".format(self._team, self._prefix, self._stage) + "/SDLF2/SQS/{}/{}{}DLQ".format(self._team, self._prefix, self._stage) ) return self._stage_dlq_name @@ -196,7 +196,7 @@ def _fetch_from_ssm(self): def get_stage_state_machine_arn(self): if not self._stage_state_machine_arn: self._stage_state_machine_arn = self._get_ssm_param( - "/SDLF/SM/{}/{}{}SM".format(self._team, self._pipeline, self._stage) + "/SDLF2/SM/{}/{}{}SM".format(self._team, self._pipeline, self._stage) ) return self._stage_state_machine_arn @@ -223,5 +223,5 @@ def _fetch_from_ssm(self): @property def get_kms_arn(self): if not self._kms_arn: - self._kms_arn = self._get_ssm_param("/SDLF/KMS/{}/DataKeyId".format(self._team)) + self._kms_arn = self._get_ssm_param("/SDLF2/KMS/{}/DataKeyId".format(self._team)) return self._kms_arn diff --git a/sdlf-dataset/template.yaml b/sdlf-dataset/template.yaml index 85121921..88525e5a 100644 --- a/sdlf-dataset/template.yaml +++ b/sdlf-dataset/template.yaml @@ -8,11 +8,11 @@ Parameters: pOrg: Description: Name of the organization owning the datalake Type: String - Default: "{{resolve:ssm:/SDLF/Misc/pOrg}}" + Default: "{{resolve:ssm:/SDLF2/Misc/pOrg}}" pDomain: Description: Data domain name Type: String - Default: "{{resolve:ssm:/SDLF/Misc/pDomain}}" + Default: "{{resolve:ssm:/SDLF2/Misc/pDomain}}" pDatasetName: Description: The name of the dataset (all lowercase, no symbols or spaces) Type: String @@ -20,11 +20,11 @@ Parameters: pEnvironment: Description: Environment name Type: String - Default: "{{resolve:ssm:/SDLF/Misc/pEnv}}" + Default: "{{resolve:ssm:/SDLF2/Misc/pEnv}}" pStageBucket: Description: The stage bucket for the solution Type: String - Default: "{{resolve:ssm:/SDLF/S3/StageBucket}}" + Default: "{{resolve:ssm:/SDLF2/S3/StageBucket}}" pTeamName: Description: Name of the team owning the pipeline (all lowercase, no symbols or spaces) Type: String @@ -72,8 +72,8 @@ Resources: rGlueCrawler: Type: AWS::Glue::Crawler Properties: - Role: !Sub "{{resolve:ssm:/SDLF/IAM/${pTeamName}/CrawlerRoleArn}}" - CrawlerSecurityConfiguration: !Sub "{{resolve:ssm:/SDLF/Glue/${pTeamName}/SecurityConfigurationId}}" + Role: !Sub "{{resolve:ssm:/SDLF2/IAM/${pTeamName}/CrawlerRoleArn}}" + CrawlerSecurityConfiguration: !Sub "{{resolve:ssm:/SDLF2/Glue/${pTeamName}/SecurityConfigurationId}}" DatabaseName: !Ref rGlueDataCatalog Name: !Sub sdlf-${pTeamName}-${pDatasetName}-post-stage-crawler Targets: @@ -84,7 +84,7 @@ Resources: Type: AWS::LakeFormation::Permissions Properties: DataLakePrincipal: - DataLakePrincipalIdentifier: !Sub "{{resolve:ssm:/SDLF/IAM/${pTeamName}/CrawlerRoleArn}}" + DataLakePrincipalIdentifier: !Sub "{{resolve:ssm:/SDLF2/IAM/${pTeamName}/CrawlerRoleArn}}" Permissions: - CREATE_TABLE - ALTER @@ -97,7 +97,7 @@ Resources: rGlueDataCatalogSsm: Type: AWS::SSM::Parameter Properties: - Name: !Sub /SDLF/Glue/${pTeamName}/${pDatasetName}/DataCatalog + Name: !Sub /SDLF2/Glue/${pTeamName}/${pDatasetName}/DataCatalog Type: String Value: !Ref rGlueDataCatalog Description: !Sub "${pTeamName} team ${pDatasetName} metadata catalog" @@ -113,7 +113,7 @@ Resources: rDatasetSsm: Type: AWS::SSM::Parameter Properties: - Name: !Sub /SDLF/Datasets/${pTeamName}/${pDatasetName} + Name: !Sub /SDLF2/Datasets/${pTeamName}/${pDatasetName} Type: String Value: !Ref pPipelineDetails # bit of a hack for datasets lambda Description: !Sub "Placeholder ${pTeamName} ${pDatasetName}" diff --git a/sdlf-foundations/lambda/replicate/src/lambda_function.py b/sdlf-foundations/lambda/replicate/src/lambda_function.py index 048e8c6b..598d6e3b 100644 --- a/sdlf-foundations/lambda/replicate/src/lambda_function.py +++ b/sdlf-foundations/lambda/replicate/src/lambda_function.py @@ -15,7 +15,7 @@ ssm = boto3.client("ssm", endpoint_url=ssm_endpoint_url) lf_endpoint_url = "https://lakeformation." + os.getenv("AWS_REGION") + ".amazonaws.com" lf = boto3.client("lakeformation", endpoint_url=lf_endpoint_url) -schemas_table = ssm.get_parameter(Name="/SDLF/Dynamo/DataSchemas")["Parameter"]["Value"] +schemas_table = ssm.get_parameter(Name="/SDLF2/Dynamo/DataSchemas")["Parameter"]["Value"] def get_current_time(): @@ -114,7 +114,7 @@ def lambda_handler(event, context): table = glue.get_table(DatabaseName=database_name, Name=table_name)["Table"] table_item = build_table_item(team, dataset, table) put_table_item(table_item) - iam_arn = ssm.get_parameter(Name="/SDLF/IAM/DataLakeAdminRoleArn")["Parameter"]["Value"] + iam_arn = ssm.get_parameter(Name="/SDLF2/IAM/DataLakeAdminRoleArn")["Parameter"]["Value"] grant_table_permissions(iam_arn, database_name, table_name, ["SELECT", "ALTER", "INSERT", "DELETE"]) except Exception as e: logger.error("Fatal error for table {} in database {}".format(table_name, database_name)) diff --git a/sdlf-foundations/template.yaml b/sdlf-foundations/template.yaml index b04341dd..d7250d9b 100644 --- a/sdlf-foundations/template.yaml +++ b/sdlf-foundations/template.yaml @@ -49,7 +49,7 @@ Parameters: 1827, 3653, ] - # the ideal would be to fetch ssm:/SDLF/VPC/Enabled and not ask the user to set this variable to true manually. + # the ideal would be to fetch ssm:/SDLF2/VPC/Enabled and not ask the user to set this variable to true manually. # however between AWS::SSM::Parameter::Value not working in CloudFormation modules, # Fn::ImportValue not being accepted in CloudFormation modules template fragments, # {{resolve:}} being evaluated later than the Conditions block, options are limited. @@ -142,8 +142,8 @@ Resources: - ssm:GetParameter - ssm:GetParameters Resource: - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/IAM/DataLakeAdminRoleArn - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/Dynamo/DataSchemas + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/IAM/DataLakeAdminRoleArn + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/Dynamo/DataSchemas ######## LAKE FORMATION ######### rDataLakeSettings: @@ -305,7 +305,7 @@ Resources: rKMSKeySsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/KMS/KeyArn + Name: /SDLF2/KMS/KeyArn Type: String Value: !GetAtt rKMSKey.Arn Description: Arn of the KMS key @@ -313,7 +313,7 @@ Resources: rLakeFormationDataAccessRoleSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/IAM/LakeFormationDataAccessRoleArn + Name: /SDLF2/IAM/LakeFormationDataAccessRoleArn Type: String Value: !GetAtt rLakeFormationDataAccessRole.Arn Description: Lake Formation Data Access Role @@ -321,7 +321,7 @@ Resources: rLakeFormationDataAccessRoleNameSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/IAM/LakeFormationDataAccessRole + Name: /SDLF2/IAM/LakeFormationDataAccessRole Type: String Value: !Ref rLakeFormationDataAccessRole Description: Lake Formation Data Access Role @@ -870,7 +870,7 @@ Resources: Action: - ssm:GetParameter - ssm:GetParameters - Resource: !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/EventBridge/* + Resource: !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/EventBridge/* rS3ObjectTriggerRule: Type: AWS::Events::Rule @@ -900,42 +900,42 @@ Resources: rS3ArtifactBucketSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/S3/ArtifactsBucket + Name: /SDLF2/S3/ArtifactsBucket Type: String Value: !Ref rArtifactsBucket Description: Name of the Artifacts S3 bucket rS3CentralBucketSsm: # kept for backward compatibility Type: AWS::SSM::Parameter Properties: - Name: /SDLF/S3/CentralBucket + Name: /SDLF2/S3/CentralBucket Type: String Value: !Ref rRawBucket Description: Name of the Central S3 bucket rS3RawBucketSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/S3/RawBucket + Name: /SDLF2/S3/RawBucket Type: String Value: !Ref rRawBucket Description: Name of the Raw S3 bucket rS3StageBucketSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/S3/StageBucket + Name: /SDLF2/S3/StageBucket Type: String Value: !Ref rStageBucket Description: Name of the Stage S3 bucket rS3AnalyticsBucketSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/S3/AnalyticsBucket + Name: /SDLF2/S3/AnalyticsBucket Type: String Value: !Ref rAnalyticsBucket Description: Name of the Analytics S3 bucket rS3AthenaBucketSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/S3/AthenaBucket + Name: /SDLF2/S3/AthenaBucket Type: String Value: !Ref rAthenaBucket Description: Name of the Athena results S3 bucket @@ -1435,42 +1435,42 @@ Resources: rDynamoObjectMetadataSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/Dynamo/ObjectCatalog + Name: /SDLF2/Dynamo/ObjectCatalog Type: String Value: !Ref rDynamoOctagonObjectMetadata Description: Name of the DynamoDB used to store metadata rDynamoTransformMappingSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/Dynamo/TransformMapping + Name: /SDLF2/Dynamo/TransformMapping Type: String Value: !Ref rDynamoOctagonDatasets Description: Name of the DynamoDB used to store mappings to transformation rDynamoPipelinesSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/Dynamo/Pipelines + Name: /SDLF2/Dynamo/Pipelines Type: String Value: !Ref rDynamoOctagonPipelines Description: Name of the DynamoDB used to store pipelines metadata rDynamoTeamMetadataSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/Dynamo/TeamMetadata + Name: /SDLF2/Dynamo/TeamMetadata Type: String Value: !Ref rDynamoOctagonTeams Description: Name of the DynamoDB used to store teams metadata rDynamoDataSchemasSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/Dynamo/DataSchemas + Name: /SDLF2/Dynamo/DataSchemas Type: String Value: !Ref rDynamoOctagonSchemas Description: Name of the DynamoDB used to store data schemas rDynamoManifestsSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/Dynamo/Manifests + Name: /SDLF2/Dynamo/Manifests Type: String Value: !Ref rDynamoOctagonManifests Description: Name of the DynamoDB used to store manifest process metadata @@ -1527,7 +1527,7 @@ Resources: rOrganizationSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/Misc/pOrg + Name: /SDLF2/Misc/pOrg Type: String Value: !Ref pOrg Description: Name of the Organization owning the datalake @@ -1535,28 +1535,28 @@ Resources: rDomainSsm: Type: AWS::SSM::Parameter Properties: - Name: !Sub /SDLF/Misc/pDomain + Name: !Sub /SDLF2/Misc/pDomain Type: String Value: !Ref pDomain Description: Data domain name rEnvironmentSsm: Type: AWS::SSM::Parameter Properties: - Name: !Sub /SDLF/Misc/pEnv + Name: !Sub /SDLF2/Misc/pEnv Type: String Value: !Ref pEnvironment Description: Environment name rDataLakeAdminRoleSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/IAM/DataLakeAdminRoleArn + Name: /SDLF2/IAM/DataLakeAdminRoleArn Type: String Value: !GetAtt rDataLakeAdminRole.Arn Description: Lake Formation Data Lake Admin Role rS3AccessLogsBucketSsm: Type: AWS::SSM::Parameter Properties: - Name: /SDLF/S3/AccessLogsBucket + Name: /SDLF2/S3/AccessLogsBucket Type: String Value: !Ref rS3AccessLogsBucket Description: S3 Access Logs Bucket diff --git a/sdlf-monitoring/template.yaml b/sdlf-monitoring/template.yaml index 9fc6ac3b..ad3c17c7 100644 --- a/sdlf-monitoring/template.yaml +++ b/sdlf-monitoring/template.yaml @@ -10,15 +10,15 @@ Parameters: pOrg: Description: Name of the organization owning the datalake Type: String - Default: "{{resolve:ssm:/SDLF/Misc/pOrg}}" + Default: "{{resolve:ssm:/SDLF2/Misc/pOrg}}" pDomain: Description: Data domain name Type: String - Default: "{{resolve:ssm:/SDLF/Misc/pDomain}}" + Default: "{{resolve:ssm:/SDLF2/Misc/pDomain}}" pEnvironment: Description: Environment name Type: String - Default: "{{resolve:ssm:/SDLF/Misc/pEnv}}" + Default: "{{resolve:ssm:/SDLF2/Misc/pEnv}}" pCloudtrailEnabled: Type: String Default: false @@ -83,7 +83,7 @@ Parameters: pDataset: Type: String Default: "" - # the ideal would be to fetch ssm:/SDLF/VPC/Enabled and not ask the user to set this variable to true manually. + # the ideal would be to fetch ssm:/SDLF2/VPC/Enabled and not ask the user to set this variable to true manually. # however between AWS::SSM::Parameter::Value not working in CloudFormation modules, # Fn::ImportValue not being accepted in CloudFormation modules template fragments, # {{resolve:}} being evaluated later than the Conditions block, options are limited. @@ -141,7 +141,7 @@ Resources: !Sub "${pOrg}-${pDomain}-${pEnvironment}-${AWS::Region}-${AWS::AccountId}-cloudtrail", ] LoggingConfiguration: - DestinationBucketName: "{{resolve:ssm:/SDLF/S3/AccessLogsBucket}}" + DestinationBucketName: "{{resolve:ssm:/SDLF2/S3/AccessLogsBucket}}" LogFilePrefix: !If [ cUseCustomBucketPrefix, @@ -152,7 +152,7 @@ Resources: ServerSideEncryptionConfiguration: - BucketKeyEnabled: True ServerSideEncryptionByDefault: - KMSMasterKeyID: !Sub "{{resolve:ssm:/SDLF/KMS/KeyArn}}" + KMSMasterKeyID: !Sub "{{resolve:ssm:/SDLF2/KMS/KeyArn}}" SSEAlgorithm: aws:kms PublicAccessBlockConfiguration: BlockPublicAcls: True @@ -207,7 +207,7 @@ Resources: Properties: LogGroupName: /aws/cloudtrail/sdlf-trail RetentionInDays: !Ref pCloudWatchLogsRetentionInDays - KmsKeyId: "{{resolve:ssm:/SDLF/KMS/KeyArn}}" + KmsKeyId: "{{resolve:ssm:/SDLF2/KMS/KeyArn}}" rTrailLogGroupRole: Type: AWS::IAM::Role @@ -251,7 +251,7 @@ Resources: IncludeManagementEvents: true ReadWriteType: All - !Ref "AWS::NoValue" - KMSKeyId: "{{resolve:ssm:/SDLF/KMS/KeyArn}}" + KMSKeyId: "{{resolve:ssm:/SDLF2/KMS/KeyArn}}" S3BucketName: !Ref rTrailBucket S3KeyPrefix: !Ref pLogFilePrefix EnableLogFileValidation: true @@ -275,7 +275,7 @@ Resources: IncludeManagementEvents: true ReadWriteType: All - !Ref "AWS::NoValue" - KMSKeyId: "{{resolve:ssm:/SDLF/KMS/KeyArn}}" + KMSKeyId: "{{resolve:ssm:/SDLF2/KMS/KeyArn}}" S3BucketName: !Ref pExternalTrailBucket S3KeyPrefix: !Ref pLogFilePrefix EnableLogFileValidation: true @@ -286,7 +286,7 @@ Resources: Type: AWS::SSM::Parameter Condition: cCloudtrailEnabled Properties: - Name: /SDLF/S3/CloudTrailBucket + Name: /SDLF2/S3/CloudTrailBucket Type: String Value: !If [cInternalBucket, !Ref rTrailBucket, !Ref pExternalTrailBucket] Description: Name of the CloudTrail S3 bucket @@ -315,7 +315,7 @@ Resources: !Sub "${pOrg}-${pDomain}-${pEnvironment}-${AWS::Region}-${AWS::AccountId}-logs", ] LoggingConfiguration: - DestinationBucketName: "{{resolve:ssm:/SDLF/S3/AccessLogsBucket}}" + DestinationBucketName: "{{resolve:ssm:/SDLF2/S3/AccessLogsBucket}}" LogFilePrefix: !If [ cUseCustomBucketPrefix, @@ -326,7 +326,7 @@ Resources: ServerSideEncryptionConfiguration: - BucketKeyEnabled: True ServerSideEncryptionByDefault: - KMSMasterKeyID: !Sub "{{resolve:ssm:/SDLF/KMS/KeyArn}}" + KMSMasterKeyID: !Sub "{{resolve:ssm:/SDLF2/KMS/KeyArn}}" SSEAlgorithm: aws:kms PublicAccessBlockConfiguration: BlockPublicAcls: True @@ -373,7 +373,7 @@ Resources: Properties: LogGroupName: !Sub /aws/kinesisfirehose/sdlf-cwlogs-to-os RetentionInDays: !Ref pCloudWatchLogsRetentionInDays - KmsKeyId: "{{resolve:ssm:/SDLF/KMS/KeyArn}}" + KmsKeyId: "{{resolve:ssm:/SDLF2/KMS/KeyArn}}" rCloudwatchLogsFirehoseLogStream: Type: AWS::Logs::LogStream @@ -417,7 +417,7 @@ Resources: Action: - kms:GenerateDataKey - kms:Decrypt - Resource: "{{resolve:ssm:/SDLF/KMS/KeyArn}}" + Resource: "{{resolve:ssm:/SDLF2/KMS/KeyArn}}" Condition: StringEquals: "kms:ViaService": !Sub "s3.${AWS::Region}.amazonaws.com" @@ -438,7 +438,7 @@ Resources: Condition: ArnEqualsIfExists: "ec2:Vpc": - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}" + - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF2/VPC/VpcId}}" - !Ref "AWS::NoValue" - !If - RunInVpc @@ -446,7 +446,7 @@ Resources: Action: - ec2:DescribeVpcAttribute Resource: - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}" + - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF2/VPC/VpcId}}" - !Ref "AWS::NoValue" - !If - RunInVpc @@ -460,7 +460,7 @@ Resources: "ec2:AuthorizedService": firehose.amazonaws.com ArnEquals: "ec2:Vpc": - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}" + - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF2/VPC/VpcId}}" - !Ref "AWS::NoValue" - Effect: Allow Action: @@ -499,7 +499,7 @@ Resources: Properties: DeliveryStreamEncryptionConfigurationInput: KeyType: CUSTOMER_MANAGED_CMK - KeyARN: "{{resolve:ssm:/SDLF/KMS/KeyArn}}" + KeyARN: "{{resolve:ssm:/SDLF2/KMS/KeyArn}}" DeliveryStreamName: sdlf-cwlogs-to-os DeliveryStreamType: DirectPut AmazonopensearchserviceDestinationConfiguration: @@ -516,7 +516,7 @@ Resources: BucketARN: !GetAtt rCloudwatchLogsFirehoseBucket.Arn EncryptionConfiguration: KMSEncryptionConfig: - AWSKMSKeyARN: "{{resolve:ssm:/SDLF/KMS/KeyArn}}" + AWSKMSKeyARN: "{{resolve:ssm:/SDLF2/KMS/KeyArn}}" CloudWatchLoggingOptions: Enabled: true LogGroupName: !Ref rCloudwatchLogsFirehoseLogGroup @@ -532,7 +532,7 @@ Resources: - cTeamLevel - !Sub "sdlf-${pTeamName}-notifications" - !Sub "sdlf-${pTeamName}-${pDataset}-notifications" - KmsMasterKeyId: !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/InfraKeyId}}" + KmsMasterKeyId: !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/InfraKeyId}}" rSnsTopicPolicy: Type: AWS::SNS::TopicPolicy @@ -559,8 +559,8 @@ Resources: Properties: Name: !If - cTeamLevel - - !Sub "/SDLF/SNS/${pTeamName}/Notifications" - - !Sub "/SDLF/SNS/${pTeamName}/${pDataset}/Notifications" + - !Sub "/SDLF2/SNS/${pTeamName}/Notifications" + - !Sub "/SDLF2/SNS/${pTeamName}/${pDataset}/Notifications" Type: String Value: !Ref rSnsTopic Description: !If diff --git a/sdlf-pipeline/template.yaml b/sdlf-pipeline/template.yaml index a55fa17b..75a0a060 100644 --- a/sdlf-pipeline/template.yaml +++ b/sdlf-pipeline/template.yaml @@ -9,15 +9,15 @@ Parameters: pOrg: Description: Name of the organization owning the datalake Type: String - Default: "{{resolve:ssm:/SDLF/Misc/pOrg}}" + Default: "{{resolve:ssm:/SDLF2/Misc/pOrg}}" pDomain: Description: Data domain name Type: String - Default: "{{resolve:ssm:/SDLF/Misc/pDomain}}" + Default: "{{resolve:ssm:/SDLF2/Misc/pDomain}}" pEnv: Description: Environment name Type: String - Default: "{{resolve:ssm:/SDLF/Misc/pEnv}}" + Default: "{{resolve:ssm:/SDLF2/Misc/pEnv}}" pTeamName: Description: The name of the team owning the pipeline (all lowercase, no symbols or spaces) Type: String @@ -57,7 +57,7 @@ Resources: Name: !Sub sdlf-${pTeamName}-${pPipelineName}-rule-${pStageName} Description: !Sub Send events to ${pStageName} queue EventPattern: !Ref pEventPattern - EventBusName: !Sub "{{resolve:ssm:/SDLF/EventBridge/${pTeamName}/EventBusName}}" + EventBusName: !Sub "{{resolve:ssm:/SDLF2/EventBridge/${pTeamName}/EventBusName}}" State: ENABLED Targets: - Id: !Sub sdlf-${pTeamName}-${pPipelineName}-rule-${pStageName} @@ -79,7 +79,7 @@ Resources: maxReceiveCount: 1 VisibilityTimeout: 60 MessageRetentionPeriod: 604800 - KmsMasterKeyId: !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/InfraKeyId}}" + KmsMasterKeyId: !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/InfraKeyId}}" rDeadLetterQueueRoutingStep: Type: AWS::SQS::Queue @@ -90,7 +90,7 @@ Resources: FifoQueue: True MessageRetentionPeriod: 1209600 VisibilityTimeout: 60 - KmsMasterKeyId: !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/InfraKeyId}}" + KmsMasterKeyId: !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/InfraKeyId}}" rQueuePolicyEventsStageRule: Type: AWS::SQS::QueuePolicy @@ -124,7 +124,7 @@ Resources: Condition: ScheduleBased Properties: Path: !Sub /sdlf-${pTeamName}/ - PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF/IAM/${pTeamName}/TeamPermissionsBoundary}}" + PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF2/IAM/${pTeamName}/TeamPermissionsBoundary}}" AssumeRolePolicyDocument: Version: "2012-10-17" Statement: @@ -149,15 +149,15 @@ Resources: Action: - kms:Decrypt Resource: - - !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/InfraKeyId}}" + - !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/InfraKeyId}}" rPostStateSchedule: Type: AWS::Scheduler::Schedule Condition: ScheduleBased Properties: Description: !Sub Trigger ${pStageName} Routing Lambda on a specified schedule - GroupName: !Sub "{{resolve:ssm:/SDLF/EventBridge/${pTeamName}/ScheduleGroupName}}" - KmsKeyArn: !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/InfraKeyId}}" + GroupName: !Sub "{{resolve:ssm:/SDLF2/EventBridge/${pTeamName}/ScheduleGroupName}}" + KmsKeyArn: !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/InfraKeyId}}" Name: !Sub sdlf-${pTeamName}-${pPipelineName}-schedule-rule-${pStageName} ScheduleExpression: !Ref pSchedule FlexibleTimeWindow: @@ -177,7 +177,7 @@ Resources: rQueueRoutingStepSsm: Type: AWS::SSM::Parameter Properties: - Name: !Sub /SDLF/SQS/${pTeamName}/${pPipelineName}${pStageName}Queue + Name: !Sub /SDLF2/SQS/${pTeamName}/${pPipelineName}${pStageName}Queue Type: String Value: !GetAtt rQueueRoutingStep.QueueName Description: !Sub "Name of the ${pStageName} ${pTeamName} ${pPipelineName} Queue" @@ -185,7 +185,7 @@ Resources: rDeadLetterQueueRoutingStepSsm: Type: AWS::SSM::Parameter Properties: - Name: !Sub /SDLF/SQS/${pTeamName}/${pPipelineName}${pStageName}DLQ + Name: !Sub /SDLF2/SQS/${pTeamName}/${pPipelineName}${pStageName}DLQ Type: String Value: !GetAtt rDeadLetterQueueRoutingStep.QueueName Description: !Sub "Name of the ${pStageName} ${pTeamName} ${pPipelineName} DLQ" @@ -193,7 +193,7 @@ Resources: rPipelineStageSsm: Type: AWS::SSM::Parameter Properties: - Name: !Sub /SDLF/Pipelines/${pTeamName}/${pPipelineName}/${pStageName} + Name: !Sub /SDLF2/Pipelines/${pTeamName}/${pPipelineName}/${pStageName} Type: String Value: placeholder Description: !Sub "Placeholder ${pTeamName} ${pPipelineName} ${pStageName}" @@ -201,4 +201,4 @@ Resources: Outputs: oPipelineReference: Description: CodePipeline reference this stack has been deployed with - Value: !Ref pPipelineReference \ No newline at end of file + Value: !Ref pPipelineReference diff --git a/sdlf-stage-dataquality/lambda/initial-check/src/lambda_function.py b/sdlf-stage-dataquality/lambda/initial-check/src/lambda_function.py index 7d4d56d2..ee6fae57 100755 --- a/sdlf-stage-dataquality/lambda/initial-check/src/lambda_function.py +++ b/sdlf-stage-dataquality/lambda/initial-check/src/lambda_function.py @@ -21,7 +21,7 @@ def get_glue_transform_details(bucket, team, dataset, env, pipeline, stage): transform_info = dynamo_interface.get_transform_table_item(f"{team}-{dataset}") - glue_database = ssm.get_parameter(Name=f"/SDLF/Glue/{team}/{dataset}/DataCatalog")["Parameter"]["Value"] + glue_database = ssm.get_parameter(Name=f"/SDLF2/Glue/{team}/{dataset}/DataCatalog")["Parameter"]["Value"] glue_capacity = {"NumberOfWorkers": 5} wait_time = 45 diff --git a/sdlf-stage-dataquality/template.yaml b/sdlf-stage-dataquality/template.yaml index 2f9f0784..6f994998 100644 --- a/sdlf-stage-dataquality/template.yaml +++ b/sdlf-stage-dataquality/template.yaml @@ -10,11 +10,11 @@ Parameters: pOrg: Description: Name of the organization owning the datalake Type: String - Default: "{{resolve:ssm:/SDLF/Misc/pOrg}}" + Default: "{{resolve:ssm:/SDLF2/Misc/pOrg}}" pDomain: Description: Data domain name Type: String - Default: "{{resolve:ssm:/SDLF/Misc/pDomain}}" + Default: "{{resolve:ssm:/SDLF2/Misc/pDomain}}" pTeamName: Description: Name of the team owning the pipeline (all lowercase, no symbols or spaces) Type: String @@ -26,7 +26,7 @@ Parameters: pEnv: Description: Environment name Type: String - Default: "{{resolve:ssm:/SDLF/Misc/pEnv}}" + Default: "{{resolve:ssm:/SDLF2/Misc/pEnv}}" pStageName: Description: Name of the stage (all lowercase, hyphen allowed, no other symbols or spaces) Type: String @@ -46,7 +46,7 @@ Parameters: pStageBucket: Description: The stage bucket for the solution Type: String - Default: "{{resolve:ssm:/SDLF/S3/StageBucket}}" + Default: "{{resolve:ssm:/SDLF2/S3/StageBucket}}" pCloudWatchLogsRetentionInDays: Description: The number of days log events are kept in CloudWatch Logs Type: Number @@ -71,7 +71,7 @@ Parameters: 1827, 3653, ] - # the ideal would be to fetch ssm:/SDLF/VPC/Enabled and not ask the user to set this variable to true manually. + # the ideal would be to fetch ssm:/SDLF2/VPC/Enabled and not ask the user to set this variable to true manually. # however between AWS::SSM::Parameter::Value not working in CloudFormation modules, # Fn::ImportValue not being accepted in CloudFormation modules template fragments, # {{resolve:}} being evaluated later than the Conditions block, options are limited. @@ -98,8 +98,8 @@ Globals: Runtime: python3.12 Handler: lambda_function.lambda_handler Layers: - - "{{resolve:ssm:/SDLF/Lambda/LatestDatalakeLibraryLayer}}" - KmsKeyArn: !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/InfraKeyId}}" + - "{{resolve:ssm:/SDLF2/Lambda/LatestDatalakeLibraryLayer}}" + KmsKeyArn: !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/InfraKeyId}}" VpcConfig: !If - RunInVpc - SecurityGroupIds: !Split [",", !Ref pVpcSecurityGroupIds] @@ -142,7 +142,7 @@ Resources: Action: - ssm:GetParameter - ssm:GetParameters - Resource: !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/* + Resource: !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/* - Effect: Allow Action: - dynamodb:BatchGetItem @@ -166,14 +166,14 @@ Resources: - kms:GenerateDataKey* - kms:ReEncrypt* Resource: - - !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/InfraKeyId}}" + - !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/InfraKeyId}}" # Routing Step Role rRoleLambdaExecutionRoutingStep: Type: AWS::IAM::Role Properties: Path: !Sub /sdlf-${pTeamName}/ - PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF/IAM/${pTeamName}/TeamPermissionsBoundary}}" + PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF2/IAM/${pTeamName}/TeamPermissionsBoundary}}" ManagedPolicyArns: - !Ref rLambdaCommonPolicy - !If @@ -217,7 +217,7 @@ Resources: Type: AWS::IAM::Role Properties: Path: !Sub /sdlf-${pTeamName}/ - PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF/IAM/${pTeamName}/TeamPermissionsBoundary}}" + PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF2/IAM/${pTeamName}/TeamPermissionsBoundary}}" ManagedPolicyArns: - !Ref rLambdaCommonPolicy - !If @@ -239,7 +239,7 @@ Resources: - Effect: Allow Action: - ssm:GetParameter - Resource: !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/Glue/* + Resource: !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/Glue/* - Effect: Allow Action: - dynamodb:GetItem @@ -255,7 +255,7 @@ Resources: reason: The actions with "*" are all ones that do not have resource limitations associated with them Properties: Path: !Sub /sdlf-${pTeamName}/ - PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF/IAM/${pTeamName}/TeamPermissionsBoundary}}" + PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF2/IAM/${pTeamName}/TeamPermissionsBoundary}}" AssumeRolePolicyDocument: Version: "2012-10-17" Statement: @@ -288,7 +288,7 @@ Resources: Action: - iam:PassRole Resource: - - !Sub "{{resolve:ssm:/SDLF/IAM/${pTeamName}/CrawlerRoleArn}}" + - !Sub "{{resolve:ssm:/SDLF2/IAM/${pTeamName}/CrawlerRoleArn}}" Condition: StringEquals: "iam:PassedToService": @@ -360,7 +360,7 @@ Resources: Properties: LogGroupName: !Sub /aws/lambda/${rLambdaRoutingStep} RetentionInDays: !Ref pCloudWatchLogsRetentionInDays - KmsKeyId: !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/InfraKeyId}}" + KmsKeyId: !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/InfraKeyId}}" rLambdaRedriveStep: Type: AWS::Serverless::Function @@ -389,7 +389,7 @@ Resources: Properties: LogGroupName: !Sub /aws/lambda/${rLambdaRedriveStep} RetentionInDays: !Ref pCloudWatchLogsRetentionInDays - KmsKeyId: !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/InfraKeyId}}" + KmsKeyId: !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/InfraKeyId}}" rLambdaStep1: Type: AWS::Serverless::Function @@ -413,7 +413,7 @@ Resources: Properties: LogGroupName: !Sub /aws/lambda/${rLambdaStep1} RetentionInDays: !Ref pCloudWatchLogsRetentionInDays - KmsKeyId: !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/InfraKeyId}}" + KmsKeyId: !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/InfraKeyId}}" ######## STEP FUNCTIONS ######### rStateMachine: @@ -423,7 +423,7 @@ Resources: DefinitionUri: ./state-machine/data-quality.asl.json DefinitionSubstitutions: lStep1: !GetAtt rLambdaStep1.Arn - dataQualityRole: !Sub "{{resolve:ssm:/SDLF/IAM/${pTeamName}/CrawlerRoleArn}}" + dataQualityRole: !Sub "{{resolve:ssm:/SDLF2/IAM/${pTeamName}/CrawlerRoleArn}}" s3DataQualityResults: !Sub "s3://${pStageBucket}/post-stage/${pTeamName}/" Role: !GetAtt rStatesExecutionRole.Arn @@ -431,7 +431,7 @@ Resources: rStateMachineSsm: Type: AWS::SSM::Parameter Properties: - Name: !Sub /SDLF/SM/${pTeamName}/${pPipeline}${pStageName}SM + Name: !Sub /SDLF2/SM/${pTeamName}/${pPipeline}${pStageName}SM Type: String Value: !Ref rStateMachine Description: !Sub "ARN of the ${pStageName} ${pTeamName} ${pPipeline} State Machine" diff --git a/sdlf-stageA/template.yaml b/sdlf-stageA/template.yaml index 02da493a..e2b5b748 100644 --- a/sdlf-stageA/template.yaml +++ b/sdlf-stageA/template.yaml @@ -10,19 +10,19 @@ Parameters: pOrg: Description: Name of the organization owning the datalake Type: String - Default: "{{resolve:ssm:/SDLF/Misc/pOrg}}" + Default: "{{resolve:ssm:/SDLF2/Misc/pOrg}}" pDomain: Description: Data domain name Type: String - Default: "{{resolve:ssm:/SDLF/Misc/pDomain}}" + Default: "{{resolve:ssm:/SDLF2/Misc/pDomain}}" pDatasetBucket: Description: The raw or central bucket for the solution Type: String - Default: "{{resolve:ssm:/SDLF/S3/CentralBucket}}" + Default: "{{resolve:ssm:/SDLF2/S3/CentralBucket}}" pStageBucket: Description: The stage bucket for the solution Type: String - Default: "{{resolve:ssm:/SDLF/S3/StageBucket}}" + Default: "{{resolve:ssm:/SDLF2/S3/StageBucket}}" pTeamName: Description: Name of the team owning the pipeline (all lowercase, no symbols or spaces) Type: String @@ -34,7 +34,7 @@ Parameters: pEnv: Description: Environment name Type: String - Default: "{{resolve:ssm:/SDLF/Misc/pEnv}}" + Default: "{{resolve:ssm:/SDLF2/Misc/pEnv}}" pStageName: Description: Name of the stage (all lowercase, hyphen allowed, no other symbols or spaces) Type: String @@ -54,15 +54,15 @@ Parameters: pElasticSearchEnabled: Description: Boolean for wether ElasticSearch is enabled Type: AWS::SSM::Parameter::Value - Default: /SDLF/ElasticSearch/Enabled + Default: /SDLF2/ElasticSearch/Enabled pKibanaStreamRole: Description: ARN of the role used to forward logs to Kinesis Type: AWS::SSM::Parameter::Value - Default: /SDLF/Lambda/KibanaStreamRoleArn + Default: /SDLF2/Lambda/KibanaStreamRoleArn pKibanaStream: Description: ARN of the Kinesis stream that collates logs Type: AWS::SSM::Parameter::Value - Default: /SDLF/Lambda/KibanaStreamArn + Default: /SDLF2/Lambda/KibanaStreamArn pCloudWatchLogsRetentionInDays: Description: The number of days log events are kept in CloudWatch Logs Type: Number @@ -90,7 +90,7 @@ Parameters: pEnableTracing: Description: Flag for whether XRay tracing is enabled Type: String - # the ideal would be to fetch ssm:/SDLF/VPC/Enabled and not ask the user to set this variable to true manually. + # the ideal would be to fetch ssm:/SDLF2/VPC/Enabled and not ask the user to set this variable to true manually. # however between AWS::SSM::Parameter::Value not working in CloudFormation modules, # Fn::ImportValue not being accepted in CloudFormation modules template fragments, # {{resolve:}} being evaluated later than the Conditions block, options are limited. @@ -119,8 +119,8 @@ Globals: Runtime: python3.12 Handler: lambda_function.lambda_handler Layers: - - "{{resolve:ssm:/SDLF/Lambda/LatestDatalakeLibraryLayer}}" - KmsKeyArn: !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/InfraKeyId}}" + - "{{resolve:ssm:/SDLF2/Lambda/LatestDatalakeLibraryLayer}}" + KmsKeyArn: !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/InfraKeyId}}" VpcConfig: !If - RunInVpc - SecurityGroupIds: !Split [",", !Ref pVpcSecurityGroupIds] @@ -163,7 +163,7 @@ Resources: Action: - ssm:GetParameter - ssm:GetParameters - Resource: !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/* + Resource: !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/* - Effect: Allow Action: - dynamodb:BatchGetItem @@ -187,14 +187,14 @@ Resources: - kms:GenerateDataKey* - kms:ReEncrypt* Resource: - - !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/InfraKeyId}}" + - !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/InfraKeyId}}" # Routing Role rRoleLambdaExecutionRoutingStep: Type: AWS::IAM::Role Properties: Path: !Sub /sdlf-${pTeamName}/ - PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF/IAM/${pTeamName}/TeamPermissionsBoundary}}" + PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF2/IAM/${pTeamName}/TeamPermissionsBoundary}}" ManagedPolicyArns: - !Ref rLambdaCommonPolicy - !If @@ -239,7 +239,7 @@ Resources: Type: AWS::IAM::Role Properties: Path: !Sub /sdlf-${pTeamName}/ - PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF/IAM/${pTeamName}/TeamPermissionsBoundary}}" + PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF2/IAM/${pTeamName}/TeamPermissionsBoundary}}" ManagedPolicyArns: - !Ref rLambdaCommonPolicy - !If @@ -276,7 +276,7 @@ Resources: Type: AWS::IAM::Role Properties: Path: !Sub /sdlf-${pTeamName}/ - PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF/IAM/${pTeamName}/TeamPermissionsBoundary}}" + PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF2/IAM/${pTeamName}/TeamPermissionsBoundary}}" ManagedPolicyArns: - !Ref rLambdaCommonPolicy - !If @@ -320,14 +320,14 @@ Resources: - kms:GenerateDataKey* - kms:CreateGrant Resource: - - !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/DataKeyId}}" + - !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/DataKeyId}}" # Step3 Role rRoleLambdaExecutionStep3: Type: AWS::IAM::Role Properties: Path: !Sub /sdlf-${pTeamName}/ - PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF/IAM/${pTeamName}/TeamPermissionsBoundary}}" + PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF2/IAM/${pTeamName}/TeamPermissionsBoundary}}" ManagedPolicyArns: - !Ref rLambdaCommonPolicy - !If @@ -375,7 +375,7 @@ Resources: Type: AWS::IAM::Role Properties: Path: !Sub /sdlf-${pTeamName}/ - PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF/IAM/${pTeamName}/TeamPermissionsBoundary}}" + PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF2/IAM/${pTeamName}/TeamPermissionsBoundary}}" ManagedPolicyArns: - !Ref rLambdaCommonPolicy - !If @@ -523,7 +523,7 @@ Resources: Properties: LogGroupName: !Sub /aws/lambda/${rLambdaRoutingStep} RetentionInDays: !Ref pCloudWatchLogsRetentionInDays - KmsKeyId: !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/InfraKeyId}}" + KmsKeyId: !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/InfraKeyId}}" rUpdateSubscriptionFilterRoutingStep: Type: AWS::Logs::SubscriptionFilter @@ -539,7 +539,7 @@ Resources: Properties: AlarmDescription: !Sub StageA ${pTeamName} ${pPipeline} Routing Lambda Alarm AlarmActions: - - !Sub "{{resolve:ssm:/SDLF/SNS/${pTeamName}/Notifications}}" + - !Sub "{{resolve:ssm:/SDLF2/SNS/${pTeamName}/Notifications}}" MetricName: Errors EvaluationPeriods: 5 Period: 60 @@ -559,7 +559,7 @@ Resources: Properties: LogGroupName: !Sub /aws/lambda/${rLambdaRedriveStep} RetentionInDays: !Ref pCloudWatchLogsRetentionInDays - KmsKeyId: !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/InfraKeyId}}" + KmsKeyId: !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/InfraKeyId}}" rUpdateSubscriptionFilterRedriveStep: Type: AWS::Logs::SubscriptionFilter @@ -577,7 +577,7 @@ Resources: Properties: LogGroupName: !Sub /aws/lambda/${rLambdaStep1} RetentionInDays: !Ref pCloudWatchLogsRetentionInDays - KmsKeyId: !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/InfraKeyId}}" + KmsKeyId: !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/InfraKeyId}}" rUpdateSubscriptionFilterStep1: Type: AWS::Logs::SubscriptionFilter @@ -595,7 +595,7 @@ Resources: Properties: LogGroupName: !Sub /aws/lambda/${rLambdaStep2} RetentionInDays: !Ref pCloudWatchLogsRetentionInDays - KmsKeyId: !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/InfraKeyId}}" + KmsKeyId: !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/InfraKeyId}}" rUpdateSubscriptionFilterStep2: Type: AWS::Logs::SubscriptionFilter @@ -613,7 +613,7 @@ Resources: Properties: LogGroupName: !Sub /aws/lambda/${rLambdaStep3} RetentionInDays: !Ref pCloudWatchLogsRetentionInDays - KmsKeyId: !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/InfraKeyId}}" + KmsKeyId: !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/InfraKeyId}}" rUpdateSubscriptionFilterStep3: Type: AWS::Logs::SubscriptionFilter @@ -631,7 +631,7 @@ Resources: Properties: LogGroupName: !Sub /aws/lambda/${rLambdaErrorStep} RetentionInDays: !Ref pCloudWatchLogsRetentionInDays - KmsKeyId: !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/InfraKeyId}}" + KmsKeyId: !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/InfraKeyId}}" rUpdateSubscriptionFilterErrorStep: Type: AWS::Logs::SubscriptionFilter @@ -647,7 +647,7 @@ Resources: Properties: AlarmDescription: !Sub StageA ${pTeamName} ${pPipeline} Error Lambda Alarm AlarmActions: - - !Sub "{{resolve:ssm:/SDLF/SNS/${pTeamName}/Notifications}}" + - !Sub "{{resolve:ssm:/SDLF2/SNS/${pTeamName}/Notifications}}" MetricName: Invocations EvaluationPeriods: 5 Period: 60 @@ -670,7 +670,7 @@ Resources: reason: The actions with "*" are all ones that do not have resource limitations associated with them Properties: Path: !Sub /sdlf-${pTeamName}/ - PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF/IAM/${pTeamName}/TeamPermissionsBoundary}}" + PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF2/IAM/${pTeamName}/TeamPermissionsBoundary}}" AssumeRolePolicyDocument: Version: "2012-10-17" Statement: @@ -716,7 +716,7 @@ Resources: rRoutingLambdaSsm: Type: AWS::SSM::Parameter Properties: - Name: !Sub /SDLF/Lambda/${pTeamName}/${pPipeline}StageARoutingLambda + Name: !Sub /SDLF2/Lambda/${pTeamName}/${pPipeline}StageARoutingLambda Type: String Value: !GetAtt rLambdaRoutingStep.Arn Description: !Sub "ARN of the StageA ${pTeamName} ${pPipeline} Routing Lambda" @@ -724,7 +724,7 @@ Resources: rStateMachineSsm: Type: AWS::SSM::Parameter Properties: - Name: !Sub /SDLF/SM/${pTeamName}/${pPipeline}${pStageName}SM + Name: !Sub /SDLF2/SM/${pTeamName}/${pPipeline}${pStageName}SM Type: String Value: !Ref rStateMachine Description: !Sub "ARN of the ${pStageName} ${pTeamName} ${pPipeline} State Machine" diff --git a/sdlf-stageB/template.yaml b/sdlf-stageB/template.yaml index 8d9a645d..48adf1b7 100644 --- a/sdlf-stageB/template.yaml +++ b/sdlf-stageB/template.yaml @@ -10,15 +10,15 @@ Parameters: pOrg: Description: Name of the organization owning the datalake Type: String - Default: "{{resolve:ssm:/SDLF/Misc/pOrg}}" + Default: "{{resolve:ssm:/SDLF2/Misc/pOrg}}" pDomain: Description: Data domain name Type: String - Default: "{{resolve:ssm:/SDLF/Misc/pDomain}}" + Default: "{{resolve:ssm:/SDLF2/Misc/pDomain}}" pDatasetBucket: Description: The raw or central bucket for the solution Type: String - Default: "{{resolve:ssm:/SDLF/S3/CentralBucket}}" + Default: "{{resolve:ssm:/SDLF2/S3/CentralBucket}}" pTeamName: Description: Name of the team owning the pipeline (all lowercase, no symbols or spaces) Type: String @@ -30,7 +30,7 @@ Parameters: pEnv: Description: Environment name Type: String - Default: "{{resolve:ssm:/SDLF/Misc/pEnv}}" + Default: "{{resolve:ssm:/SDLF2/Misc/pEnv}}" pStageName: Description: Name of the stage (all lowercase, hyphen allowed, no other symbols or spaces) Type: String @@ -50,15 +50,15 @@ Parameters: pElasticSearchEnabled: Description: Boolean for wether ElasticSearch is enabled Type: AWS::SSM::Parameter::Value - Default: /SDLF/ElasticSearch/Enabled + Default: /SDLF2/ElasticSearch/Enabled pKibanaStreamRole: Description: ARN of the role used to forward logs to Kinesis Type: AWS::SSM::Parameter::Value - Default: /SDLF/Lambda/KibanaStreamRoleArn + Default: /SDLF2/Lambda/KibanaStreamRoleArn pKibanaStream: Description: ARN of the Kinesis stream that collates logs Type: AWS::SSM::Parameter::Value - Default: /SDLF/Lambda/KibanaStreamArn + Default: /SDLF2/Lambda/KibanaStreamArn pCloudWatchLogsRetentionInDays: Description: The number of days log events are kept in CloudWatch Logs Type: Number @@ -86,7 +86,7 @@ Parameters: pEnableTracing: Description: Flag for whether XRay tracing is enabled Type: String - # the ideal would be to fetch ssm:/SDLF/VPC/Enabled and not ask the user to set this variable to true manually. + # the ideal would be to fetch ssm:/SDLF2/VPC/Enabled and not ask the user to set this variable to true manually. # however between AWS::SSM::Parameter::Value not working in CloudFormation modules, # Fn::ImportValue not being accepted in CloudFormation modules template fragments, # {{resolve:}} being evaluated later than the Conditions block, options are limited. @@ -115,8 +115,8 @@ Globals: Runtime: python3.12 Handler: lambda_function.lambda_handler Layers: - - "{{resolve:ssm:/SDLF/Lambda/LatestDatalakeLibraryLayer}}" - KmsKeyArn: !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/InfraKeyId}}" + - "{{resolve:ssm:/SDLF2/Lambda/LatestDatalakeLibraryLayer}}" + KmsKeyArn: !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/InfraKeyId}}" VpcConfig: !If - RunInVpc - SecurityGroupIds: !Split [",", !Ref pVpcSecurityGroupIds] @@ -159,7 +159,7 @@ Resources: Action: - ssm:GetParameter - ssm:GetParameters - Resource: !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/* + Resource: !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/* - Effect: Allow Action: - dynamodb:BatchGetItem @@ -183,14 +183,14 @@ Resources: - kms:GenerateDataKey* - kms:ReEncrypt* Resource: - - !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/InfraKeyId}}" + - !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/InfraKeyId}}" # Routing Step Role rRoleLambdaExecutionRoutingStep: Type: AWS::IAM::Role Properties: Path: !Sub /sdlf-${pTeamName}/ - PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF/IAM/${pTeamName}/TeamPermissionsBoundary}}" + PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF2/IAM/${pTeamName}/TeamPermissionsBoundary}}" ManagedPolicyArns: - !Ref rLambdaCommonPolicy - !If @@ -234,7 +234,7 @@ Resources: Type: AWS::IAM::Role Properties: Path: !Sub /sdlf-${pTeamName}/ - PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF/IAM/${pTeamName}/TeamPermissionsBoundary}}" + PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF2/IAM/${pTeamName}/TeamPermissionsBoundary}}" ManagedPolicyArns: - !Ref rLambdaCommonPolicy - !If @@ -254,7 +254,7 @@ Resources: Type: AWS::IAM::Role Properties: Path: !Sub /sdlf-${pTeamName}/ - PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF/IAM/${pTeamName}/TeamPermissionsBoundary}}" + PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF2/IAM/${pTeamName}/TeamPermissionsBoundary}}" ManagedPolicyArns: - !Ref rLambdaCommonPolicy - !If @@ -291,7 +291,7 @@ Resources: Type: AWS::IAM::Role Properties: Path: !Sub /sdlf-${pTeamName}/ - PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF/IAM/${pTeamName}/TeamPermissionsBoundary}}" + PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF2/IAM/${pTeamName}/TeamPermissionsBoundary}}" ManagedPolicyArns: - !Ref rLambdaCommonPolicy - !If @@ -421,7 +421,7 @@ Resources: Properties: LogGroupName: !Sub /aws/lambda/${rLambdaRoutingStep} RetentionInDays: !Ref pCloudWatchLogsRetentionInDays - KmsKeyId: !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/InfraKeyId}}" + KmsKeyId: !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/InfraKeyId}}" rUpdateSubscriptionFilterRoutingStep: Type: AWS::Logs::SubscriptionFilter @@ -437,7 +437,7 @@ Resources: Properties: AlarmDescription: !Sub StageB ${pTeamName} ${pPipeline} Routing Lambda Alarm AlarmActions: - - !Sub "{{resolve:ssm:/SDLF/SNS/${pTeamName}/Notifications}}" + - !Sub "{{resolve:ssm:/SDLF2/SNS/${pTeamName}/Notifications}}" MetricName: Errors EvaluationPeriods: 5 Period: 300 @@ -457,7 +457,7 @@ Resources: Properties: LogGroupName: !Sub /aws/lambda/${rLambdaRedriveStep} RetentionInDays: !Ref pCloudWatchLogsRetentionInDays - KmsKeyId: !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/InfraKeyId}}" + KmsKeyId: !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/InfraKeyId}}" rUpdateSubscriptionFilterRedriveStep: Type: AWS::Logs::SubscriptionFilter @@ -475,7 +475,7 @@ Resources: Properties: LogGroupName: !Sub /aws/lambda/${rLambdaStep1} RetentionInDays: !Ref pCloudWatchLogsRetentionInDays - KmsKeyId: !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/InfraKeyId}}" + KmsKeyId: !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/InfraKeyId}}" rUpdateSubscriptionFilterStep1: Type: AWS::Logs::SubscriptionFilter @@ -493,7 +493,7 @@ Resources: Properties: LogGroupName: !Sub /aws/lambda/${rLambdaStep3} RetentionInDays: !Ref pCloudWatchLogsRetentionInDays - KmsKeyId: !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/InfraKeyId}}" + KmsKeyId: !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/InfraKeyId}}" rUpdateSubscriptionFilterStep3: Type: AWS::Logs::SubscriptionFilter @@ -511,7 +511,7 @@ Resources: Properties: LogGroupName: !Sub /aws/lambda/${rLambdaErrorStep} RetentionInDays: !Ref pCloudWatchLogsRetentionInDays - KmsKeyId: !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/InfraKeyId}}" + KmsKeyId: !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/InfraKeyId}}" rUpdateSubscriptionFilterErrorStep: Type: AWS::Logs::SubscriptionFilter @@ -527,7 +527,7 @@ Resources: Properties: AlarmDescription: !Sub StageB ${pTeamName} ${pPipeline} Error Lambda Alarm AlarmActions: - - !Sub "{{resolve:ssm:/SDLF/SNS/${pTeamName}/Notifications}}" + - !Sub "{{resolve:ssm:/SDLF2/SNS/${pTeamName}/Notifications}}" MetricName: Invocations EvaluationPeriods: 5 Period: 300 @@ -550,7 +550,7 @@ Resources: reason: The actions with "*" are all ones that do not have resource limitations associated with them Properties: Path: !Sub /sdlf-${pTeamName}/ - PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF/IAM/${pTeamName}/TeamPermissionsBoundary}}" + PermissionsBoundary: !Sub "{{resolve:ssm:/SDLF2/IAM/${pTeamName}/TeamPermissionsBoundary}}" AssumeRolePolicyDocument: Version: "2012-10-17" Statement: @@ -610,7 +610,7 @@ Resources: rStateMachineSsm: Type: AWS::SSM::Parameter Properties: - Name: !Sub /SDLF/SM/${pTeamName}/${pPipeline}${pStageName}SM + Name: !Sub /SDLF2/SM/${pTeamName}/${pPipeline}${pStageName}SM Type: String Value: !Ref rStateMachine Description: !Sub "ARN of the ${pStageName} ${pTeamName} ${pPipeline} State Machine" diff --git a/sdlf-team/lambda/datasets-dynamodb/src/lambda_function.py b/sdlf-team/lambda/datasets-dynamodb/src/lambda_function.py index 0e17e72e..abf112ed 100644 --- a/sdlf-team/lambda/datasets-dynamodb/src/lambda_function.py +++ b/sdlf-team/lambda/datasets-dynamodb/src/lambda_function.py @@ -49,7 +49,7 @@ def lambda_handler(event, context): paginator = ssm.get_paginator("get_parameters_by_path") datasets_pages = paginator.paginate( - Path=f"/SDLF/Datasets/{team_name}", + Path=f"/SDLF2/Datasets/{team_name}", PaginationConfig={"MaxItems": 30}, ) diff --git a/sdlf-team/lambda/pipelines-dynamodb/src/lambda_function.py b/sdlf-team/lambda/pipelines-dynamodb/src/lambda_function.py index b25f6a32..309d0338 100644 --- a/sdlf-team/lambda/pipelines-dynamodb/src/lambda_function.py +++ b/sdlf-team/lambda/pipelines-dynamodb/src/lambda_function.py @@ -51,7 +51,7 @@ def lambda_handler(event, context): paginator = ssm.get_paginator("get_parameters_by_path") stages_pages = paginator.paginate( - Path=f"/SDLF/Pipelines/{team_name}", + Path=f"/SDLF2/Pipelines/{team_name}", Recursive=True, PaginationConfig={"MaxItems": 30}, ) diff --git a/sdlf-team/template.yaml b/sdlf-team/template.yaml index b5d6b824..5517c4b7 100644 --- a/sdlf-team/template.yaml +++ b/sdlf-team/template.yaml @@ -9,31 +9,31 @@ Parameters: pOrg: Description: Name of the organization owning the datalake Type: String - Default: "{{resolve:ssm:/SDLF/Misc/pOrg}}" + Default: "{{resolve:ssm:/SDLF2/Misc/pOrg}}" pDomain: Description: Data domain name Type: String - Default: "{{resolve:ssm:/SDLF/Misc/pDomain}}" + Default: "{{resolve:ssm:/SDLF2/Misc/pDomain}}" pAnalyticsBucket: Description: The analytics bucket for the solution Type: String - Default: "{{resolve:ssm:/SDLF/S3/AnalyticsBucket}}" + Default: "{{resolve:ssm:/SDLF2/S3/AnalyticsBucket}}" pCentralBucket: Description: The central bucket for the solution Type: String - Default: "{{resolve:ssm:/SDLF/S3/CentralBucket}}" + Default: "{{resolve:ssm:/SDLF2/S3/CentralBucket}}" pAthenaBucket: Description: S3 Athena bucket Type: String - Default: "{{resolve:ssm:/SDLF/S3/AthenaBucket}}" + Default: "{{resolve:ssm:/SDLF2/S3/AthenaBucket}}" pEnvironment: Description: Environment name Type: String - Default: "{{resolve:ssm:/SDLF/Misc/pEnv}}" + Default: "{{resolve:ssm:/SDLF2/Misc/pEnv}}" pArtifactsBucket: Description: The artifacts bucket used by CodeBuild and CodePipeline Type: String - Default: "{{resolve:ssm:/SDLF/S3/ArtifactsBucket}}" + Default: "{{resolve:ssm:/SDLF2/S3/ArtifactsBucket}}" # pSNSNotificationsEmail: # Description: Email address for SNS notifications # Type: String @@ -41,14 +41,14 @@ Parameters: pStageBucket: Description: The stage bucket for the solution Type: String - Default: "{{resolve:ssm:/SDLF/S3/StageBucket}}" + Default: "{{resolve:ssm:/SDLF2/S3/StageBucket}}" pTeamName: Description: Name of the team (all lowercase, no symbols or spaces) Type: String AllowedPattern: "[a-z0-9]{2,12}" pLakeFormationDataAccessRole: Type: String - Default: "{{resolve:ssm:/SDLF/IAM/LakeFormationDataAccessRoleArn}}" + Default: "{{resolve:ssm:/SDLF2/IAM/LakeFormationDataAccessRoleArn}}" pCloudWatchLogsRetentionInDays: Description: The number of days log events are kept in CloudWatch Logs Type: Number @@ -73,7 +73,7 @@ Parameters: 1827, 3653, ] - # the ideal would be to fetch ssm:/SDLF/VPC/Enabled and not ask the user to set this variable to true manually. + # the ideal would be to fetch ssm:/SDLF2/VPC/Enabled and not ask the user to set this variable to true manually. # however between AWS::SSM::Parameter::Value not working in CloudFormation modules, # Fn::ImportValue not being accepted in CloudFormation modules template fragments, # {{resolve:}} being evaluated later than the Conditions block, options are limited. @@ -303,14 +303,14 @@ Resources: rKMSInfraKeySsm: Type: AWS::SSM::Parameter Properties: - Name: !Sub /SDLF/KMS/${pTeamName}/InfraKeyId + Name: !Sub /SDLF2/KMS/${pTeamName}/InfraKeyId Type: String Value: !GetAtt rKMSInfraKey.Arn Description: !Sub Arn of the ${pTeamName} KMS infrastructure key rKMSDataKeySsm: Type: AWS::SSM::Parameter Properties: - Name: !Sub /SDLF/KMS/${pTeamName}/DataKeyId + Name: !Sub /SDLF2/KMS/${pTeamName}/DataKeyId Type: String Value: !GetAtt rKMSDataKey.Arn Description: !Sub Arn of the ${pTeamName} KMS data key @@ -318,7 +318,7 @@ Resources: rEventBusSsm: Type: AWS::SSM::Parameter Properties: - Name: !Sub /SDLF/EventBridge/${pTeamName}/EventBusName + Name: !Sub /SDLF2/EventBridge/${pTeamName}/EventBusName Type: String Value: !Ref rEventBus Description: !Sub Name of the ${pTeamName} event bus @@ -326,7 +326,7 @@ Resources: rScheduleGroupSsm: Type: AWS::SSM::Parameter Properties: - Name: !Sub /SDLF/EventBridge/${pTeamName}/ScheduleGroupName + Name: !Sub /SDLF2/EventBridge/${pTeamName}/ScheduleGroupName Type: String Value: !Ref rScheduleGroup Description: !Sub Name of the ${pTeamName} schedule group @@ -334,7 +334,7 @@ Resources: rGlueSecurityConfigurationSsm: Type: AWS::SSM::Parameter Properties: - Name: !Sub /SDLF/Glue/${pTeamName}/SecurityConfigurationId + Name: !Sub /SDLF2/Glue/${pTeamName}/SecurityConfigurationId Type: String Value: !Sub sdlf-${pTeamName}-glue-security-config # unfortunately AWS::Glue::SecurityConfiguration doesn't provide any return value Description: !Sub Name of the ${pTeamName} Glue security configuration @@ -431,7 +431,7 @@ Resources: Action: - ssm:GetParameter - ssm:GetParameters - Resource: !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/* + Resource: !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/* - Sid: AllowOctagonDynamoAccess Effect: Allow Action: @@ -542,7 +542,7 @@ Resources: Condition: ArnEqualsIfExists: "ec2:Vpc": - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}" + - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF2/VPC/VpcId}}" - !Ref "AWS::NoValue" rDatalakeCrawlerRole: @@ -615,7 +615,7 @@ Resources: Resource: - !GetAtt rKMSInfraKey.Arn - !GetAtt rKMSDataKey.Arn - - "{{resolve:ssm:/SDLF/KMS/KeyArn}}" + - "{{resolve:ssm:/SDLF2/KMS/KeyArn}}" - Effect: Allow Action: - lakeformation:GetDataAccess # W11 exception @@ -633,14 +633,14 @@ Resources: rTeamIAMManagedPolicySsm: Type: AWS::SSM::Parameter Properties: - Name: !Sub /SDLF/IAM/${pTeamName}/TeamPermissionsBoundary + Name: !Sub /SDLF2/IAM/${pTeamName}/TeamPermissionsBoundary Type: String Value: !Ref rTeamIAMManagedPolicy Description: The permissions boundary IAM Managed policy for the team rDatalakeCrawlerRoleArnSsm: Type: AWS::SSM::Parameter Properties: - Name: !Sub /SDLF/IAM/${pTeamName}/CrawlerRoleArn + Name: !Sub /SDLF2/IAM/${pTeamName}/CrawlerRoleArn Type: String Value: !GetAtt rDatalakeCrawlerRole.Arn Description: The ARN of the Crawler role @@ -736,8 +736,8 @@ Resources: Action: - ssm:GetParametersByPath Resource: - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/Pipelines/${pTeamName} - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/Datasets/${pTeamName} + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/Pipelines/${pTeamName} + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/Datasets/${pTeamName} rDatasetsDynamodbLambda: Type: AWS::Serverless::Function @@ -849,8 +849,8 @@ Resources: Action: - ssm:GetParametersByPath Resource: - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/Pipelines/${pTeamName} - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/Datasets/${pTeamName} + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/Pipelines/${pTeamName} + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF2/Datasets/${pTeamName} rPipelinesDynamodbLambda: Type: AWS::Serverless::Function @@ -900,7 +900,7 @@ Resources: rSNSTopicSsm: Type: AWS::SSM::Parameter Properties: - Name: !Sub /SDLF/SNS/${pTeamName}/Notifications + Name: !Sub /SDLF2/SNS/${pTeamName}/Notifications Type: String Value: !Ref rSNSTopic Description: The ARN of the team-specific SNS Topic @@ -1013,7 +1013,7 @@ Resources: rAthenaWorkgroupSsm: Type: AWS::SSM::Parameter Properties: - Name: !Sub /SDLF/Athena/${pTeamName}/WorkgroupName + Name: !Sub /SDLF2/Athena/${pTeamName}/WorkgroupName Type: String Value: !Ref rAthenaWorkgroup Description: The name of the Athena workgroup diff --git a/sdlf-utils/workshop-examples/10-deployment/sdlf-main-datalake-engineering/pipeline-main.yaml b/sdlf-utils/workshop-examples/10-deployment/sdlf-main-datalake-engineering/pipeline-main.yaml index 89c8fcb4..ebadcaf4 100644 --- a/sdlf-utils/workshop-examples/10-deployment/sdlf-main-datalake-engineering/pipeline-main.yaml +++ b/sdlf-utils/workshop-examples/10-deployment/sdlf-main-datalake-engineering/pipeline-main.yaml @@ -21,7 +21,7 @@ Resources: "detail-type": ["Object Created"], "detail": { "bucket": { - "name": ["{{resolve:ssm:/SDLF/S3/CentralBucket}}"] + "name": ["{{resolve:ssm:/SDLF2/S3/CentralBucket}}"] }, "object": { "key": [{ "prefix": "engineering/legislators/" }] @@ -34,7 +34,7 @@ Resources: Type: awslabs::sdlf::stageB::MODULE Properties: pPipelineReference: !Ref pPipelineReference - pDatasetBucket: "{{resolve:ssm:/SDLF/S3/StageBucket}}" + pDatasetBucket: "{{resolve:ssm:/SDLF2/S3/StageBucket}}" pStageName: B pPipeline: main pTeamName: engineering diff --git a/sdlf-utils/workshop-examples/legislators/deploy.sh b/sdlf-utils/workshop-examples/legislators/deploy.sh index 96a8915f..826b031f 100755 --- a/sdlf-utils/workshop-examples/legislators/deploy.sh +++ b/sdlf-utils/workshop-examples/legislators/deploy.sh @@ -39,7 +39,7 @@ echo $TEAM_NAME REGION=$(aws configure get region --profile "$PROFILE") if ! "$sflag" then - S3_BUCKET=$(aws --region "$REGION" --profile "$PROFILE" ssm get-parameter --name /SDLF/S3/ArtifactsBucket --query "Parameter.Value" --output text) + S3_BUCKET=$(aws --region "$REGION" --profile "$PROFILE" ssm get-parameter --name /SDLF2/S3/ArtifactsBucket --query "Parameter.Value" --output text) fi echo "Checking if bucket exists ..." @@ -54,7 +54,7 @@ if ! aws s3 ls "$S3_BUCKET" --profile "$PROFILE"; then fi fi -ARTIFACTS_BUCKET=$(aws --region "$REGION" --profile "$PROFILE" ssm get-parameter --name /SDLF/S3/ArtifactsBucket --query "Parameter.Value" --output text) +ARTIFACTS_BUCKET=$(aws --region "$REGION" --profile "$PROFILE" ssm get-parameter --name /SDLF2/S3/ArtifactsBucket --query "Parameter.Value" --output text) aws s3 cp "$DIRNAME/scripts/legislators-glue-job.py" "s3://$ARTIFACTS_BUCKET/artifacts/" --profile "$PROFILE" mkdir "$DIRNAME"/output @@ -63,9 +63,9 @@ function send_legislators() { ORIGIN="$DIRNAME/data/" - CENTRAL_BUCKET=$(aws --region "$REGION" --profile "$PROFILE" ssm get-parameter --name /SDLF/S3/CentralBucket --query "Parameter.Value" --output text) - STAGE_BUCKET=$(aws --region "$REGION" --profile "$PROFILE" ssm get-parameter --name /SDLF/S3/StageBucket --query "Parameter.Value" --output text) - KMS_KEY=$(aws --region "$REGION" --profile "$PROFILE" ssm get-parameter --name /SDLF/KMS/$TEAM_NAME/DataKeyId --query "Parameter.Value" --output text) + CENTRAL_BUCKET=$(aws --region "$REGION" --profile "$PROFILE" ssm get-parameter --name /SDLF2/S3/CentralBucket --query "Parameter.Value" --output text) + STAGE_BUCKET=$(aws --region "$REGION" --profile "$PROFILE" ssm get-parameter --name /SDLF2/S3/StageBucket --query "Parameter.Value" --output text) + KMS_KEY=$(aws --region "$REGION" --profile "$PROFILE" ssm get-parameter --name /SDLF2/KMS/$TEAM_NAME/DataKeyId --query "Parameter.Value" --output text) S3_DESTINATION=s3://$CENTRAL_BUCKET/ COUNT=0 diff --git a/sdlf-utils/workshop-examples/legislators/scripts/legislators-glue-job.yaml b/sdlf-utils/workshop-examples/legislators/scripts/legislators-glue-job.yaml index ec8e73a1..f1bb1819 100644 --- a/sdlf-utils/workshop-examples/legislators/scripts/legislators-glue-job.yaml +++ b/sdlf-utils/workshop-examples/legislators/scripts/legislators-glue-job.yaml @@ -9,7 +9,7 @@ Parameters: pArtifactsBucket: Description: The artifacts bucket used by CodeBuild and CodePipeline Type: AWS::SSM::Parameter::Value - Default: /SDLF/S3/ArtifactsBucket + Default: /SDLF2/S3/ArtifactsBucket pTeamName: Description: The team name Type: String @@ -17,7 +17,7 @@ Parameters: pEnableVpc: Description: Deploy SDLF resources in a VPC Type: AWS::SSM::Parameter::Value - Default: /SDLF/VPC/Enabled + Default: /SDLF2/VPC/Enabled Conditions: RunInVpc: !Equals [!Ref pEnableVpc, true] @@ -54,9 +54,9 @@ Resources: - kms:GenerateDataKey* - kms:ReEncrypt* Resource: - - !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/InfraKeyId:1}}" - - !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/DataKeyId:1}}" - - "{{resolve:ssm:/SDLF/KMS/KeyArn:1}}" + - !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/InfraKeyId:1}}" + - !Sub "{{resolve:ssm:/SDLF2/KMS/${pTeamName}/DataKeyId:1}}" + - "{{resolve:ssm:/SDLF2/KMS/KeyArn:1}}" rGlueJob: Type: AWS::Glue::Job