build stage-lambda and stage-glue by default

awslabs · Jul 10, 2024 · 511a8b2 · 511a8b2
1 parent 31e0e18
commit 511a8b2
Show file tree

Hide file tree

Showing 5 changed files with 213 additions and 1 deletion.
diff --git a/deploy.sh b/deploy.sh
@@ -257,7 +257,7 @@ devops_account () {
     template_protection "$STACK_NAME" "$REGION" "$DEVOPS_AWS_PROFILE"
     rm -Rf "$DIRNAME"/output
 
-    declare -a REPOSITORIES=("sdlf-cicd" "sdlf-foundations" "sdlf-team" "sdlf-pipeline" "sdlf-dataset" "sdlf-datalakeLibrary" "sdlf-stageA" "sdlf-stageB" "sdlf-main")
+    declare -a REPOSITORIES=("sdlf-cicd" "sdlf-foundations" "sdlf-team" "sdlf-pipeline" "sdlf-dataset" "sdlf-datalakeLibrary" "sdlf-stageA" "sdlf-stageB" "sdlf-stage-lambda" "sdlf-stage-glue" "sdlf-main")
     if "$MONITORING"
     then
         REPOSITORIES+=("sdlf-monitoring")

diff --git a/sdlf-cicd/lambda/domain-cicd/src/lambda_function.py b/sdlf-cicd/lambda/domain-cicd/src/lambda_function.py
@@ -289,11 +289,21 @@ def create_team_pipeline_cicd_stack(
                     "ParameterValue": f"/SDLF/{git_platform}/StageA{git_platform}",
                     "UsePreviousValue": False,
                 },
+                {
+                    "ParameterKey": "pStageLambdaRepository",
+                    "ParameterValue": f"/SDLF/{git_platform}/StageLambda{git_platform}",
+                    "UsePreviousValue": False,
+                },
                 {
                     "ParameterKey": "pStageBRepository",
                     "ParameterValue": f"/SDLF/{git_platform}/StageB{git_platform}",
                     "UsePreviousValue": False,
                 },
+                {
+                    "ParameterKey": "pStageGlueRepository",
+                    "ParameterValue": f"/SDLF/{git_platform}/StageGlue{git_platform}",
+                    "UsePreviousValue": False,
+                },
                 {
                     "ParameterKey": "pDatasetRepository",
                     "ParameterValue": f"/SDLF/{git_platform}/Dataset{git_platform}",

diff --git a/sdlf-cicd/template-cicd-sdlf-repositories.gitlab.yaml b/sdlf-cicd/template-cicd-sdlf-repositories.gitlab.yaml
@@ -27,9 +27,15 @@ Parameters:
   pStageARepository:
     Type: String
     Default: sdlf-stageA
+  pStageLambdaRepository:
+    Type: String
+    Default: sdlf-stage-lambda
   pStageBRepository:
     Type: String
     Default: sdlf-stageB
+  pStageGlueRepository:
+    Type: String
+    Default: sdlf-stage-glue
   pDatalakeLibraryRepository:
     Type: String
     Default: sdlf-datalakeLibrary
@@ -124,6 +130,17 @@ Resources:
       Name: !Ref pStageARepository
 #      Path: !Ref pSdlfGitLabGroup
 
+  rStageLambdaGitLab:
+    Type: GitLab::Projects::Project
+    Metadata:
+      cfn-lint:
+        config:
+          ignore_checks:
+          - E3001
+    Properties:
+      Name: !Ref pStageLambdaRepository
+#      Path: !Ref pSdlfGitLabGroup
+
   rStageBGitLab:
     Type: GitLab::Projects::Project
     Metadata:
@@ -135,6 +152,17 @@ Resources:
       Name: !Ref pStageBRepository
 #      Path: !Ref pSdlfGitLabGroup
 
+  rStageGlueGitLab:
+    Type: GitLab::Projects::Project
+    Metadata:
+      cfn-lint:
+        config:
+          ignore_checks:
+          - E3001
+    Properties:
+      Name: !Ref pStageGlueRepository
+#      Path: !Ref pSdlfGitLabGroup
+
   rDatalakeLibraryGitLab:
     Type: GitLab::Projects::Project
     Metadata:
@@ -225,6 +253,14 @@ Resources:
       Value: !Ref pStageARepository # !GetAtt rStageAGitLab.Name
       Description: Name of the StageA repository
 
+  rStageLambdaGitLabSsm:
+    Type: AWS::SSM::Parameter
+    Properties:
+      Name: /SDLF/GitLab/StageLambdaGitLab
+      Type: String
+      Value: !Ref pStageLambdaRepository # !GetAtt rStageLambdaGitLab.Name
+      Description: Name of the Stage-Lambda repository
+
   rStageBGitLabSsm:
     Type: AWS::SSM::Parameter
     Properties:
@@ -233,6 +269,14 @@ Resources:
       Value: !Ref pStageBRepository # !GetAtt rStageBGitLab.Name
       Description: Name of the StageB repository
 
+  rStageGlueGitLabSsm:
+    Type: AWS::SSM::Parameter
+    Properties:
+      Name: /SDLF/GitLab/StageGlueGitLab
+      Type: String
+      Value: !Ref pStageGlueRepository # !GetAtt rStageGlueGitLab.Name
+      Description: Name of the Stage-Glue repository
+
   rDatalakeLibraryGitLabSsm:
     Type: AWS::SSM::Parameter
     Properties:

diff --git a/sdlf-cicd/template-cicd-sdlf-repositories.yaml b/sdlf-cicd/template-cicd-sdlf-repositories.yaml
@@ -24,9 +24,15 @@ Parameters:
   pStageARepository:
     Type: String
     Default: sdlf-stageA
+  pStageLambdaRepository:
+    Type: String
+    Default: sdlf-stage-lambda
   pStageBRepository:
     Type: String
     Default: sdlf-stageB
+  pStageGlueRepository:
+    Type: String
+    Default: sdlf-stage-glue
   pDatalakeLibraryRepository:
     Type: String
     Default: sdlf-datalakeLibrary
@@ -139,6 +145,21 @@ Resources:
       RepositoryName: !Ref pStageARepository
       KmsKeyId: !Ref pKMSKey
 
+  rStageLambdaCodeCommit:
+    Type: AWS::CodeCommit::Repository
+    Metadata:
+      cfn-lint:
+        config:
+          ignore_checks:
+            - E3002
+    Properties:
+      Code:
+        BranchName: main
+        S3: ../sdlf-stage-lambda
+      RepositoryDescription: sdlf-stage-lambda repository
+      RepositoryName: !Ref pStageLambdaRepository
+      KmsKeyId: !Ref pKMSKey
+
   rStageBCodeCommit:
     Type: AWS::CodeCommit::Repository
     Metadata:
@@ -154,6 +175,21 @@ Resources:
       RepositoryName: !Ref pStageBRepository
       KmsKeyId: !Ref pKMSKey
 
+  rStageGlueCodeCommit:
+    Type: AWS::CodeCommit::Repository
+    Metadata:
+      cfn-lint:
+        config:
+          ignore_checks:
+            - E3002
+    Properties:
+      Code:
+        BranchName: main
+        S3: ../sdlf-stage-glue
+      RepositoryDescription: sdlf-stage-glue repository
+      RepositoryName: !Ref pStageGlueRepository
+      KmsKeyId: !Ref pKMSKey
+
   rDatalakeLibraryCodeCommit:
     Type: AWS::CodeCommit::Repository
     Metadata:
@@ -248,6 +284,14 @@ Resources:
       Value: !GetAtt rStageACodeCommit.Name
       Description: Name of the StageA repository
 
+  rStageLambdaCodeCommitSsm:
+    Type: AWS::SSM::Parameter
+    Properties:
+      Name: /SDLF/CodeCommit/StageLambdaCodeCommit
+      Type: String
+      Value: !GetAtt rStageLambdaCodeCommit.Name
+      Description: Name of the Stage-Lambda repository
+
   rStageBCodeCommitSsm:
     Type: AWS::SSM::Parameter
     Properties:
@@ -256,6 +300,14 @@ Resources:
       Value: !GetAtt rStageBCodeCommit.Name
       Description: Name of the StageB repository
 
+  rStageGlueCodeCommitSsm:
+    Type: AWS::SSM::Parameter
+    Properties:
+      Name: /SDLF/CodeCommit/StageGlueCodeCommit
+      Type: String
+      Value: !GetAtt rStageGlueCodeCommit.Name
+      Description: Name of the Stage-Glue repository
+
   rDatalakeLibraryCodeCommitSsm:
     Type: AWS::SSM::Parameter
     Properties:

diff --git a/sdlf-cicd/template-cicd-team-pipeline.yaml b/sdlf-cicd/template-cicd-team-pipeline.yaml
@@ -71,9 +71,15 @@ Parameters:
   pStageARepository:
     Type: AWS::SSM::Parameter::Value<String>
     Default: /SDLF/CodeCommit/StageACodeCommit
+  pStageLambdaRepository:
+    Type: AWS::SSM::Parameter::Value<String>
+    Default: /SDLF/CodeCommit/StageLambdaCodeCommit
   pStageBRepository:
     Type: AWS::SSM::Parameter::Value<String>
     Default: /SDLF/CodeCommit/StageBCodeCommit
+  pStageGlueRepository:
+    Type: AWS::SSM::Parameter::Value<String>
+    Default: /SDLF/CodeCommit/StageGlueCodeCommit
   pDatasetRepository:
     Type: AWS::SSM::Parameter::Value<String>
     Default: /SDLF/CodeCommit/DatasetCodeCommit
@@ -139,7 +145,9 @@ Resources:
                     - !Sub arn:${AWS::Partition}:codecommit:${AWS::Region}:${AWS::AccountId}:${pDatalakeLibraryRepository}
                     - !Sub arn:${AWS::Partition}:codecommit:${AWS::Region}:${AWS::AccountId}:${pPipelineRepository}
                     - !Sub arn:${AWS::Partition}:codecommit:${AWS::Region}:${AWS::AccountId}:${pStageARepository}
+                    - !Sub arn:${AWS::Partition}:codecommit:${AWS::Region}:${AWS::AccountId}:${pStageLambdaRepository}
                     - !Sub arn:${AWS::Partition}:codecommit:${AWS::Region}:${AWS::AccountId}:${pStageBRepository}
+                    - !Sub arn:${AWS::Partition}:codecommit:${AWS::Region}:${AWS::AccountId}:${pStageGlueRepository}
                     - !Sub arn:${AWS::Partition}:codecommit:${AWS::Region}:${AWS::AccountId}:${pDatasetRepository}
                 - !Ref "AWS::NoValue"
               - !If
@@ -158,15 +166,19 @@ Resources:
                         - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pDatalakeLibraryRepository}"
                         - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pPipelineRepository}"
                         - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pStageARepository}"
+                        - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pStageLambdaRepository}"
                         - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pStageBRepository}"
+                        - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pStageGlueRepository}"
                         - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pDatasetRepository}"
                       "codestar-connections:FullRepositoryId":
                         - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/{{resolve:ssm:/SDLF/${pGitPlatform}/${pTeamName}/Main${pGitPlatform}}}"
                         - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pCicdRepository}"
                         - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pDatalakeLibraryRepository}"
                         - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pPipelineRepository}"
                         - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pStageARepository}"
+                        - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pStageLambdaRepository}"
                         - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pStageBRepository}"
+                        - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pStageGlueRepository}"
                         - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pDatasetRepository}"
                 - !Ref "AWS::NoValue"
               - Effect: Allow
@@ -284,6 +296,19 @@ Resources:
                   BranchName: !FindInMap [pCodeCommitBranch, !Ref pEnvironment, branch]
                   PollForSourceChanges: false
                 RunOrder: 1
+              - Name: SourceStageLambda
+                ActionTypeId:
+                  Category: Source
+                  Owner: AWS
+                  Provider: CodeCommit
+                  Version: "1"
+                OutputArtifacts:
+                  - Name: SourceStageLambdaArtifact
+                Configuration:
+                  RepositoryName: !Ref pStageLambdaRepository
+                  BranchName: !FindInMap [pCodeCommitBranch, !Ref pEnvironment, branch]
+                  PollForSourceChanges: false
+                RunOrder: 1
               - Name: SourceStageB
                 ActionTypeId:
                   Category: Source
@@ -297,6 +322,19 @@ Resources:
                   BranchName: !FindInMap [pCodeCommitBranch, !Ref pEnvironment, branch]
                   PollForSourceChanges: false
                 RunOrder: 1
+              - Name: SourceStageGlue
+                ActionTypeId:
+                  Category: Source
+                  Owner: AWS
+                  Provider: CodeCommit
+                  Version: "1"
+                OutputArtifacts:
+                  - Name: SourceStageGlueArtifact
+                Configuration:
+                  RepositoryName: !Ref pStageGlueRepository
+                  BranchName: !FindInMap [pCodeCommitBranch, !Ref pEnvironment, branch]
+                  PollForSourceChanges: false
+                RunOrder: 1
               - Name: SourceDataset
                 ActionTypeId:
                   Category: Source
@@ -380,6 +418,20 @@ Resources:
                   BranchName: !FindInMap [pCodeCommitBranch, !Ref pEnvironment, branch]
                   OutputArtifactFormat: CODE_ZIP
                 RunOrder: 1
+              - Name: SourceStageLambda
+                ActionTypeId:
+                  Category: Source
+                  Owner: AWS
+                  Provider: CodeStarSourceConnection
+                  Version: "1"
+                OutputArtifacts:
+                  - Name: SourceStageLambdaArtifact
+                Configuration:
+                  ConnectionArn: !Sub "{{resolve:ssm:/SDLF/${pGitPlatform}/CodeConnection}}"
+                  FullRepositoryId: !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pStageLambdaRepository}"
+                  BranchName: !FindInMap [pCodeCommitBranch, !Ref pEnvironment, branch]
+                  OutputArtifactFormat: CODE_ZIP
+                RunOrder: 1
               - Name: SourceStageB
                 ActionTypeId:
                   Category: Source
@@ -394,6 +446,20 @@ Resources:
                   BranchName: !FindInMap [pCodeCommitBranch, !Ref pEnvironment, branch]
                   OutputArtifactFormat: CODE_ZIP
                 RunOrder: 1
+              - Name: SourceStageGlue
+                ActionTypeId:
+                  Category: Source
+                  Owner: AWS
+                  Provider: CodeStarSourceConnection
+                  Version: "1"
+                OutputArtifacts:
+                  - Name: SourceStageGlueArtifact
+                Configuration:
+                  ConnectionArn: !Sub "{{resolve:ssm:/SDLF/${pGitPlatform}/CodeConnection}}"
+                  FullRepositoryId: !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pStageGlueRepository}"
+                  BranchName: !FindInMap [pCodeCommitBranch, !Ref pEnvironment, branch]
+                  OutputArtifactFormat: CODE_ZIP
+                RunOrder: 1
               - Name: SourceDataset
                 ActionTypeId:
                   Category: Source
@@ -579,6 +645,26 @@ Resources:
                    {"name":"TEAM_NAME", "value":"${pSdlfModuleTeam}", "type":"PLAINTEXT"},
                    {"name":"MODULE_NAME", "value":"stageA", "type":"PLAINTEXT"}]
               RunOrder: 1
+            -
+              Name: BuildStageLambda
+              InputArtifacts:
+              - Name: SourceCicdArtifact
+              - Name: SourceStageLambdaArtifact
+              ActionTypeId:
+                Category: Build
+                Owner: AWS
+                Version: "1"
+                Provider: CodeBuild
+              Configuration:
+                PrimarySource: SourceStageLambdaArtifact
+                ProjectName: !Ref pBuildCloudformationModuleStage
+                EnvironmentVariables: !Sub >-
+                  [{"name":"ENVIRONMENT", "value":"${pEnvironment}", "type":"PLAINTEXT"},
+                   {"name":"DOMAIN_NAME", "value":"${pSdlfModuleDomain}", "type":"PLAINTEXT"},
+                   {"name":"DOMAIN_ACCOUNT_ID", "value":"${pChildAccountId}", "type":"PLAINTEXT"},
+                   {"name":"TEAM_NAME", "value":"${pSdlfModuleTeam}", "type":"PLAINTEXT"},
+                   {"name":"MODULE_NAME", "value":"stageLambda", "type":"PLAINTEXT"}]
+              RunOrder: 1
             -
               Name: BuildStageB
               InputArtifacts:
@@ -599,6 +685,26 @@ Resources:
                    {"name":"TEAM_NAME", "value":"${pSdlfModuleTeam}", "type":"PLAINTEXT"},
                    {"name":"MODULE_NAME", "value":"stageB", "type":"PLAINTEXT"}]
               RunOrder: 1
+            -
+              Name: BuildStageGlue
+              InputArtifacts:
+              - Name: SourceCicdArtifact
+              - Name: SourceStageGlueArtifact
+              ActionTypeId:
+                Category: Build
+                Owner: AWS
+                Version: "1"
+                Provider: CodeBuild
+              Configuration:
+                PrimarySource: SourceStageGlueArtifact
+                ProjectName: !Ref pBuildCloudformationModuleStage
+                EnvironmentVariables: !Sub >-
+                  [{"name":"ENVIRONMENT", "value":"${pEnvironment}", "type":"PLAINTEXT"},
+                   {"name":"DOMAIN_NAME", "value":"${pSdlfModuleDomain}", "type":"PLAINTEXT"},
+                   {"name":"DOMAIN_ACCOUNT_ID", "value":"${pChildAccountId}", "type":"PLAINTEXT"},
+                   {"name":"TEAM_NAME", "value":"${pSdlfModuleTeam}", "type":"PLAINTEXT"},
+                   {"name":"MODULE_NAME", "value":"stageGlue", "type":"PLAINTEXT"}]
+              RunOrder: 1
             -
               Name: BuildDataset
               InputArtifacts: