[sdlf-dataset] support for multiple instances

also add a Deployment Instance output
awslabs · Nov 18, 2024 · 86e808d · 86e808d
1 parent e785fa5
commit 86e808d
Showing 1 changed file with 46 additions and 35 deletions.
diff --git a/sdlf-dataset/src/dataset.yaml b/sdlf-dataset/src/dataset.yaml
@@ -5,33 +5,41 @@ Parameters:
   pPipelineReference:
     Type: String
     Default: none
+  pDeploymentInstance:
+    Type: String
+    Description: A string uniquely identifying this deployment in this AWS account
+    Default: dev
   pOrg:
     Description: Name of the organization owning the datalake
     Type: String
-    Default: "{{resolve:ssm:/SDLF/Misc/pOrg}}"
+    Default: "{{resolve:ssm:/sdlf/storage/rOrganization/dev}}"
   pDomain:
     Description: Data domain name
     Type: String
-    Default: "{{resolve:ssm:/SDLF/Misc/pDomain}}"
+    Default: "{{resolve:ssm:/sdlf/storage/rDomain/dev}}"
   pRawBucket:
     Description: The raw bucket for the solution
     Type: String
-    Default: "{{resolve:ssm:/SDLF/S3/RawBucket}}"
+    Default: "{{resolve:ssm:/sdlf/storage/rRawBucket/dev}}"
   pStageBucket:
     Description: The stage bucket for the solution
     Type: String
-    Default: "{{resolve:ssm:/SDLF/S3/StageBucket}}"
+    Default: "{{resolve:ssm:/sdlf/storage/rStageBucket/dev}}"
   pAnalyticsBucket:
     Description: The analytics bucket for the solution
     Type: String
-    Default: "{{resolve:ssm:/SDLF/S3/AnalyticsBucket}}"
+    Default: "{{resolve:ssm:/sdlf/storage/rAnalyticsBucket/dev}}"
   pArtifactsBucket:
-    Description: The artifacts bucket used by CodeBuild and CodePipeline
+    Description: S3 bucket used to store artifacts (from CICD or generated by data pipelines)
+    Type: String
+    Default: "{{resolve:ssm:/sdlf/storage/rArtifactsBucket/dev}}"
+  pBucketKey:
+    Description: KMS key set as bucket key for the solutions' buckets
     Type: String
-    Default: "{{resolve:ssm:/SDLF/S3/ArtifactsBucket}}"
+    Default: "{{resolve:ssm:/sdlf/storage/rKMSKey/dev}}"
   pLakeFormationDataAccessRole:
     Type: String
-    Default: "{{resolve:ssm:/SDLF/IAM/LakeFormationDataAccessRoleArn}}"
+    Default: "{{resolve:ssm:/sdlf/storage/rLakeFormationDataAccessRoleArn/dev}}"
   pDatasetName:
     Description: The name of the dataset (all lowercase, no symbols or spaces)
     Type: String
@@ -124,13 +132,13 @@ Resources:
     UpdateReplacePolicy: Retain
     DeletionPolicy: RetainExceptOnCreate
     Properties:
-      AliasName: !Sub alias/sdlf-${pDatasetName}-kms-infra-key
+      AliasName: !Sub alias/sdlf-kms-infra-key-${pDeploymentInstance}
       TargetKeyId: !Ref rKMSInfraKey
 
   rKMSInfraKeySsm:
     Type: AWS::SSM::Parameter
     Properties:
-      Name: /sdlf/dataset/rKMSInfraKey
+      Name: !Sub /sdlf/dataset/rKMSInfraKey/${pDeploymentInstance}
       Type: String
       Value: !GetAtt rKMSInfraKey.Arn
       Description: !Sub ${pDatasetName} KMS infrastructure key ARN
@@ -171,14 +179,14 @@ Resources:
     UpdateReplacePolicy: Retain
     DeletionPolicy: RetainExceptOnCreate
     Properties:
-      AliasName: !Sub alias/sdlf-${pDatasetName}-kms-data-key
+      AliasName: !Sub alias/sdlf-kms-data-key-${pDeploymentInstance}
       TargetKeyId: !Ref rKMSDataKey
 
   rKMSDataKeySsm:
     Type: AWS::SSM::Parameter
     Condition: IsS3Prefix
     Properties:
-      Name: !Sub /sdlf/dataset/rKMSDataKey
+      Name: !Sub /sdlf/dataset/rKMSDataKey/${pDeploymentInstance}
       Type: String
       Value: !GetAtt rKMSDataKey.Arn
       Description: !Sub ${pDatasetName} KMS data key ARN
@@ -187,7 +195,7 @@ Resources:
   rGlueSecurityConfiguration:
     Type: AWS::Glue::SecurityConfiguration
     Properties:
-      Name: !Sub sdlf-${pDatasetName}-glue-security-config
+      Name: !Sub sdlf-${pDatasetName}-glue-security-config # TODO ${pDeploymentInstance}
       EncryptionConfiguration:
         CloudWatchEncryption:
           CloudWatchEncryptionMode: SSE-KMS
@@ -200,20 +208,20 @@ Resources:
             KmsKeyArn: !If
               - IsS3Prefix
               - !GetAtt rKMSDataKey.Arn
-              - "{{resolve:ssm:/sdlf/storage/rKMSKey}}"
+              - !Ref pBucketKey
 
   rGlueSecurityConfigurationSsm:
     Type: AWS::SSM::Parameter
     Properties:
-      Name: !Sub /sdlf/dataset/rGlueSecurityConfiguration
+      Name: !Sub /sdlf/dataset/rGlueSecurityConfiguration/${pDeploymentInstance}
       Type: String
       Value: !Sub sdlf-${pDatasetName}-glue-security-config # unfortunately AWS::Glue::SecurityConfiguration doesn't provide any return value
       Description: !Sub Name of the ${pDatasetName} Glue security configuration
 
   rEMRSecurityConfiguration:
     Type: AWS::EMR::SecurityConfiguration
     Properties:
-      Name: !Sub sdlf-${pDatasetName}-emr-security-config
+      Name: !Sub sdlf-${pDatasetName}-emr-security-config # TODO ${pDeploymentInstance}
       SecurityConfiguration: !If
         - IsS3Prefix
         - !Sub |
@@ -246,11 +254,11 @@ Resources:
                 "AtRestEncryptionConfiguration" : {
                   "S3EncryptionConfiguration" : {
                     "EncryptionMode" : "SSE-KMS",
-                    "AwsKmsKey": "{{resolve:ssm:/sdlf/storage/rKMSKey}}"
+                    "AwsKmsKey": "${pBucketKey}"
                   },
                   "LocalDiskEncryptionConfiguration" : {
                     "EncryptionKeyProviderType" : "AwsKms",
-                    "AwsKmsKey" : "{{resolve:ssm:/sdlf/storage/rKMSKey}}",
+                    "AwsKmsKey" : "${pBucketKey}",
                     "EnableEbsEncryption" : true
                   }
                 }
@@ -264,7 +272,7 @@ Resources:
   rEMRSecurityConfigurationSsm:
     Type: AWS::SSM::Parameter
     Properties:
-      Name: !Sub /sdlf/dataset/rEMRSecurityConfiguration
+      Name: !Sub /sdlf/dataset/rEMRSecurityConfiguration/${pDeploymentInstance}
       Type: String
       Value: !Ref rEMRSecurityConfiguration
       Description: !Sub Name of the ${pDatasetName} EMR security configuration
@@ -338,9 +346,9 @@ Resources:
                   - IsS3Prefix
                   - - !GetAtt rKMSInfraKey.Arn
                     - !GetAtt rKMSDataKey.Arn
-                    - "{{resolve:ssm:/sdlf/storage/rKMSKey}}"
+                    - !Ref pBucketKey
                   - - !GetAtt rKMSInfraKey.Arn
-                    - "{{resolve:ssm:/sdlf/storage/rKMSKey}}"
+                    - !Ref pBucketKey
               - Effect: Allow
                 Action:
                   - lakeformation:GetDataAccess # W11 exception
@@ -357,7 +365,7 @@ Resources:
   rDatalakeCrawlerRoleArnSsm:
     Type: AWS::SSM::Parameter
     Properties:
-      Name: !Sub /sdlf/dataset/rDatalakeCrawlerRole
+      Name: !Sub /sdlf/dataset/rDatalakeCrawlerRole/${pDeploymentInstance}
       Type: String
       Value: !GetAtt rDatalakeCrawlerRole.Arn
       Description: ARN of the Crawler role
@@ -381,7 +389,7 @@ Resources:
   rRawGlueDataCatalogSsm:
     Type: AWS::SSM::Parameter
     Properties:
-      Name: /sdlf/dataset/rRawGlueDataCatalog
+      Name: !Sub /sdlf/dataset/rRawGlueDataCatalog/${pDeploymentInstance}
       Type: String
       Value: !Ref rRawGlueDataCatalog
       Description: !Sub "${pDatasetName} raw metadata catalog"
@@ -437,7 +445,7 @@ Resources:
   rRawGlueCrawlerSsm:
     Type: AWS::SSM::Parameter
     Properties:
-      Name: !Sub /sdlf/dataset/rRawGlueCrawler
+      Name: !Sub /sdlf/dataset/rRawGlueCrawler/${pDeploymentInstance}
       Type: String
       Value: !Ref rRawGlueCrawler
       Description: !Sub "${pDatasetName} Raw Glue crawler"
@@ -453,7 +461,7 @@ Resources:
   rStageGlueDataCatalogSsm:
     Type: AWS::SSM::Parameter
     Properties:
-      Name: /sdlf/dataset/rStageGlueDataCatalog
+      Name: !Sub /sdlf/dataset/rStageGlueDataCatalog/${pDeploymentInstance}
       Type: String
       Value: !Ref rStageGlueDataCatalog
       Description: !Sub "${pDatasetName} stage metadata catalog"
@@ -509,7 +517,7 @@ Resources:
   rStageGlueCrawlerSsm:
     Type: AWS::SSM::Parameter
     Properties:
-      Name: !Sub /sdlf/dataset/rStageGlueCrawler
+      Name: !Sub /sdlf/dataset/rStageGlueCrawler/${pDeploymentInstance}
       Type: String
       Value: !Ref rStageGlueCrawler
       Description: !Sub "${pDatasetName} Stage Glue crawler"
@@ -525,7 +533,7 @@ Resources:
   rAnalyticsGlueDataCatalogSsm:
     Type: AWS::SSM::Parameter
     Properties:
-      Name: /sdlf/dataset/rAnalyticsGlueDataCatalog
+      Name: !Sub /sdlf/dataset/rAnalyticsGlueDataCatalog/${pDeploymentInstance}
       Type: String
       Value: !Ref rAnalyticsGlueDataCatalog
       Description: !Sub "${pDatasetName} analytics metadata catalog"
@@ -581,7 +589,7 @@ Resources:
   rAnalyticsGlueCrawlerSsm:
     Type: AWS::SSM::Parameter
     Properties:
-      Name: !Sub /sdlf/dataset/rAnalyticsGlueCrawler
+      Name: !Sub /sdlf/dataset/rAnalyticsGlueCrawler/${pDeploymentInstance}
       Type: String
       Value: !Ref rAnalyticsGlueCrawler
       Description: !Sub "${pDatasetName} Analytics Glue crawler"
@@ -617,7 +625,7 @@ Resources:
   rEventBusSsm:
     Type: AWS::SSM::Parameter
     Properties:
-      Name: !Sub /sdlf/dataset/rEventBus
+      Name: !Sub /sdlf/dataset/rEventBus/${pDeploymentInstance}
       Type: String
       Value: !Ref rEventBus
       Description: !Sub Name of the ${pDatasetName} event bus
@@ -630,7 +638,7 @@ Resources:
   rScheduleGroupSsm:
     Type: AWS::SSM::Parameter
     Properties:
-      Name: !Sub /sdlf/dataset/rScheduleGroup
+      Name: !Sub /sdlf/dataset/rScheduleGroup/${pDeploymentInstance}
       Type: String
       Value: !Ref rScheduleGroup
       Description: !Sub Name of the ${pDatasetName} schedule group
@@ -726,9 +734,9 @@ Resources:
               - IsS3Prefix
               - - !GetAtt rKMSInfraKey.Arn
                 - !GetAtt rKMSDataKey.Arn
-                - "{{resolve:ssm:/sdlf/storage/rKMSKey}}"
+                - !Ref pBucketKey
               - - !GetAtt rKMSInfraKey.Arn
-                - "{{resolve:ssm:/sdlf/storage/rKMSKey}}"
+                - !Ref pBucketKey
           - Effect: Allow
             Action:
               - ssm:GetParameter
@@ -876,7 +884,7 @@ Resources:
   rIamManagedPolicySsm:
     Type: AWS::SSM::Parameter
     Properties:
-      Name: !Sub /sdlf/dataset/rIamManagedPolicy
+      Name: !Sub /sdlf/dataset/rIamManagedPolicy/${pDeploymentInstance}
       Type: String
       Value: !Ref rIamManagedPolicy
       Description: The permissions boundary IAM Managed policy for the team
@@ -975,7 +983,7 @@ Resources:
   rDynamoPipelineExecutionHistorySsm:
     Type: AWS::SSM::Parameter
     Properties:
-      Name: !Sub /sdlf/dataset/rDynamoPipelineExecutionHistory
+      Name: !Sub /sdlf/dataset/rDynamoPipelineExecutionHistory/${pDeploymentInstance}
       Type: String
       Value: !Ref rDynamoPipelineExecutionHistory
       Description: Name of the DynamoDB used to store pipeline execution metadata
@@ -1009,7 +1017,7 @@ Resources:
   rDynamoManifestsSsm:
     Type: AWS::SSM::Parameter
     Properties:
-      Name: !Sub /sdlf/dataset/rDynamoManifests
+      Name: !Sub /sdlf/dataset/rDynamoManifests/${pDeploymentInstance}
       Type: String
       Value: !Ref rDynamoManifests
       Description: Name of the DynamoDB used to store manifest process metadata
@@ -1018,3 +1026,6 @@ Outputs:
   oPipelineReference:
     Description: CodePipeline reference this stack has been deployed with
     Value: !Ref pPipelineReference
+  oDeploymentInstance:
+    Description: A string uniquely identifying this deployment in this AWS account
+    Value: !Ref pDeploymentInstance