Allow lake formation

data-collection/deploy/deploy-data-collection.yaml

@@ -17,6 +17,7 @@ Metadata:
           - ScheduleFrequent
           - CFNSourceBucket
           - DataBucketsKmsKeysArns
+          - LakeFormationEnabled
       - Label:
           default: 'Available modules'
         Parameters:
@@ -59,7 +60,9 @@ Metadata:
       CFNSourceBucket:
         default: "DO NOT CHANGE - A bucket that contains WA-Labs CloudFormation templates. Must be always 'aws-managed-cost-intelligence-dashboards'"
       DataBucketsKmsKeysArns:
-        default: ""
+        default: "Data Buckets Kms Keys Arns"
+      LakeFormationEnabled:
+        default: "Lake Formation Enabled"
       IncludeTAModule:
         default: 'Include AWS Trusted Advisor Data Collection Module'
       IncludeRightsizingModule:
@@ -167,6 +170,11 @@ Parameters:
     Type: String
     Description: "ARNs of KMS Keys for data buckets and/or Glue Catalog. Comma separated list, no spaces. Keep empty if data Buckets and Glue Catalog are not Encrypted with KMS. You can also set it to '*' to grant decrypt permission for all the keys."
     Default: ""
+  LakeFormationEnabled:
+    Type: String
+    Description: Choose 'yes' if Lake Formation permission model is in place for the account. If you are not sure, leave it as 'no'. You need to install prerequisites stack first. 
+    Default: "no"
+    AllowedValues: ["yes", "no"]
   IncludeTAModule:
     Type: String
     Description: Collects AWS Trusted Advisor recommendations data
@@ -298,6 +306,7 @@ Conditions:
     - ""
   ProdCFNTemplateUsed: !Equals [ !Ref CFNSourceBucket,  'aws-managed-cost-intelligence-dashboards' ]
   NeedDataBucketsKms: !Not [ !Equals [ !Ref DataBucketsKmsKeysArns, "" ] ]
+  LakeFormationEnabled: !Equals [ !Ref LakeFormationEnabled, "yes" ]
 
 Resources:
   S3Bucket:
@@ -713,6 +722,8 @@ Resources:
                   - glue:DeleteTable
                   - glue:TagResource
                   - glue:UpdateTable
+                  - lakeformation:AddLFTagsToResource
+                  - lakeformation:RemoveLFTagsFromResource
                 Resource:
                   - !Sub "arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:catalog"
                   - !Sub "arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:database/${DatabaseName}"
@@ -733,6 +744,8 @@ Resources:
           DATABASE_NAME: !Ref DatabaseName
           RESOURCE_PREFIX: !Ref ResourcePrefix
           BUCKET_NAME: !Sub "${DestinationBucket}${AWS::AccountId}"
+          LAKEFORMATION_TAG_VALUE: !If [LakeFormationEnabled, !ImportValue cid-LakeFormation-TagValue, '']
+          LAKEFORMATION_TAG_KEY: !If [LakeFormationEnabled, !ImportValue cid-LakeFormation-TagKey, '']
       Code:
         ZipFile: |
           import os
@@ -743,7 +756,10 @@ Resources:
 
           database_name = os.environ['DATABASE_NAME']
           resource_prefix = os.environ['RESOURCE_PREFIX']
+          lakeformation_tag_value = os.environ.get('LAKEFORMATION_TAG_VALUE')
+          lakeformation_tag_key = os.environ.get('LAKEFORMATION_TAG_KEY')
           glue_client = boto3.client('glue')
+          lakeformation_client = boto3.client('lakeformation')
 
           def lambda_handler(event, context): #pylint: disable=unused-argument
               print(json.dumps(event))
@@ -783,12 +799,27 @@ Resources:
           def create_or_update(table_input):
               try:
                   glue_client.create_table(DatabaseName=database_name, TableInput=table_input)
-                  return  'SUCCESS', 'created'
               except glue_client.exceptions.AlreadyExistsException:
                   glue_client.update_table(DatabaseName=database_name, TableInput=table_input)
-                  return  'SUCCESS', 'updated'
+              if lakeformation_tag_key and lakeformation_tag_value:
+                  try:
+                      lakeformation_client.add_lf_tags_to_resource(
+                          Resource={'Table': {'DatabaseName': database_name, 'Name': table_input['Name']}},
+                          LFTags=[{"TagKey": lakeformation_tag_key, "TagValue": lakeformation_tag_value}]
+                      )
+                  except lakeformation_client.exceptions.AlreadyExistsException as exc:
+                      print(f"Lake Formation Error: {exc}")
+              return  'SUCCESS', 'updated or created'
 
           def delete(table_input):
+              if lakeformation_tag_key and lakeformation_tag_value:
+                  try:
+                      lakeformation_client.remove_lf_tags_from_resource(
+                          Resource={'Table': {'DatabaseName': database_name, 'Name': table_input['Name']}},
+                          LFTags=[{"TagKey": lakeformation_tag_key, "TagValue": lakeformation_tag_value}]
+                      )
+                  except lakeformation_client.exceptions.AlreadyExistsException as exc:
+                      print(f"Lake Formation Error: {exc}")
               try:
                   glue_client.delete_table(DatabaseName=database_name, Name=table_input['Name'])
                   return  'SUCCESS', 'deleted'
@@ -939,14 +970,15 @@ Resources:
         MultiAccountRoleName: !Sub "${ResourcePrefix}${MultiAccountRoleName}"
         Schedule: !Ref Schedule
         ResourcePrefix: !Ref ResourcePrefix
+        LambdaManageGlueTableARN: !GetAtt LambdaManageGlueTable.Arn
         LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn
         AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}"
         CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ]
         StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath]
         StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn
         SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn
 
-  RightsizeModule:
+  RightsizeModule: # TO BE DEPRECATED
     Type: AWS::CloudFormation::Stack
     Condition: DeployRightsizingModule
     Properties:
@@ -1009,6 +1041,7 @@ Resources:
         StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath]
         StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn
         SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn
+        LambdaManageGlueTableARN: !GetAtt LambdaManageGlueTable.Arn
 
   BackupModule:
     Type: AWS::CloudFormation::Stack
@@ -1030,6 +1063,7 @@ Resources:
         StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath]
         StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn
         SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn
+        # LambdaManageGlueTableARN: !GetAtt LambdaManageGlueTable.Arn #FIXME: add table management
 
   InventoryCollectorModule:
     Type: AWS::CloudFormation::Stack
@@ -1106,7 +1140,7 @@ Resources:
         StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn
         SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn
 
-  EcsChargebackModule:
+  EcsChargebackModule: # TO DEPRECATE
     Type: AWS::CloudFormation::Stack
     Condition: DeployEcsChargebackModule
     Properties:
@@ -1132,7 +1166,7 @@ Resources:
             - !Sub "${AWS::Region}"
             - !Join [ '', !Split [ ' ', !Ref RegionsInScope  ] ] # remove spaces
 
-  RDSUsageModule:
+  RDSUsageModule: # TO DEPRECATE
     Type: AWS::CloudFormation::Stack
     Condition: DeployRDSUtilizationModule
     Properties:
@@ -1178,6 +1212,7 @@ Resources:
         StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath]
         StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn
         SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn
+        LambdaManageGlueTableARN: !GetAtt LambdaManageGlueTable.Arn #FIXME: add table management
 
   BudgetsModule:
     Type: AWS::CloudFormation::Stack
@@ -1199,8 +1234,9 @@ Resources:
         StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath]
         StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn
         SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn
+        LambdaManageGlueTableARN: !GetAtt LambdaManageGlueTable.Arn
 
-  TransitGatewayModule:
+  TransitGatewayModule: #TO DEPRECATE? 
     Type: AWS::CloudFormation::Stack
     Condition: DeployTransitGatewayModule
     Properties:
@@ -1244,6 +1280,7 @@ Resources:
         StepFunctionTemplate: !FindInMap [StepFunctionCode, standalone-v1, TemplatePath]
         StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn
         SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn
+        #LambdaManageGlueTableARN: !GetAtt LambdaManageGlueTable.Arn #TODO
 
   HealthEventsModule:
     Type: AWS::CloudFormation::Stack
@@ -1265,6 +1302,7 @@ Resources:
         StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath]
         StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn
         SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn
+        LambdaManageGlueTableARN: !GetAtt LambdaManageGlueTable.Arn
 
   LicenseManagerModule:
     Type: AWS::CloudFormation::Stack
@@ -1286,6 +1324,7 @@ Resources:
         StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath]
         StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn
         SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn
+        LambdaManageGlueTableARN: !GetAtt LambdaManageGlueTable.Arn
 
   ServiceQuotasModule:
     Type: AWS::CloudFormation::Stack
@@ -1312,6 +1351,7 @@ Resources:
             - RegionsInScopeIsEmpty
             - !Sub "${AWS::Region}"
             - !Join [ '', !Split [ ' ', !Ref RegionsInScope  ] ] # remove spaces
+        LambdaManageGlueTableARN: !GetAtt LambdaManageGlueTable.Arn
 
   QuickSightModule:
     Type: AWS::CloudFormation::Stack
@@ -1331,6 +1371,7 @@ Resources:
         StepFunctionTemplate: !FindInMap [StepFunctionCode, standalone-v1, TemplatePath]
         StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn
         SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn
+        LambdaManageGlueTableARN: !GetAtt LambdaManageGlueTable.Arn
 
   AccountCollector:
     Type: AWS::CloudFormation::Stack

data-collection/deploy/module-budgets.yaml

@@ -51,6 +51,9 @@ Parameters:
     Type: String
     Description: "ARNs of KMS Keys for data buckets and/or Glue Catalog. Comma separated list, no spaces. Keep empty if data Buckets and Glue Catalog are not Encrypted with KMS. You can also set it to '*' to grant decrypt permission for all the keys."
     Default: ""
+  LambdaManageGlueTableARN:
+    Type: String
+    Description: ARN of a Lambda for Managing GlueTable
 
 Outputs:
   StepFunctionARN:
@@ -271,6 +274,61 @@ Resources:
         S3Targets:
           - Path: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-data/"
 
+  ModuleGlueTable:
+    Type: Custom::ManageGlueTable
+    Properties:
+      ServiceToken: !Ref LambdaManageGlueTableARN
+      DatabaseName: !Ref DatabaseName
+      TableInput:
+        Name: budgets_data
+        TableType: EXTERNAL_TABLE
+        PartitionKeys:
+        - Name: payer_id
+          Type: string
+        - Name: year
+          Type: string
+        - Name: month
+          Type: string
+        StorageDescriptor:
+          Columns:
+          - Name: budgetname
+            Type: string
+          - Name: budgetlimit
+            Type: struct<Amount:string,Unit:string>
+          - Name: costfilters
+            Type: struct<Filter:array<string>>
+          - Name: costtypes
+            Type: struct<IncludeTax:boolean,IncludeSubscription:boolean,UseBlended:boolean,IncludeRefund:boolean,IncludeCredit:boolean,IncludeUpfront:boolean,IncludeRecurring:boolean,IncludeOtherSubscription:boolean,IncludeSupport:boolean,IncludeDiscount:boolean,UseAmortized:boolean>
+          - Name: timeunit
+            Type: string
+          - Name: timeperiod
+            Type: struct<Start:string,End:string>
+          - Name: calculatedspend
+            Type: struct<ActualSpend:struct<Amount:string,Unit:string>,ForecastedSpend:struct<Amount:string,Unit:string>>
+          - Name: budgettype
+            Type: string
+          - Name: lastupdatedtime
+            Type: string
+          - Name: collection_time
+            Type: string
+          - Name: account_id
+            Type: string
+          - Name: account_name
+            Type: string
+          - Name: tags
+            Type: array<struct<Key:string,Value:string>>
+          - Name: plannedbudgetlimits_flat
+            Type: array<string>
+          InputFormat: org.apache.hadoop.mapred.TextInputFormat
+          OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+          Location: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-data/"
+          SerdeInfo:
+            SerializationLibrary: org.openx.data.jsonserde.JsonSerDe
+            Parameters:
+              paths: Account_ID,Account_Name,BudgetLimit,BudgetName,BudgetType,CalculatedSpend,CostFilters,CostTypes,LastUpdatedTime,PlannedBudgetLimits_Flat,Tags,TimePeriod,TimeUnit,collection_time
+        Parameters:
+          UPDATED_BY_CRAWLER: !Sub "${ResourcePrefix}${CFDataName}-Crawler"
+
   ModuleStepFunction:
     Type: AWS::StepFunctions::StateMachine
     Properties:

data-collection/deploy/module-health-events.yaml

@@ -51,6 +51,9 @@ Parameters:
     Type: String
     Description: "ARNs of KMS Keys for data buckets and/or Glue Catalog. Comma separated list, no spaces. Keep empty if data Buckets and Glue Catalog are not Encrypted with KMS. You can also set it to '*' to grant decrypt permission for all the keys."
     Default: ""
+  LambdaManageGlueTableARN:
+    Type: String
+    Description: ARN of a Lambda for Managing GlueTable
 
 Conditions:
   NeedDataBucketsKms: !Not [ !Equals [ !Ref DataBucketsKmsKeysArns, "" ] ]
@@ -459,6 +462,83 @@ Resources:
         S3Targets:
           - Path: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-detail-data/"
 
+  ModuleGlueTable:
+    Type: Custom::ManageGlueTable
+    Properties:
+      ServiceToken: !Ref LambdaManageGlueTableARN
+      DatabaseName: !Ref DatabaseName
+      TableInput:
+        Name: health_events_detail_data
+        TableType: EXTERNAL_TABLE
+        PartitionKeys:
+        - Name: payer_id
+          Type: string
+        - Name: year
+          Type: string
+        - Name: month
+          Type: string
+        - Name: day
+          Type: string
+        StorageDescriptor:
+          Columns:
+          - Name: payer_account_id
+            Type: string
+          - Name: account_id
+            Type: string
+          - Name: event_code
+            Type: string
+          - Name: event_category
+            Type: string
+          - Name: event_scope
+            Type: string
+          - Name: status_code
+            Type: string
+          - Name: service
+            Type: string
+          - Name: region
+            Type: string
+          - Name: event_description
+            Type: string
+          - Name: affected_entity_value
+            Type: string
+          - Name: affected_entity_arn
+            Type: string
+          - Name: affected_entity_status_code
+            Type: string
+          - Name: affected_entity_last_update
+            Type: string
+          - Name: affected_entity_url
+            Type: string
+          - Name: availability_zone
+            Type: string
+          - Name: deprecated_versions
+            Type: string
+          - Name: tags
+            Type: string
+          - Name: start_time
+            Type: string
+          - Name: end_time
+            Type: string
+          - Name: last_updated_time
+            Type: string
+          - Name: event_metadata
+            Type: string
+          - Name: event_source
+            Type: string
+          - Name: event_arn
+            Type: string
+          - Name: ingestion_time
+            Type: string
+          InputFormat: org.apache.hadoop.mapred.TextInputFormat
+          OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+          Location: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-detail-data/"
+          SerdeInfo:
+            SerializationLibrary: org.openx.data.jsonserde.JsonSerDe
+            Parameters:
+              paths: account_id,affected_entity_arn,affected_entity_last_update,affected_entity_status_code,affected_entity_url,affected_entity_value,availability_zone,deprecated_versions,end_time,event_arn,event_category,event_code,event_description,event_metadata,event_scope,event_source,ingestion_time,last_updated_time,payer_account_id,region,service,start_time,status_code,tags
+        Parameters:
+          UPDATED_BY_CRAWLER: !Sub "${ResourcePrefix}${CFDataName}-Crawler"
+
   ModuleStepFunction:
     Type: AWS::StepFunctions::StateMachine
     Properties: