glue crawlers

awslabs · Mar 26, 2024 · 612e154 · 612e154
1 parent 9547fc6
commit 612e154
Show file tree

Hide file tree

Showing 6 changed files with 12 additions and 8 deletions.
diff --git a/sdlf-cicd/template-cicd-domain-team-role.yaml b/sdlf-cicd/template-cicd-domain-team-role.yaml
@@ -258,7 +258,7 @@ Resources:
               - glue:StopCrawler
               - glue:TagResource
               - glue:UntagResource
-            Resource: !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:crawler/sdlf-${pTeamName}-*
+            Resource: !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:crawler/*
           - Effect: Allow
             Action:
               - glue:CreateDatabase

diff --git a/sdlf-dataset/template.yaml b/sdlf-dataset/template.yaml
@@ -75,7 +75,6 @@ Resources:
       Role: !Sub "{{resolve:ssm:/SDLF2/IAM/${pTeamName}/CrawlerRoleArn}}"
       CrawlerSecurityConfiguration: !Sub "{{resolve:ssm:/SDLF2/Glue/${pTeamName}/SecurityConfigurationId}}"
       DatabaseName: !Ref rGlueDataCatalog
-      Name: !Sub sdlf-${pTeamName}-${pDatasetName}-post-stage-crawler
       Targets:
         S3Targets:
           - Path: !Sub s3://${pStageBucket}/post-stage/${pTeamName}/${pDatasetName}

diff --git a/sdlf-stageB/lambda/stage-b-fetch-metadata/src/lambda_function.py b/sdlf-stageB/lambda/stage-b-fetch-metadata/src/lambda_function.py
@@ -1,10 +1,14 @@
+import os
+
+import boto3
 from datalake_library import octagon
 from datalake_library.commons import init_logger
 from datalake_library.configuration.resource_configs import DynamoConfiguration
 from datalake_library.interfaces.dynamo_interface import DynamoInterface
 
 logger = init_logger(__name__)
-
+ssm_endpoint_url = "https://ssm." + os.getenv("AWS_REGION") + ".amazonaws.com"
+ssm = boto3.client("ssm", endpoint_url=ssm_endpoint_url)
 
 def get_glue_transform_details(bucket, team, dataset, pipeline, stage):
     dynamo_config = DynamoConfiguration()
@@ -67,7 +71,8 @@ def lambda_handler(event, context):
         event["body"]["glue"] = get_glue_transform_details(
             bucket, team, dataset, pipeline, stage
         )  # custom user code called
-        event["body"]["glue"]["crawler_name"] = "-".join(["sdlf", team, dataset, "post-stage-crawler"])
+        event["body"]["glue"]["crawler_name"] = ssm.get_parameter(Name=f"/SDLF2/Glue/{team}/{dataset}/GlueCrawler")["Parameter"]["Value"]
+        "-".join(["sdlf", team, dataset, "post-stage-crawler"])
         event["body"]["peh_id"] = peh_id
         octagon_client.update_pipeline_execution(
             status="{} {} Processing".format(stage, component), component=component

diff --git a/sdlf-stageB/template.yaml b/sdlf-stageB/template.yaml
@@ -584,7 +584,7 @@ Resources:
                   - glue:StartCrawler
                   - glue:GetCrawler
                 Resource:
-                  - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:crawler/sdlf-${pTeamName}-*
+                  - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:crawler/*
               - Effect: Allow
                 Action:
                   - xray:PutTraceSegments # W11 exception

diff --git a/sdlf-team/template.yaml b/sdlf-team/template.yaml
@@ -336,7 +336,7 @@ Resources:
     Properties:
       Name: !Sub /SDLF2/Glue/${pTeamName}/SecurityConfigurationId
       Type: String
-      Value: !Sub sdlf-${pTeamName}-glue-security-config # unfortunately AWS::Glue::SecurityConfiguration doesn't provide any return value
+      Value: !Sub sdlf2-${pTeamName}-glue-security-config # unfortunately AWS::Glue::SecurityConfiguration doesn't provide any return value
       Description: !Sub Name of the ${pTeamName} Glue security configuration
   #### END KMS STACK
 
@@ -481,7 +481,7 @@ Resources:
               - glue:StartCrawler
               - glue:GetCrawler
             Resource:
-              - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:crawler/sdlf-${pTeamName}-*
+              - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:crawler/*
           - Effect: Allow
             Action:
               - glue:GetTable

diff --git a/sdlf-utils/workshop-examples/legislators/scripts/legislators-glue-job.yaml b/sdlf-utils/workshop-examples/legislators/scripts/legislators-glue-job.yaml
@@ -80,5 +80,5 @@ Resources:
       MaxCapacity: 2.0
       GlueVersion: "4.0"
       Name: !Sub sdlf-${pTeamName}-${pDatasetName}-glue-job
-      SecurityConfiguration: !Sub "{{resolve:ssm:/SDLF/Glue/${pTeamName}/SecurityConfigurationId:1}}"
+      SecurityConfiguration: !Sub "{{resolve:ssm:/SDLF2/Glue/${pTeamName}/SecurityConfigurationId:1}}"
       Role: !Ref rGlueRole