strip dbt_ out of item names for AGO searching.

CityOfPhiladelphia · Apr 9, 2024 · 3557358 · 3557358
1 parent 65a2862
commit 3557358
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 3 deletions.
diff --git a/databridge_etl_tools/ago/ago.py b/databridge_etl_tools/ago/ago.py
@@ -50,7 +50,11 @@ def __init__(self,
         self.ago_org_url = ago_org_url
         self.ago_user = ago_user
         self.ago_password = ago_pw
-        self.item_name = ago_item_name
+        if ago_item_name.rstrip().startswith('dbt_'):
+            self.logger.info('stripping "dbt_" prefix from item name..')
+            self.item_name = ago_item_name.rstrip().strip('dbt_')
+        else:
+            self.item_name = ago_item_name
         self.s3_bucket = s3_bucket
         self.s3_key = s3_key
         # Our org id, publicly viewable so fine to hardcode.
@@ -1416,7 +1420,8 @@ def post_index(field, is_unique):
             url = f'https://services.arcgis.com/{self.ago_org_id}/arcgis/rest/admin/services/{self.item_name}/FeatureServer/0/addToDefinition'
 
             headers = { 'Content-Type': 'application/x-www-form-urlencoded' }
-            self.logger.info(f'\nPosting index for {field}...')
+            self.logger.info(f'\nPosting index for {field} against {url}...')
+            self.logger.info(f'Data passed: {jsonData}')
             self.logger.info(jsonData)
             r = requests.post(f'{url}?token={ago_token}', data = {'f': 'json', 'addToDefinition': jsonData }, headers=headers, timeout=360)
 
@@ -1427,6 +1432,9 @@ def post_index(field, is_unique):
                 possible your index was actually rejected. ESRI just doesnt code in proper errors).
                 ''')
             # Seen this error before that prevents an index from being added. Sleep and try to add again.
+            elif 'Invalid URL' in r.text:
+                print('Invalid URL error, does your map name differ from the table name?? Please fix if so.')
+                sys.exit(1)
             elif 'Operation failed. The index entry of length' in r.text:
                 self.logger.info('Got a retriable error, retrying in 10 minutes...')
                 sleep(200)
@@ -1480,8 +1488,9 @@ def post_index(field, is_unique):
         headers = {'Content-Type': 'application/x-www-form-urlencoded'}
         r = requests.get(f'{check_url}&token={ago_token}', headers=headers, timeout=3600)
         data = r.json()
-        # Pull indexes out of the fuater server json definition
+        # Pull indexes out of the feature server json definition
         ago_indexes = data['indexes']
+
         # Get just the names of the indexes
         ago_indexes_list = [ x['name'] for x in ago_indexes ]
 

diff --git a/databridge_etl_tools/db2/db2.py b/databridge_etl_tools/db2/db2.py
@@ -506,10 +506,13 @@ def copy_to_enterprise(self):
                 UPDATE {self.enterprise_schema}.i{reg_id} SET base_id={row_count + 1}, last_id={row_count} WHERE id_type = 2;
                 '''
             try:
+                self.logger.info(update_stmt_1)
                 self.pg_cursor.execute(update_stmt_1)
                 # Remove locks just before we perform a rename.
                 self.remove_locks(self.enterprise_dataset_name, self.enterprise_schema)
+                self.logger.info(update_stmt_2)
                 self.pg_cursor.execute(update_stmt_2)
+                self.logger.info(update_stmt_3)
                 self.pg_cursor.execute(update_stmt_3)
                 self.pg_cursor.execute('COMMIT')
                 #####################