airtable flag for adding objectid

databridge_etl_tools/airtable/airtable.py

@@ -11,16 +11,18 @@
 
 
 class Airtable():
-    def __init__(self, app_id:str, api_key:str, table_name:str, s3_bucket:str, s3_key:str, get_fields=None):
+    def __init__(self, app_id:str, api_key:str, table_name:str, s3_bucket:str, s3_key:str, add_objectid:bool, get_fields=None):
         self.app_id = app_id
         self.api_key = api_key
         self.table_name = table_name
         self.s3_bucket = s3_bucket
         self.s3_key = s3_key
         self.offset = None
         self.rows_per_page = 1000
+        self.add_objectid = add_objectid
         self.get_fields = get_fields
         self.csv_path = f'/tmp/{self.table_name}.csv'
+        self.counter = 0
 
     def get_fieldnames(self):
         '''Get field names with an initial request, but if get_fields was passed
@@ -55,6 +57,9 @@ def get_fieldnames(self):
         # Enforce lower-case headers because
         # we're going into postgres and don't want mixed case.
         fieldnames = [x.lower() for x in fieldnames]
+
+        if self.add_objectid:
+            fieldnames.insert(0, 'objectid')
 
         return fieldnames
 
@@ -93,9 +98,13 @@ def process_row(self, row: Dict) -> Dict:
                 row[key] = json.dumps(value)
 
             # lowercase all fields, we don't want mixed case in postgres
-            row_lower = {k.lower(): v for k, v in row.items()}
+            row = {k.lower(): v for k, v in row.items()}
+
+        if self.add_objectid:
+            self.counter += 1
+            row['objectid'] = self.counter
 
-        return row_lower
+        return row
 
     def load_to_s3(self):
         s3 = boto3.resource('s3')

databridge_etl_tools/airtable/airtable_commands.py

@@ -10,6 +10,7 @@
 @click.option('--table_name', required=True)
 @click.option('--s3_bucket', required=True)
 @click.option('--s3_key', required=True)
+@click.option('--add_objectid', required=False, is_flag=True, help='Adds an objectid to the CSV')
 @click.option('--get_fields', required=False, help='Fields you want to extract, comma separated string.')
 def airtable(ctx, **kwargs):
     ctx.obj = Airtable(**kwargs)