commit meesage: Updated tables-backfilling to relax the rules for certain files (#28)

Co-authored-by: Simran Mattu <[email protected]>

Author: simranmattu14

woudc_extcsv/__init__.py

@@ -221,6 +221,7 @@ def __init__(self, content, reporter=None):
 
         LOGGER.debug('Reading into csv')
         self._raw = content
+        self._raw = content.lstrip('\ufeff')
         reader = csv.reader(StringIO(self._raw))
 
         LOGGER.debug('Parsing object model')
@@ -827,10 +828,10 @@ def parse_datestamp(self, table, datestamp, line_num):
                 success = False
 
         present_year = datetime.now().year
-        if year is not None and year not in range(1940, present_year + 1):
+        if year is not None and year not in range(1924, present_year + 1):
             if not self._add_to_report(303, line_num, table=table,
                                        component='year',
-                                       lower='1940', upper='PRESENT'):
+                                       lower='1924', upper='PRESENT'):
                 success = False
         if month is not None and month not in range(1, 12 + 1):
             if not self._add_to_report(303, line_num, table=table,
@@ -1066,7 +1067,7 @@ def check_table_height(self, table, definition, num_rows):
         headerline = self.line_num(table)
         success = True
 
-        if num_rows == 0:
+        if num_rows == 0 and lower != 0:
             if is_required:
                 if not self._add_to_report(208, headerline, table=table):
                     success = False
@@ -1388,9 +1389,11 @@ def validate_dataset_tables(self):
                 success = False
 
             LOGGER.debug('Finished validating table {}'.format(table))
-
         for table in extra_tables:
-            table_type = table.rstrip('0123456789_')
+            if table.startswith("SAOZ_DATA_V"):
+                table_type = table
+            else:
+                table_type = table.rstrip('0123456789_')
             if table_type not in optional_tables and table_type != 'comments':
                 if not self._add_to_report(202, table=table, dataset=dataset):
                     success = False

woudc_extcsv/tables-backfilling.yml

@@ -204,15 +204,15 @@ Datasets:
           required_fields:
             - IntegratedO3
             - SondeTotalO3
-            - BackgroundCorrection
-            - SampleTemperatureType
           optional_fields:
             - CorrectionCode
             - NormalizationFactor
+            - BackgroundCorrection
+            - SampleTemperatureType
         PROFILE:
           rows: 1+
           occurrences: 1
-          required_fields:
+          optional_fields:
             - Duration
             - Pressure
             - O3PartialPressure
@@ -226,7 +226,6 @@ Datasets:
             - SondeCurrent
             - Latitude
             - Longitude
-          optional_fields:
             - PumpMotorCurrent
             - PumpMotorVoltage
             - Height
@@ -281,6 +280,17 @@ Datasets:
             - MotorCurrent
             - HeadPressure
             - VacuumPressure
+        AUXILIARY_DATA:
+          rows: 1
+          occurrences: 0-1
+          optional_fields:
+            - MeteoSonde
+            - ib1
+            - ib2
+            - PumpRate
+            - BackgroundCorr
+            - SampleTemperatureType
+            - MinutesGroundO3
         OZONE_REFERENCE:
           rows: 1
           occurrences: 0-1
@@ -293,6 +303,66 @@ Datasets:
             - WLCode
             - ObsType
             - UTC_Mean
+        PRELAUNCH:
+          rows: 1+
+          occurrences: 0-1
+          optional_fields:
+            - Duration
+            - Pressure
+            - O3PartialPressure
+            - Temperature
+            - WindSpeed
+            - WindDirection
+            - LevelCode
+            - GPHeight
+            - RelativeHumidity
+            - SampleTemperature
+            - SondeCurrent
+            - Latitude
+            - Longitude
+            - PumpMotorCurrent
+            - PumpMotorVoltage
+            - Height
+        PROFILE_UNCERTAINTY:
+          rows: 1+
+          occurrences: 0-1
+          optional_fields:
+            - Duration
+            - Pressure
+            - O3PartialPressure
+            - Temperature
+            - WindSpeed
+            - WindDirection
+            - LevelCode
+            - GPHeight
+            - RelativeHumidity
+            - SampleTemperature
+            - SondeCurrent
+            - Latitude
+            - Longitude
+            - PumpMotorCurrent
+            - PumpMotorVoltage
+            - Height
+        DESELECTED_DATA:
+          rows: 0+
+          occurrences: 0-1
+          optional_fields:
+            - Duration
+            - Pressure
+            - O3PartialPressure
+            - Temperature
+            - WindSpeed
+            - WindDirection
+            - LevelCode
+            - GPHeight
+            - RelativeHumidity
+            - SampleTemperature
+            - SondeCurrent
+            - Latitude
+            - Longitude
+            - PumpMotorCurrent
+            - PumpMotorVoltage
+            - Height
         data_table: PROFILE
   RocketSonde:
     "1.0":