Add: warnings on unsupported fares fields

src/expected_fields.py

@@ -0,0 +1,95 @@
+EXPECTED_AREAS_FIELDS = [
+    'area_id',
+    'area_name',
+    'greater_area_id'
+]
+
+EXPECTED_TIMEFRAMES_FIELDS = [
+    'timeframe_id',
+    'start_time',
+    'end_time'
+]
+
+EXPECTED_RIDER_CATEGORIES_FIELDS = [
+    'rider_category_id',
+    'min_age',
+    'max_age',
+    'rider_category_name',
+    'eligibility_url'
+]
+
+EXPECTED_FARE_CONTAINERS_FIELDS = [
+    'fare_container_id',
+    'fare_container_name',
+    'minimum_initial_purchase',
+    'amount',
+    'currency',
+    'rider_category_id'
+]
+
+EXPECTED_FARE_PRODUCTS_FIELDS = [
+    'fare_product_id',
+    'fare_product_name',
+    'rider_category_id',
+    'fare_container_id',
+    'bundle_amount',
+    'duration_start',
+    'duration_amount',
+    'duration_unit',
+    'duration_type',
+    'offset_amount',
+    'offset_unit',
+    'service_id',
+    'timeframe_id',
+    'timeframe_type',
+    'cap_required',
+    'eligible_cap_id',
+    'amount',
+    'min_amount',
+    'max_amount',
+    'currency'
+]
+
+EXPECTED_FARE_LEG_RULES_FIELDS = [
+    'leg_group_id',
+    'fare_leg_name',
+    'network_id',
+    'from_area_id',
+    'contains_area_id',
+    'to_area_id',
+    'is_symmetrical',
+    'from_timeframe_id',
+    'to_timeframe_id',
+    'min_distance',
+    'max_distance',
+    'distance_type',
+    'service_id',
+    'amount',
+    'min_amount',
+    'max_amount',
+    'currency',
+    'fare_product_id',
+    'fare_container_id',
+    'rider_category_id',
+    'eligible_cap_id'
+]
+
+EXPECTED_FARE_TRANSFER_RULES_FIELDS = [
+    'from_leg_group_id',
+    'to_leg_group_id',
+    'is_symmetrical',
+    'spanning_limit',
+    'transfer_id',
+    'transfer_sequence',
+    'duration_limit',
+    'duration_limit_type',
+    'fare_transfer_type',
+    'amount',
+    'min_amount',
+    'max_amount',
+    'currency',
+    'fare_product_id',
+    'fare_container_id',
+    'rider_category_id',
+    'eligible_cap_id'
+]

src/read_fares_entities.py

@@ -9,6 +9,7 @@
 from .fare_transfer_rule_checkers import check_leg_groups, check_spans_and_transfer_ids, check_durations
 from .errors import *
 from .warnings import *
+from .expected_fields import *
 
 def areas(gtfs_root_dir, errors, warnings):
     greater_area_id_by_area_id = {}
@@ -32,7 +33,7 @@ def for_each_area(line, line_num_error_msg):
         add_warning(NO_AREAS, '', warnings)
         return []
 
-    read_csv_file(areas_path, ['area_id'], errors, for_each_area)
+    read_csv_file(areas_path, ['area_id'], EXPECTED_AREAS_FIELDS, errors, warnings, for_each_area)
 
     for area_id in greater_area_id_by_area_id:
         greater_area_id = greater_area_id_by_area_id[area_id]
@@ -100,7 +101,7 @@ def for_each_timeframe(line, line_num_error_msg):
         add_warning(NO_TIMEFRAMES, '', warnings)
         return timeframes
 
-    read_csv_file(timeframes_path, ['timeframe_id', 'start_time', 'end_time'], errors, for_each_timeframe)
+    read_csv_file(timeframes_path, ['timeframe_id', 'start_time', 'end_time'], EXPECTED_TIMEFRAMES_FIELDS, errors, warnings, for_each_timeframe)
 
     return timeframes
 
@@ -146,7 +147,7 @@ def for_each_rider_category(line, line_num_error_msg):
         add_warning(NO_RIDER_CATEGORIES, '', warnings)
         return rider_categories
 
-    read_csv_file(rider_categories_path, ['rider_category_id'], errors, for_each_rider_category)
+    read_csv_file(rider_categories_path, ['rider_category_id'], EXPECTED_RIDER_CATEGORIES_FIELDS, errors, warnings, for_each_rider_category)
 
     return rider_categories
 
@@ -186,7 +187,7 @@ def for_each_fare_container(line, line_num_error_msg):
         add_warning(NO_FARE_CONTAINERS, '', warnings)
         return rider_category_by_fare_container
 
-    read_csv_file(fare_containers_path, ['fare_container_id', 'fare_container_name'], errors, for_each_fare_container)
+    read_csv_file(fare_containers_path, ['fare_container_id', 'fare_container_name'], EXPECTED_FARE_CONTAINERS_FIELDS, errors, warnings, for_each_fare_container)
 
     return rider_category_by_fare_container
 
@@ -235,7 +236,7 @@ def for_each_fare_product(line, line_num_error_msg):
         add_warning(NO_FARE_PRODUCTS, '', warnings)
         return linked_entities_by_fare_product
 
-    read_csv_file(fare_products_path, ['fare_product_id', 'fare_product_name'], errors, for_each_fare_product)
+    read_csv_file(fare_products_path, ['fare_product_id', 'fare_product_name'], EXPECTED_FARE_PRODUCTS_FIELDS, errors, warnings, for_each_fare_product)
 
     return linked_entities_by_fare_product
 
@@ -290,7 +291,7 @@ def for_each_fare_leg_rule(line, line_num_error_msg):
         check_linked_flr_ftr_entities(fare_leg_rules_path, line, line_num_error_msg, rider_categories, rider_category_by_fare_container, linked_entities_by_fare_product, errors)
 
     if path.isfile(fare_leg_rules_path):
-        read_csv_file(fare_leg_rules_path, [], errors, for_each_fare_leg_rule)
+        read_csv_file(fare_leg_rules_path, [], EXPECTED_FARE_LEG_RULES_FIELDS, errors, warnings, for_each_fare_leg_rule)
     else:
         add_warning(NO_FARE_LEG_RULES, '', warnings)
 
@@ -334,7 +335,7 @@ def for_each_fare_transfer_rule(line, line_num_error_msg):
         check_linked_flr_ftr_entities(fare_transfer_rules_path, line, line_num_error_msg, rider_categories, rider_category_by_fare_container, linked_entities_by_fare_product, errors)
 
     if path.isfile(fare_transfer_rules_path):
-        read_csv_file(fare_transfer_rules_path, [], errors, for_each_fare_transfer_rule)
+        read_csv_file(fare_transfer_rules_path, [], EXPECTED_FARE_TRANSFER_RULES_FIELDS, errors, warnings, for_each_fare_transfer_rule)
     else:
         add_warning(NO_FARE_TRANSFER_RULES, '', warnings)
 

src/read_gtfs_entities.py

@@ -88,9 +88,9 @@ def for_each_calendar_date(line, line_num_error_msg):
         return service_ids
 
     if calendar_exists:
-        read_csv_file(calendar_path, ['service_id'], errors, for_each_calendar)
+        read_csv_file(calendar_path, ['service_id'], [], errors, warnings, for_each_calendar)
 
     if calendar_dates_exists:
-        read_csv_file(calendar_dates_path, ['service_id'], errors, for_each_calendar_date)
+        read_csv_file(calendar_dates_path, ['service_id'], [], errors, warnings, for_each_calendar_date)
 
     return service_ids

src/utils.py

@@ -2,13 +2,14 @@
 from src.errors import add_error
 from .decimals_by_currency import decimals_by_currency
 from .errors import *
+from .warnings import *
 
 def get_filename_of_path(path):
     path_split = path.split('/')
     file = path_split[len(path_split) - 1]
     return file
 
-def read_csv_file(path, required_fields, errors, func):
+def read_csv_file(path, required_fields, expected_fields, errors, warnings, func):
     filename = get_filename_of_path(path)
     with open(path, 'r') as csvfile:
         reader = csv.DictReader(csvfile)
@@ -18,7 +19,16 @@ def read_csv_file(path, required_fields, errors, func):
                 extra_info = 'field: ' + required_field
                 add_error(REQUIRED_FIELD_MISSING, '', errors, filename, extra_info)
                 return False
-
+
+        if len(expected_fields) > 0:
+            unexpected_fields = []
+            for field in reader.fieldnames:
+                if not field in expected_fields:
+                    unexpected_fields.append(field)
+            if len(unexpected_fields) > 0:
+                extra_info = '\nColumn(s): ' + str(unexpected_fields)
+                add_warning(UNEXPECTED_FIELDS, '', warnings, filename, extra_info)
+
         for line in reader:
             line_num_error_msg = '\nLine: ' + str(reader.line_num)
             func(line, line_num_error_msg)

src/warnings.py

@@ -1,4 +1,5 @@
 # generic warnings
+UNEXPECTED_FIELDS = 'A GTFS fares-v2 file has column name(s) not defined in the specification.'
 UNUSED_AREA_IDS = 'Areas defined in areas.txt are unused in other fares files.'
 UNUSED_NETWORK_IDS = 'Networks defined in routes.txt are unused in other fares files.'
 UNUSED_TIMEFRAME_IDS = 'Timeframes defined in timeframes.txt are unused in other fares files.'

tests/test_data/warnings_test_gtfs/fare_leg_rules.txt

@@ -1,3 +1,3 @@
-leg_group_id,area_id,network_id
+leg_group_id,from_area_id,network_id
 1,1,
 2,,2