print stuff for testing help

cal-itp · Sep 21, 2022 · 1228708 · 1228708
1 parent dd218eb
commit 1228708
Show file tree

Hide file tree

Showing 4 changed files with 10 additions and 4 deletions.
diff --git a/airflow/dags/check_feed_aggregators/check_aggregators.yml b/airflow/dags/check_feed_aggregators/check_aggregators.yml
@@ -11,6 +11,7 @@ arguments:
   - "/secrets/agencies-data/data_agencies.yaml"
   - "--output={{get_bucket()}}/feed_aggregator_checks/dt={{execution_date.to_date_string()}}/checks.jsonl"
   - "--output-format=JSONL"
+  - "--progress"
 
 is_delete_operator_pod: true
 get_logs: true

diff --git a/jobs/gtfs-aggregator-checker/gtfs_aggregator_checker/__init__.py b/jobs/gtfs-aggregator-checker/gtfs_aggregator_checker/__init__.py
@@ -3,6 +3,7 @@
 import urllib.request
 from collections import OrderedDict
 
+import typer
 import yaml
 
 from .transitfeeds import get_transitfeeds_urls
@@ -35,6 +36,7 @@ def check_feeds(yml_file=None, csv_file=None, url=None, progress=False):
             "transitland": {"status": "missing"},
         }
     elif csv_file:
+        typer.echo(f"reading urls from {csv_file}")
         with open(csv_file, "r") as f:
             urls = f.read().strip().splitlines()
             for url in urls:
@@ -44,6 +46,7 @@ def check_feeds(yml_file=None, csv_file=None, url=None, progress=False):
                     "transitland": {"status": "missing"},
                 }
     else:
+        typer.echo(f"reading urls from {yml_file}")
         with open(yml_file, "r") as f:
             agencies_obj = yaml.load(f, Loader=yaml.SafeLoader)
             for agency in agencies_obj.values():

diff --git a/jobs/gtfs-aggregator-checker/gtfs_aggregator_checker/transitfeeds.py b/jobs/gtfs-aggregator-checker/gtfs_aggregator_checker/transitfeeds.py
@@ -1,5 +1,6 @@
 from urllib.error import HTTPError
 
+import typer
 from bs4 import BeautifulSoup
 from tqdm import tqdm
 
@@ -18,7 +19,7 @@ def resolve_url(url):
 
 
 def get_transitfeeds_urls(progress=False):
-    print("fetching transit feeds URLs")
+    typer.echo("fetching transit feeds URLs")
 
     page_urls = []
     provider_urls = []
@@ -49,7 +50,7 @@ def get_transitfeeds_urls(progress=False):
         try:
             html = curl_cached(feed_url)
         except HTTPError:
-            print("failed to fetch:", feed_url)
+            typer.echo(f"failed to fetch: {feed_url}")
             continue
 
         soup = BeautifulSoup(html, "html.parser")

diff --git a/jobs/gtfs-aggregator-checker/gtfs_aggregator_checker/transitland.py b/jobs/gtfs-aggregator-checker/gtfs_aggregator_checker/transitland.py
@@ -1,6 +1,7 @@
 import json
 from typing import List, Tuple
 
+import typer
 from tqdm import tqdm
 
 from .cache import curl_cached
@@ -33,7 +34,7 @@ def get_feeds(after=None):
 
 
 def get_transitland_urls(progress=False) -> List[Tuple[str, str]]:
-    print("fetching transitland URLs")
+    typer.echo("fetching transitland URLs")
     if not API_KEY:
         raise RuntimeError("TRANSITLAND_API_KEY must be set")
 
@@ -52,5 +53,5 @@ def get_transitland_urls(progress=False) -> List[Tuple[str, str]]:
         if not after:
             break
     else:
-        print("WARNING: hit loop limit for transitland")
+        typer.echo("WARNING: hit loop limit for transitland")
     return urls