start addressing dependabot-flagged issues and add missing image buil…

…d workflow (#1824)

* start addressing dependabot-flagged issues

* create feed checker image CI and adjust other names to be consistent

* print stuff for testing help

* bump image and deploy to test

* also update reqs

.github/workflows/build-gtfs-aggregator-checker-image.yml

@@ -0,0 +1,28 @@
+name: Build and push gtfs-aggregator-checker image
+
+on:
+  push:
+    branches:
+      - 'main'
+    paths:
+      - '.github/workflows/build-gtfs-aggregator-checker.yml'
+      - 'jobs/gtfs-aggregator-checker/**'
+
+jobs:
+  build_push:
+    name: Package docker image
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: jobs/gtfs-aggregator-checker
+          push: true
+          tags: ghcr.io/${{github.repository}}/gtfs-aggregator-checker:latest

airflow/dags/check_feed_aggregators/check_aggregators.yml

@@ -1,6 +1,6 @@
 operator: operators.PodOperator
 name: 'check-aggregators'
-image: 'ghcr.io/cal-itp/data-infra/gtfs-aggregator-checker:latest'
+image: 'ghcr.io/cal-itp/data-infra/gtfs-aggregator-checker:{{ image_tag() }}'
 
 cmds:
   - python3
@@ -11,6 +11,7 @@ arguments:
   - "/secrets/agencies-data/data_agencies.yaml"
   - "--output={{get_bucket()}}/feed_aggregator_checks/dt={{execution_date.to_date_string()}}/checks.jsonl"
   - "--output-format=JSONL"
+  - "--progress"
 
 is_delete_operator_pod: true
 get_logs: true

jobs/gtfs-aggregator-checker/Dockerfile

@@ -2,8 +2,8 @@ FROM python:3.7-buster
 
 LABEL org.opencontainers.image.source https://github.com/cal-itp/data-infra
 
-RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python -
-ENV PATH="${PATH}:/root/.poetry/bin"
+RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py | python3 -
+ENV PATH="/root/.local/bin:${PATH}"
 
 RUN mkdir /app
 WORKDIR /app

jobs/gtfs-aggregator-checker/gtfs_aggregator_checker/__init__.py

@@ -3,6 +3,7 @@
 import urllib.request
 from collections import OrderedDict
 
+import typer
 import yaml
 
 from .transitfeeds import get_transitfeeds_urls
@@ -35,6 +36,7 @@ def check_feeds(yml_file=None, csv_file=None, url=None, progress=False):
             "transitland": {"status": "missing"},
         }
     elif csv_file:
+        typer.echo(f"reading urls from {csv_file}")
         with open(csv_file, "r") as f:
             urls = f.read().strip().splitlines()
             for url in urls:
@@ -44,6 +46,7 @@ def check_feeds(yml_file=None, csv_file=None, url=None, progress=False):
                     "transitland": {"status": "missing"},
                 }
     else:
+        typer.echo(f"reading urls from {yml_file}")
         with open(yml_file, "r") as f:
             agencies_obj = yaml.load(f, Loader=yaml.SafeLoader)
             for agency in agencies_obj.values():

jobs/gtfs-aggregator-checker/gtfs_aggregator_checker/transitfeeds.py

@@ -1,5 +1,6 @@
 from urllib.error import HTTPError
 
+import typer
 from bs4 import BeautifulSoup
 from tqdm import tqdm
 
@@ -18,7 +19,7 @@ def resolve_url(url):
 
 
 def get_transitfeeds_urls(progress=False):
-    print("fetching transit feeds URLs")
+    typer.echo("fetching transit feeds URLs")
 
     page_urls = []
     provider_urls = []
@@ -49,7 +50,7 @@ def get_transitfeeds_urls(progress=False):
         try:
             html = curl_cached(feed_url)
         except HTTPError:
-            print("failed to fetch:", feed_url)
+            typer.echo(f"failed to fetch: {feed_url}")
             continue
 
         soup = BeautifulSoup(html, "html.parser")

jobs/gtfs-aggregator-checker/gtfs_aggregator_checker/transitland.py

@@ -1,6 +1,7 @@
 import json
 from typing import List, Tuple
 
+import typer
 from tqdm import tqdm
 
 from .cache import curl_cached
@@ -33,7 +34,7 @@ def get_feeds(after=None):
 
 
 def get_transitland_urls(progress=False) -> List[Tuple[str, str]]:
-    print("fetching transitland URLs")
+    typer.echo("fetching transitland URLs")
     if not API_KEY:
         raise RuntimeError("TRANSITLAND_API_KEY must be set")
 
@@ -52,5 +53,5 @@ def get_transitland_urls(progress=False) -> List[Tuple[str, str]]:
         if not after:
             break
     else:
-        print("WARNING: hit loop limit for transitland")
+        typer.echo("WARNING: hit loop limit for transitland")
     return urls

jobs/gtfs-rt-parser/Dockerfile

@@ -8,8 +8,8 @@ ENV GTFS_VALIDATOR_VERSION=v1.0.0
 RUN apt-get update -y \
     && apt-get install -y python3 python3-pip
 
-RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python3 -
-ENV PATH="${PATH}:/root/.poetry/bin"
+RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py | python3 -
+ENV PATH="/root/.local/bin:${PATH}"
 
 # formerly the "1.0.0-SNAPSHOT" from S3
 COPY ./rt-validator.jar ${GTFS_RT_VALIDATOR_JAR}