feat: monitor queued jobs

setup.cfg

@@ -1,13 +1,14 @@
 [metadata]
 name = github-workflows-monitoring
-version = 0.1.3
+version = 0.2.0
 license-file = LICENSE
 
 [options]
 python_requires = >=3.8
 packages = find:
 install_requires =
         Flask>=2.2,<3
+        Flask-APScheduler==1.13.1
 
 [flake8]
 max-line-length = 120

src/app.py

@@ -4,6 +4,7 @@
 import os
 
 from flask import Flask, abort, request
+from flask_apscheduler import APScheduler
 
 
 from const import GithubHeaders, LOGGING_CONFIG
@@ -12,6 +13,8 @@
 dictConfig(LOGGING_CONFIG)
 
 app = Flask(__name__)
+scheduler = APScheduler()
+scheduler.init_app(app)
 
 # set to WARNING to disable access log
 log = logging.getLogger("werkzeug")
@@ -131,7 +134,32 @@ def process_workflow_job():
     return True
 
 
+@scheduler.task('interval', id='monitor_queued', seconds=30)
+def monitor_queued_jobs():
+    """ Return the job that has been queued and not starting for long time. """
+    app.logger.debug("Starting monitor_queued_jobs")
+    if not jobs:
+        return
+
+    old_queued_job = min(jobs.items(), key=lambda x: x[1])
+    job_id, time_start = old_queued_job
+    delay = datetime.now().timestamp() - time_start
+
+    if delay <= int(os.getenv("QUEUED_JOBS_DELAY_THRESHOLD", 150)):
+        return
+
+    context_details = {
+        "action": "monitor_queued",
+        "job_id": job_id,
+        "started_at": time_start,
+        "delay": delay,
+    }
+
+    app.logger.info(dict_to_logfmt(context_details))
+
+
 allowed_events = {"workflow_job": process_workflow_job}
+scheduler.start()
 
 
 @app.route("/github-webhook", methods=["POST"])

tests/requirements.txt

@@ -1,4 +1,5 @@
 Flask
+Flask-APScheduler==1.13.1
 pytest
 pytest-cov
 flake8