Timeout before the openshift route times out

dev/eda-cr/eda-minimal-openshift-cr.yml

@@ -0,0 +1,9 @@
+---
+apiVersion: eda.ansible.com/v1alpha1
+kind: EDA
+metadata:
+  name: eda
+  namespace: eda-demo
+spec:
+  ingress_type: route
+  automation_server_url: awx.example.com 

eda-demo.yml

@@ -4,3 +4,6 @@ metadata:
   name: eda-demo
 spec:
   no_log: false
+  extra_settings:
+    - setting: EDA_ALLOW_LOCAL_RESOURCE_MANAGEMENT
+      vaue: true

roles/eda/templates/eda-api.configmap.yaml.j2

@@ -38,12 +38,29 @@ data:
                 proxy_set_header X-Forwarded-Proto $scheme;
                 proxy_set_header X-Forwarded-Port $server_port;
                 proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+                # Return 503 Service Unavailable with JSON response if gunicorn fails to respond
+                proxy_read_timeout {{ eda_nginx_read_timeout }}s;
+                error_page 504 =503 /json_503;
+                error_page 502 =503 /json_503;  # Optional, in case gunicorn is completely down    
             }
             location / {
                 root {{ static_path }};
                 try_files /index.html =404;
                 access_log /var/log/nginx/proxy_access.log;
                 error_log /var/log/nginx/proxy_error.log;
             }
+            location = /json_503 {
+                # Custom JSON response for 503 Service Unavailable
+                internal;
+                add_header Content-Type application/json;
+
+                # Check if X-Request-ID is set and include it in the response
+                if ($http_x_request_id) {
+                    return 503 '{"status": "error", "message": "Service Unavailable", "code": 503, "request_id": "$http_x_request_id"}';
+                }
+
+                # If X-Request-ID is not set, just return the basic JSON response
+                return 503 '{"status": "error", "message": "Service Unavailable", "code": 503}';
+            }    
         }
     }

roles/eda/templates/eda-api.deployment.yaml.j2

@@ -128,7 +128,7 @@ spec:
               name: '{{ ansible_operator_meta.name }}-{{ deployment_type }}-env-properties'
         env:
         - name: ANSIBLE_REVERSE_RESOURCE_SYNC
-          value: 'true'
+          value: 'false'
         - name: EDA_ACTIVATION_DB_HOST
           valueFrom:
             secretKeyRef:
@@ -200,7 +200,7 @@ spec:
         args:
         - /bin/bash
         - -c
-        - gunicorn --bind 0.0.0.0:{{ api_django_port }} --workers {{ combined_api.gunicorn_workers }} aap_eda.wsgi:application
+        - gunicorn --bind 0.0.0.0:{{ api_django_port }} --workers {{ combined_api.gunicorn_workers }} --timeout {{ gunicorn_timeout }} --graceful-timeout {{ gunicorn_timeout_grace_period }} aap_eda.wsgi:application
         envFrom:
           - configMapRef:
               name: '{{ ansible_operator_meta.name }}-{{ deployment_type }}-env-properties'

roles/eda/templates/eda-event-stream.configmap.yaml.j2

@@ -33,6 +33,23 @@ data:
                 proxy_set_header        X-Forwarded-Host $host;
                 proxy_set_header        X-Forwarded-Port $server_port;
                 proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
+                # Return 503 Service Unavailable with JSON response if gunicorn fails to respond
+                proxy_read_timeout {{ eda_nginx_read_timeout }}s;
+                error_page 504 =503 /json_503;
+                error_page 502 =503 /json_503;  # Optional, in case gunicorn is completely down    
             }
+            location = /json_503 {
+                # Custom JSON response for 503 Service Unavailable
+                internal;
+                add_header Content-Type application/json;
+
+                # Check if X-Request-ID is set and include it in the response
+                if ($http_x_request_id) {
+                    return 503 '{"status": "error", "message": "Service Unavailable", "code": 503, "request_id": "$http_x_request_id"}';
+                }
+
+                # If X-Request-ID is not set, just return the basic JSON response
+                return 503 '{"status": "error", "message": "Service Unavailable", "code": 503}';
+            }    
         }
     }

roles/eda/templates/eda-event-stream.deployment.yaml.j2

@@ -142,7 +142,7 @@ spec:
         args:
         - /bin/bash
         - -c
-        - gunicorn --bind 0.0.0.0:{{ event_stream_django_port }} --workers {{ combined_event_stream.gunicorn_workers }} aap_eda.wsgi:application
+        - gunicorn --bind 0.0.0.0:{{ event_stream_django_port }} --workers {{ combined_event_stream.gunicorn_workers }} --timeout {{ gunicorn_timeout }} --graceful-timeout {{ gunicorn_timeout_grace_period }} aap_eda.wsgi:application
         envFrom:
           - configMapRef:
               name: '{{ ansible_operator_meta.name }}-{{ deployment_type }}-env-properties'

roles/eda/templates/eda.configmap.yaml.j2

@@ -91,6 +91,10 @@ data:
                 proxy_set_header X-Forwarded-Proto $remote_scheme;
                 proxy_set_header X-Forwarded-Port $server_port;
                 proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+                # Return 503 Service Unavailable with JSON response if gunicorn fails to respond
+                proxy_read_timeout {{ eda_nginx_read_timeout }}s;
+                error_page 504 =503 /json_503;
+                error_page 502 =503 /json_503;  # Optional, in case gunicorn is completely down    
             }
 
             location ~* \.(json|woff|woff2|jpe?g|png|gif|ico|svg|css|js)$ {
@@ -105,5 +109,18 @@ data:
                 add_header Cache-Control "public, max-age=0, s-maxage=0, must-revalidate" always;
                 try_files $uri $uri/ /index.html =404;
             }
+            location = /json_503 {
+              # Custom JSON response for 503 Service Unavailable
+              internal;
+              add_header Content-Type application/json;
+
+              # Check if X-Request-ID is set and include it in the response
+              if ($http_x_request_id) {
+                  return 503 '{"status": "error", "message": "Service Unavailable", "code": 503, "request_id": "$http_x_request_id"}';
+              }
+
+              # If X-Request-ID is not set, just return the basic JSON response
+              return 503 '{"status": "error", "message": "Service Unavailable", "code": 503}';
+          }   
         }
       }

roles/eda/vars/main.yml

@@ -14,3 +14,9 @@ eda_event_stream_prefix_path: "{{ event_stream.prefix | default('/eda-event-stre
 event_stream_nginx_port: 8000
 event_stream_server_name: "{{ ansible_operator_meta.name }}-event-stream"
 event_stream_django_port: 8002
+
+# timeout defaults for nginx and gunicorn_workers
+client_request_timeout: 30
+gunicorn_timeout: '{{ (([(client_request_timeout | int), 10] | max) / 3) | int }}'
+gunicorn_timeout_grace_period: 2
+eda_nginx_read_timeout: '{{ (([(client_request_timeout | int), 10] | max) / 2) | int }}'