Timeout before the openshift route times out

config/crd/bases/galaxy_v1beta1_galaxy_crd.yaml

@@ -202,7 +202,7 @@ spec:
               gunicorn_timeout:
                 description: The timeout for the gunicorn process.
                 type: integer
-                default: 90
+                default: 10
               hostname:
                 description: The hostname of the instance
                 type: string
@@ -238,15 +238,15 @@ spec:
                 type: string
               nginx_proxy_read_timeout:
                 description: NGINX proxy read timeout
-                default: 120s
+                default: 15s
                 type: string
               nginx_proxy_connect_timeout:
                 description: NGINX proxy connect timeout
-                default: 120s
+                default: 15s
                 type: string
               nginx_proxy_send_timeout:
                 description: NGINX proxy send timeout
-                default: 120s
+                default: 15s
                 type: string
               service_annotations:
                 description: Annotations to add to the service

roles/galaxy-api/README.md

@@ -25,7 +25,7 @@ Role Variables
     * `debug`: Wether to run galaxy in debug mode.
 * `image`: The image name. Default: quay.io/ansible/galaxy-ng
 * `image_version`: The image tag. Default: main
-* `gunicorn_timeout`: The timeout for the gunicorn process. Default: 90
+* `gunicorn_timeout`: The timeout for the gunicorn process. Default: computed from `client_request_timeout`
 * `storage_type`: A string for specifying storage configuration type.
 * `file_storage_access_mode`: The access mode for the volume.
 * `file_storage_size`: The storage size.

roles/galaxy-api/defaults/main.yml

@@ -6,6 +6,9 @@ raw_spec: "{{ vars['_galaxy_ansible_com_galaxy']['spec'] }}"
 container_auth_public_key_name: 'container_auth_public_key.pem'
 container_auth_private_key_name: 'container_auth_private_key.pem'
 
+# common default values
+client_request_timeout: 30
+
 # Keycloack SSO settings
 keycloak_admin_role: hubadmin
 keycloak_port: 443
@@ -20,5 +23,7 @@ keycloak_protocol_available: false
 keycloak_port_available: false
 keycloak_realm_available: false
 
-gunicorn_timeout: 90
+# gunicorn default values
+gunicorn_timeout: "{{ (([(client_request_timeout | int), 10] | max) / 3) | int }}"
+gunicorn_timeout_grace_period: 2
 gunicorn_api_workers: 2

roles/galaxy-api/templates/galaxy-api.deployment.yaml.j2

@@ -178,6 +178,8 @@ spec:
 {% endif %}
             - name: GUNICORN_TIMEOUT
               value: "{{ gunicorn_timeout }}"
+            - name: GUNICORN_TIMEOUT_GRACE_PERIOD
+              value: "{{ gunicorn_timeout_grace_period }}"
             - name: GUNICORN_WORKERS
               value: "{{ gunicorn_api_workers }}"
 {% if signing_secret is defined %}

roles/galaxy-content/README.md

@@ -14,7 +14,7 @@ Role Variables
     * `log_level`: The desired log level.
 * `image`: The image name. Default: quay.io/ansible/galaxy-ng
 * `image_version`: The image tag. Default: main
-* `gunicorn_timeout`: The timeout for the gunicorn process. Default: 90
+* `gunicorn_timeout`: The timeout for the gunicorn process. Default: computed based on `client_request_timeout`
 
 Requirements
 ------------

roles/galaxy-content/defaults/main.yml

@@ -1,9 +1,13 @@
 ---
+# common default values
+client_request_timeout: 30
 ingress_type: none
 
 # Here we use  _galaxy_ansible_com_galaxy to get un-modified cr
 # see: https://github.com/operator-framework/operator-sdk/issues/1770
 raw_spec: "{{ vars['_galaxy_ansible_com_galaxy']['spec'] }}"
 
-gunicorn_timeout: 90
+# gunicorn default values
+gunicorn_timeout: "{{ (([(client_request_timeout | int), 10] | max) / 3) | int }}"
+gunicorn_timeout_grace_period: 2
 gunicorn_content_workers: 2

roles/galaxy-content/templates/galaxy-content.deployment.yaml.j2

@@ -207,6 +207,8 @@ spec:
 {% endif %}
             - name: GUNICORN_TIMEOUT
               value: "{{ gunicorn_timeout }}"
+            - name: GUNICORN_TIMEOUT_GRACE_PERIOD
+              value: "{{ gunicorn_timeout_grace_period }}"
             - name: GUNICORN_WORKERS
               value: "{{ gunicorn_content_workers }}"
 {% if signing_secret is defined %}

roles/galaxy-route/defaults/main.yml

@@ -2,19 +2,20 @@
 # Host to create the root with.
 # If not specific will default to <instance-name>-<namespace>-<routerCanonicalHostname>
 #
-route_host: ''
 
+# common default values
+client_request_timeout: 30
+route_host: ''
 hostname: '{{ deployment_type }}.example.com'
 
-
 # https://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size
 nginx_client_max_body_size: 10m
 
-nginx_proxy_read_timeout: 120s
-nginx_proxy_connect_timeout: 120s
-nginx_proxy_send_timeout: 120s
+nginx_proxy_read_timeout: '{{ (([(client_request_timeout | int), 10] | max) / 2) | int }}'
+nginx_proxy_connect_timeout: '{{ (([(client_request_timeout | int), 10] | max) / 2) | int }}'
+nginx_proxy_send_timeout: '{{ (([(client_request_timeout | int), 10] | max) / 2) | int }}'
 
-haproxy_timeout: 180s
+haproxy_timeout: '{{ (([(client_request_timeout | int), 10] | max) / 2) | int }}'
 
 # https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/
 # https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/

roles/galaxy-web/templates/galaxy-web.configmap.yaml.j2

@@ -102,6 +102,23 @@ data:
                 # redirects, we set the Host: header above already.
                 proxy_redirect off;
                 proxy_pass http://pulp-content;
+                error_page 504 =503 /json_503;  # Return 503 Service Unavailable with JSON response if gunicorn fails to respond
+                error_page 502 =503 /json_503;  # Optional, in case gunicorn is completely down
+            }
+
+
+            location = /json_503 {
+                # Custom JSON response for 503 Service Unavailable
+                internal;
+                add_header Content-Type application/json;
+
+                # Check if X-Request-ID is set and include it in the response
+                if ($http_x_request_id) {
+                    return 503 '{"status": "error", "message": "Service Unavailable", "code": 503, "request_id": "$http_x_request_id"}';
+                }
+
+                # If X-Request-ID is not set, just return the basic JSON response
+                return 503 '{"status": "error", "message": "Service Unavailable", "code": 503}';
             }
 
             location /{{ pulp_combined_settings.galaxy_api_path_prefix }}/pulp/api/v3/ {
@@ -112,6 +129,8 @@ data:
                 # redirects, we set the Host: header above already.
                 proxy_redirect off;
                 proxy_pass http://pulp-api;
+                error_page 504 =503 /json_503;  # Return 503 Service Unavailable with JSON response if gunicorn fails to respond
+                error_page 502 =503 /json_503;  # Optional, in case gunicorn is completely down
             }
 
             location /auth/login/ {