diff --git a/config/crd/bases/galaxy_v1beta1_galaxy_crd.yaml b/config/crd/bases/galaxy_v1beta1_galaxy_crd.yaml index bb6a426a..d5175ff8 100644 --- a/config/crd/bases/galaxy_v1beta1_galaxy_crd.yaml +++ b/config/crd/bases/galaxy_v1beta1_galaxy_crd.yaml @@ -202,7 +202,7 @@ spec: gunicorn_timeout: description: The timeout for the gunicorn process. type: integer - default: 90 + default: 10 hostname: description: The hostname of the instance type: string @@ -238,15 +238,15 @@ spec: type: string nginx_proxy_read_timeout: description: NGINX proxy read timeout - default: 120s + default: 15s type: string nginx_proxy_connect_timeout: description: NGINX proxy connect timeout - default: 120s + default: 15s type: string nginx_proxy_send_timeout: description: NGINX proxy send timeout - default: 120s + default: 15s type: string service_annotations: description: Annotations to add to the service diff --git a/roles/galaxy-api/README.md b/roles/galaxy-api/README.md index d1a1adab..0822c981 100644 --- a/roles/galaxy-api/README.md +++ b/roles/galaxy-api/README.md @@ -25,7 +25,7 @@ Role Variables * `debug`: Wether to run galaxy in debug mode. * `image`: The image name. Default: quay.io/ansible/galaxy-ng * `image_version`: The image tag. Default: main -* `gunicorn_timeout`: The timeout for the gunicorn process. Default: 90 +* `gunicorn_timeout`: The timeout for the gunicorn process. Default: computed from `client_request_timeout` * `storage_type`: A string for specifying storage configuration type. * `file_storage_access_mode`: The access mode for the volume. * `file_storage_size`: The storage size. diff --git a/roles/galaxy-api/defaults/main.yml b/roles/galaxy-api/defaults/main.yml index a1545028..831eb030 100644 --- a/roles/galaxy-api/defaults/main.yml +++ b/roles/galaxy-api/defaults/main.yml @@ -6,6 +6,9 @@ raw_spec: "{{ vars['_galaxy_ansible_com_galaxy']['spec'] }}" container_auth_public_key_name: 'container_auth_public_key.pem' container_auth_private_key_name: 'container_auth_private_key.pem' +# common default values +client_request_timeout: 30 + # Keycloack SSO settings keycloak_admin_role: hubadmin keycloak_port: 443 @@ -20,5 +23,7 @@ keycloak_protocol_available: false keycloak_port_available: false keycloak_realm_available: false -gunicorn_timeout: 90 +# gunicorn default values +gunicorn_timeout: "{{ (([(client_request_timeout | int), 10] | max) / 3) | int }}" +gunicorn_timeout_grace_period: 2 gunicorn_api_workers: 2 diff --git a/roles/galaxy-api/templates/galaxy-api.deployment.yaml.j2 b/roles/galaxy-api/templates/galaxy-api.deployment.yaml.j2 index 96883edf..a48ac177 100644 --- a/roles/galaxy-api/templates/galaxy-api.deployment.yaml.j2 +++ b/roles/galaxy-api/templates/galaxy-api.deployment.yaml.j2 @@ -178,6 +178,8 @@ spec: {% endif %} - name: GUNICORN_TIMEOUT value: "{{ gunicorn_timeout }}" + - name: GUNICORN_TIMEOUT_GRACE_PERIOD + value: "{{ gunicorn_timeout_grace_period }}" - name: GUNICORN_WORKERS value: "{{ gunicorn_api_workers }}" {% if signing_secret is defined %} diff --git a/roles/galaxy-content/README.md b/roles/galaxy-content/README.md index b116913e..f4e42f38 100644 --- a/roles/galaxy-content/README.md +++ b/roles/galaxy-content/README.md @@ -14,7 +14,7 @@ Role Variables * `log_level`: The desired log level. * `image`: The image name. Default: quay.io/ansible/galaxy-ng * `image_version`: The image tag. Default: main -* `gunicorn_timeout`: The timeout for the gunicorn process. Default: 90 +* `gunicorn_timeout`: The timeout for the gunicorn process. Default: computed based on `client_request_timeout` Requirements ------------ diff --git a/roles/galaxy-content/defaults/main.yml b/roles/galaxy-content/defaults/main.yml index d8e09b23..4f2f3a66 100644 --- a/roles/galaxy-content/defaults/main.yml +++ b/roles/galaxy-content/defaults/main.yml @@ -1,9 +1,13 @@ --- +# common default values +client_request_timeout: 30 ingress_type: none # Here we use _galaxy_ansible_com_galaxy to get un-modified cr # see: https://github.com/operator-framework/operator-sdk/issues/1770 raw_spec: "{{ vars['_galaxy_ansible_com_galaxy']['spec'] }}" -gunicorn_timeout: 90 +# gunicorn default values +gunicorn_timeout: "{{ (([(client_request_timeout | int), 10] | max) / 3) | int }}" +gunicorn_timeout_grace_period: 2 gunicorn_content_workers: 2 diff --git a/roles/galaxy-content/templates/galaxy-content.deployment.yaml.j2 b/roles/galaxy-content/templates/galaxy-content.deployment.yaml.j2 index 628fc891..e81090d3 100644 --- a/roles/galaxy-content/templates/galaxy-content.deployment.yaml.j2 +++ b/roles/galaxy-content/templates/galaxy-content.deployment.yaml.j2 @@ -207,6 +207,8 @@ spec: {% endif %} - name: GUNICORN_TIMEOUT value: "{{ gunicorn_timeout }}" + - name: GUNICORN_TIMEOUT_GRACE_PERIOD + value: "{{ gunicorn_timeout_grace_period }}" - name: GUNICORN_WORKERS value: "{{ gunicorn_content_workers }}" {% if signing_secret is defined %} diff --git a/roles/galaxy-route/defaults/main.yml b/roles/galaxy-route/defaults/main.yml index 0bbdb109..c0fb1c05 100644 --- a/roles/galaxy-route/defaults/main.yml +++ b/roles/galaxy-route/defaults/main.yml @@ -2,19 +2,20 @@ # Host to create the root with. # If not specific will default to -- # -route_host: '' +# common default values +client_request_timeout: 30 +route_host: '' hostname: '{{ deployment_type }}.example.com' - # https://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size nginx_client_max_body_size: 10m -nginx_proxy_read_timeout: 120s -nginx_proxy_connect_timeout: 120s -nginx_proxy_send_timeout: 120s +nginx_proxy_read_timeout: '{{ (([(client_request_timeout | int), 10] | max) / 2) | int }}' +nginx_proxy_connect_timeout: '{{ (([(client_request_timeout | int), 10] | max) / 2) | int }}' +nginx_proxy_send_timeout: '{{ (([(client_request_timeout | int), 10] | max) / 2) | int }}' -haproxy_timeout: 180s +haproxy_timeout: '{{ (([(client_request_timeout | int), 10] | max) / 2) | int }}' # https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/ # https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/ diff --git a/roles/galaxy-web/templates/galaxy-web.configmap.yaml.j2 b/roles/galaxy-web/templates/galaxy-web.configmap.yaml.j2 index b0730bc3..7751a283 100644 --- a/roles/galaxy-web/templates/galaxy-web.configmap.yaml.j2 +++ b/roles/galaxy-web/templates/galaxy-web.configmap.yaml.j2 @@ -102,6 +102,23 @@ data: # redirects, we set the Host: header above already. proxy_redirect off; proxy_pass http://pulp-content; + error_page 504 =503 /json_503; # Return 503 Service Unavailable with JSON response if gunicorn fails to respond + error_page 502 =503 /json_503; # Optional, in case gunicorn is completely down + } + + + location = /json_503 { + # Custom JSON response for 503 Service Unavailable + internal; + add_header Content-Type application/json; + + # Check if X-Request-ID is set and include it in the response + if ($http_x_request_id) { + return 503 '{"status": "error", "message": "Service Unavailable", "code": 503, "request_id": "$http_x_request_id"}'; + } + + # If X-Request-ID is not set, just return the basic JSON response + return 503 '{"status": "error", "message": "Service Unavailable", "code": 503}'; } location /{{ pulp_combined_settings.galaxy_api_path_prefix }}/pulp/api/v3/ { @@ -112,6 +129,8 @@ data: # redirects, we set the Host: header above already. proxy_redirect off; proxy_pass http://pulp-api; + error_page 504 =503 /json_503; # Return 503 Service Unavailable with JSON response if gunicorn fails to respond + error_page 502 =503 /json_503; # Optional, in case gunicorn is completely down } location /auth/login/ {