Add delay as some tests are flaky in aws cloud

Adjust measurement error

tests/suite/fixtures.py

@@ -326,11 +326,12 @@ def virtual_server_setup(request, kube_apis, crd_ingress_controller, ingress_con
     :return: VirtualServerSetup
     """
     print("------------------------- Deploy Virtual Server Example -----------------------------------")
+    vs_source = f"{TEST_DATA}/{request.param['example']}/standard/virtual-server.yaml"
     vs_name = create_virtual_server_from_yaml(kube_apis.custom_objects,
-                                              f"{TEST_DATA}/{request.param['example']}/standard/virtual-server.yaml",
+                                              vs_source,
                                               test_namespace)
-    vs_host = get_first_vs_host_from_yaml(f"{TEST_DATA}/{request.param['example']}/standard/virtual-server.yaml")
-    vs_paths = get_paths_from_vs_yaml(f"{TEST_DATA}/{request.param['example']}/standard/virtual-server.yaml")
+    vs_host = get_first_vs_host_from_yaml(vs_source)
+    vs_paths = get_paths_from_vs_yaml(vs_source)
     common_app = create_example_app(kube_apis, request.param['app_type'], test_namespace)
     wait_until_all_pods_are_ready(kube_apis.v1, test_namespace)
 

tests/suite/resources_utils.py

@@ -850,7 +850,7 @@ def ensure_connection(request_url) -> None:
     :param request_url: url to request
     :return:
     """
-    for _ in range(0, 4):
+    for _ in range(4):
         try:
             resp = requests.get(request_url, verify=False)
             if resp.status_code == 404:
@@ -914,3 +914,19 @@ def get_events(v1: CoreV1Api, namespace) -> []:
     print(f"Get the events in the namespace: {namespace}")
     res = v1.list_namespaced_event(namespace)
     return res.items
+
+
+def ensure_response_from_backend(req_url, host) -> None:
+    """
+    Wait for 502 to disappear.
+
+    :param req_url: url to request
+    :param host:
+    :return:
+    """
+    for _ in range(5):
+        resp = requests.get(req_url, headers={"host": host}, verify=False)
+        if resp.status_code != 502:
+            return
+        time.sleep(2)
+    pytest.fail(f"Keep getting 502 from {req_url} after 10 seconds. Exiting...")

tests/suite/test_smoke.py

@@ -2,9 +2,10 @@
 import pytest
 
 from suite.fixtures import PublicEndpoint
-from suite.resources_utils import create_secret_from_yaml, delete_secret, ensure_connection_to_public_endpoint
-from suite.resources_utils import create_items_from_yaml, delete_items_from_yaml, create_common_app, delete_common_app
-from suite.resources_utils import wait_until_all_pods_are_ready
+from suite.resources_utils import create_secret_from_yaml, delete_secret, \
+    ensure_connection_to_public_endpoint, create_items_from_yaml, \
+    delete_items_from_yaml, create_common_app, delete_common_app, \
+    wait_until_all_pods_are_ready, wait_before_test, ensure_response_from_backend
 from suite.yaml_utils import get_first_ingress_host_from_yaml
 from settings import TEST_DATA
 
@@ -53,6 +54,7 @@ class TestSmoke:
     @pytest.mark.parametrize("path", paths)
     def test_response_code_200_and_server_name(self, smoke_setup, path):
         req_url = f"https://{smoke_setup.public_endpoint.public_ip}:{smoke_setup.public_endpoint.port_ssl}/{path}"
+        ensure_response_from_backend(req_url, smoke_setup.ingress_host)
         resp = requests.get(req_url, headers={"host": smoke_setup.ingress_host}, verify=False)
         assert resp.status_code == 200
         assert f"Server name: {path}" in resp.text

tests/suite/test_virtual_server_advanced_routing.py

@@ -4,7 +4,7 @@
 
 from settings import TEST_DATA
 from suite.custom_resources_utils import patch_virtual_server_from_yaml
-from suite.resources_utils import wait_before_test
+from suite.resources_utils import wait_before_test, ensure_response_from_backend
 
 
 def execute_assertions(resp_1, resp_2, resp_3):
@@ -22,6 +22,8 @@ def execute_assertions(resp_1, resp_2, resp_3):
                          indirect=True)
 class TestAdvancedRouting:
     def test_flow_with_header(self, kube_apis, crd_ingress_controller, virtual_server_setup):
+        ensure_response_from_backend(virtual_server_setup.backend_1_url, virtual_server_setup.vs_host)
+
         resp_1 = requests.get(virtual_server_setup.backend_1_url,
                               headers={"host": virtual_server_setup.vs_host, "x-version": "future"})
         resp_2 = requests.get(virtual_server_setup.backend_1_url,

tests/suite/test_virtual_server_split_traffic.py

@@ -52,7 +52,7 @@ def test_several_requests(self, kube_apis, crd_ingress_controller, virtual_serve
         ratios = [round(i/sum_weights, 1) for i in weights]
 
         counter_v1, counter_v2 = 0, 0
-        for _ in range(0, 100):
+        for _ in range(100):
             resp = requests.get(virtual_server_setup.backend_1_url,
                                 headers={"host": virtual_server_setup.vs_host})
             if upstreams[0] in resp.text in resp.text:
@@ -62,5 +62,5 @@ def test_several_requests(self, kube_apis, crd_ingress_controller, virtual_serve
             else:
                 pytest.fail(f"An unexpected backend in response: {resp.text}")
 
-        assert round(counter_v1/(counter_v1 + counter_v2), 1) == ratios[0]
-        assert round(counter_v2/(counter_v1 + counter_v2), 1) == ratios[1]
+        assert abs(round(counter_v1/(counter_v1 + counter_v2), 1) - ratios[0]) <= 0.2
+        assert abs(round(counter_v2/(counter_v1 + counter_v2), 1) - ratios[1]) <= 0.2