Make toolbox cli tests run on other workers

Mark all non-cli tests with a group, so they run sequentially.
Speed up from ~29min to ~24min for `make toolbox`.

Makefile

@@ -104,7 +104,7 @@ ui: pipenv check-secrets.yaml
 	$(PYTEST) --ui $(flags) testsuite/tests/ui
 
 toolbox: pipenv check-secrets.yaml
-	$(PYTEST) --toolbox $(flags) testsuite/tests/toolbox
+	$(PYTEST) -n4 --dist loadgroup --toolbox $(flags) testsuite/tests/toolbox
 
 test-images:
 	$(PYTEST) --images $(flags) testsuite/tests/images

testsuite/tests/toolbox/test_account.py

@@ -6,6 +6,10 @@
 
 from testsuite.toolbox import toolbox
 
+pytestmark = [
+    pytest.mark.xdist_group(name="toolbox"),
+]
+
 
 @pytest.fixture(scope="module")
 def create_cmd(threescale_src1):

testsuite/tests/toolbox/test_activedoc.py

@@ -11,6 +11,10 @@
 
 SWAGGER_LINK = "https://raw.githubusercontent.com/OAI/OpenAPI-Specification/master/examples/v2.0/json/petstore.json"
 
+pytestmark = [
+    pytest.mark.xdist_group(name="toolbox"),
+]
+
 
 @pytest.fixture(scope="module")
 def my_app_plan(request, custom_app_plan, service):

testsuite/tests/toolbox/test_app_plans.py

@@ -14,6 +14,10 @@
 from testsuite.utils import blame
 from testsuite import rawobj
 
+pytestmark = [
+    pytest.mark.xdist_group(name="toolbox"),
+]
+
 
 @pytest.fixture(scope="module")
 def service(request, custom_service):

testsuite/tests/toolbox/test_application.py

@@ -9,6 +9,10 @@
 from testsuite.utils import blame
 from testsuite import rawobj
 
+pytestmark = [
+    pytest.mark.xdist_group(name="toolbox"),
+]
+
 
 @pytest.fixture(scope="module")
 def my_services(custom_service, service, request):

testsuite/tests/toolbox/test_backend.py

@@ -12,7 +12,11 @@
 import testsuite.utils
 from testsuite import TESTED_VERSION  # noqa # pylint: disable=unused-import
 
-pytestmark = pytest.mark.skipif("TESTED_VERSION < Version('2.7')")
+pytestmark = [
+    pytest.mark.skipif("TESTED_VERSION < Version('2.7')"),
+    pytest.mark.xdist_group(name="toolbox"),
+]
+
 
 HTTP_METHODS = ["GET", "POST", "PUT", "DELETE", "HEAD", "OPTIONS", "PATCH"]
 # 'TRACE', 'CONNECT' are not supported by 3scale

testsuite/tests/toolbox/test_cli.py

@@ -9,7 +9,9 @@
 from testsuite.toolbox import toolbox
 from testsuite import TESTED_VERSION  # noqa # pylint: disable=unused-import
 
-pytestmark = pytest.mark.skipif("TESTED_VERSION < Version('2.7')")
+pytestmark = [
+    pytest.mark.skipif("TESTED_VERSION < Version('2.7')"),
+]
 
 # removed 'update' as unsupported command
 

testsuite/tests/toolbox/test_method.py

@@ -11,6 +11,10 @@
 from testsuite.toolbox import constants
 from testsuite.toolbox import toolbox
 
+pytestmark = [
+    pytest.mark.xdist_group(name="toolbox"),
+]
+
 
 @pytest.fixture(scope="module")
 def hits(request, service):

testsuite/tests/toolbox/test_metric.py

@@ -11,6 +11,10 @@
 from testsuite.toolbox import constants
 from testsuite.toolbox import toolbox
 
+pytestmark = [
+    pytest.mark.xdist_group(name="toolbox"),
+]
+
 
 @pytest.fixture(scope="module")
 def metric_obj(request, service):

testsuite/tests/toolbox/test_openapi.py

@@ -16,6 +16,10 @@
 from testsuite.toolbox import toolbox
 from testsuite.utils import blame
 
+pytestmark = [
+    pytest.mark.xdist_group(name="toolbox"),
+]
+
 # authentization in 3scale and mapping to OAS(http://spec.openapis.org/oas/v3.0.3#security-scheme-object):
 #
 # - 1 token -> see uber.json

testsuite/tests/toolbox/test_policies_imp_exp.py

@@ -8,6 +8,10 @@
 from testsuite.config import settings
 from testsuite.toolbox import toolbox
 
+pytestmark = [
+    pytest.mark.xdist_group(name="toolbox"),
+]
+
 
 @pytest.fixture(scope="module")
 def policy_file(policy_configs):

testsuite/tests/toolbox/test_product_backend_exp.py

@@ -10,6 +10,10 @@
 from testsuite.utils import blame
 from testsuite import rawobj
 
+pytestmark = [
+    pytest.mark.xdist_group(name="toolbox"),
+]
+
 
 @pytest.fixture(scope="module")
 def export_import_file():

testsuite/tests/toolbox/test_product_copy.py

@@ -9,7 +9,10 @@
 from testsuite.utils import blame, blame_desc
 from testsuite import rawobj, TESTED_VERSION  # noqa # pylint: disable=unused-import
 
-pytestmark = pytest.mark.skipif("TESTED_VERSION < Version('2.7')")
+pytestmark = [
+    pytest.mark.skipif("TESTED_VERSION < Version('2.7')"),
+    pytest.mark.xdist_group(name="toolbox"),
+]
 
 
 @pytest.fixture(scope="module", params=["copy_service", "product_copy", "service_copy"])

testsuite/tests/toolbox/test_product_update.py

@@ -9,7 +9,10 @@
 from testsuite.utils import blame, blame_desc
 from testsuite import rawobj, TESTED_VERSION  # noqa # pylint: disable=unused-import
 
-pytestmark = pytest.mark.skipif("TESTED_VERSION < Version('2.7')")
+pytestmark = [
+    pytest.mark.skipif("TESTED_VERSION < Version('2.7')"),
+    pytest.mark.xdist_group(name="toolbox"),
+]
 
 
 @pytest.fixture(scope="module", params=["product_copy", "service_copy"])

testsuite/tests/toolbox/test_proxy.py

@@ -5,6 +5,10 @@
 
 from testsuite.toolbox import toolbox
 
+pytestmark = [
+    pytest.mark.xdist_group(name="toolbox"),
+]
+
 
 @pytest.fixture(scope="module")
 def create_cmd(threescale_src1):

testsuite/tests/toolbox/test_proxycfg.py

@@ -7,6 +7,10 @@
 from testsuite import rawobj
 from testsuite.toolbox import toolbox
 
+pytestmark = [
+    pytest.mark.xdist_group(name="toolbox"),
+]
+
 
 @pytest.fixture(scope="module")
 def create_cmd(threescale_src1):

testsuite/tests/toolbox/test_remote.py

@@ -8,6 +8,10 @@
 
 from testsuite.toolbox import toolbox
 
+pytestmark = [
+    pytest.mark.xdist_group(name="toolbox"),
+]
+
 
 EMPTY_LIST = "Empty remote list.\n"
 

testsuite/tests/toolbox/test_service.py

@@ -8,6 +8,10 @@
 from testsuite.toolbox import toolbox
 from testsuite.utils import randomize
 
+pytestmark = [
+    pytest.mark.xdist_group(name="toolbox"),
+]
+
 
 @pytest.fixture(scope="module")
 def empty_list(service, create_cmd):

testsuite/tests/toolbox/test_service_csv.py

@@ -10,6 +10,10 @@
 from testsuite.toolbox import toolbox
 from testsuite.utils import blame
 
+pytestmark = [
+    pytest.mark.xdist_group(name="toolbox"),
+]
+
 
 DATA = [
     "service_name",