New Build (#242)

* context

* test plugin

* test

* update tests to run on remote manager

* add setup_manager to job kubernetes_integration_tests_py3

* test _file_resource_create

* add test kubernetes_integration_tests_py3

* add test kubernetes_integration_tests_py3

* add test kubernetes_integration_tests_py3

* update test_features.py Not ready

* remove managerorb

* update config

* minor changes

* attempt to add webbrowser module

* foo

* logging the message

* fixing the hashing

* merge additions

---------

Co-authored-by: kaplanyaniv <[email protected]>
Co-authored-by: Nelynehemia <[email protected]>
Co-authored-by: Nely Nehemia <[email protected]>

Author: 3 people

.circleci/config.yml

@@ -4,7 +4,6 @@ orbs:
   node: cloudify/public-unittest-orb@volatile
   wagonorb: cloudify/wagon-bulder-orb@volatile
   releaseorb: cloudify/release-orb@volatile
-  managerorb: cloudify/manager-orb@volatile
 
 checkout:
   post:
@@ -25,22 +24,40 @@ executors:
       image: ubuntu-2004:202201-02
 
 commands:
+
+  setup_manager:
+    steps:
+      - run: |
+          if [[ -z "${CLOUDIFY_HOST}" ]]; then
+              exit 1
+          fi
+      - run: |
+          if [[ -z "${CLOUDIFY_TENANT}" ]] && [ -z "${CIRCLE_PROJECT_REPONAME}" ]; then
+              exit 1
+          fi
+      - run: |
+          if [[ -z "${CLOUDIFY_TOKEN}" ]]; then
+              exit 1
+          fi
+      - run: |
+          response=$(curl --write-out '%{http_code}' --silent --insecure --header "Tenant: ${CLOUDIFY_TENANT}" --header "Authentication-Token: ${CLOUDIFY_TOKEN}" https://$CLOUDIFY_HOST/api/v3.1/status --output /dev/null)
+          if [[ $response != 200 ]]; then
+              echo "Failed to get manager status";
+              exit 1
+          fi
+      - run: pip3 install https://github.com/cloudify-incubator/cloudify-ecosystem-test/archive/refs/heads/master.zip
+      - run: git submodule update --init --recursive --remote
+
+
   prepare_test_manager:
     steps:
-      - run: ecosystem-test prepare-test-manager -l $TEST_LICENSE -es gcp_credentials=$gcp_credentials --yum-package python-netaddr --yum-package git -p $(find ~/project/workspace/build/ -name *manylinux-py311-none-linux_x86_64.wgn*)  ~/project/plugin.yaml
-      - run: ecosystem-test upload-plugin -PN utilities
-      - run: ecosystem-test upload-plugin -PN ansible
-      - run: ecosystem-test upload-plugin -PN gcp
+      - run: ecosystem-test prepare-remote-test-manager -es gcp_credentials=$gcp_credentials -p $(find ~/project/workspace/build/ -name *manylinux-py311-none-linux_x86_64.wgn*) ~/project/plugin.yaml
+      - run: curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+      - run: sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
 
   run_gke_test:
     steps:
-      - run: echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
-      - run: curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
-      - run: sudo apt-get update
-      - run: sudo apt-get install google-cloud-sdk-gke-gcloud-auth-plugin
-      - run: sudo apt-get install kubectl
-      - run: gcloud config set project trammell-project
-      - run: ecosystem-test local-blueprint-test -b examples/blueprint-examples/kubernetes/gcp-gke/blueprint.yaml --test-id=gcp-gke  -i resource_prefix=kube-$CIRCLE_BUILD_NUM --on-failure=uninstall-force --timeout=3000 --nested-test=.circleci/test_features.py
+       - run: ecosystem-test remote-blueprint-test -b examples/blueprint-examples/kubernetes/gcp-gke/blueprint.yaml --test-id=gcp-gke  -i resource_prefix=kube-$CIRCLE_BUILD_NUM --on-failure=uninstall-force --timeout=3000 --nested-test=.circleci/test_features.py
 
 jobs:
   kubernetes_integration_tests_py3:
@@ -49,7 +66,9 @@ jobs:
       CLOUDIFY_SSL_TRUST_ALL: true
     steps:
       - checkout
-      - managerorb/create_manager_container_latest
+      - attach_workspace:
+          at: workspace
+      - setup_manager
       - prepare_test_manager
       - run_gke_test
 
@@ -87,14 +106,16 @@ workflows:
             - wagonorb/wagon_311
           filters:
             branches:
-              only: /([0-9\.]*\-build|master|dev)/
+              only: /([0-9\.]*\-build|master|dev|RND-1248-k-8-s-plugin-nightly-failing)/
       - releaseorb/release:
           filters:
             branches:
               only: /master/
           requires:
             - node/unittests_job
             - wagonorb/wagon
+            - wagonorb/wagon_311
+            - wagonorb/arch64_wagon
             - wagonorb/rhel_wagon
             - kubernetes_integration_tests_py3
       - releaseorb/merge_docs_job:

.circleci/test_features.py

@@ -23,13 +23,18 @@
 
 import pytest
 
-from ecosystem_tests.dorkl import cleanup_on_failure
 from ecosystem_tests.dorkl.commands import handle_process
-from ecosystem_tests.dorkl.cloudify_api import (
-    cloudify_exec,
-    blueprints_upload,
-    deployments_create,
-    executions_start)
+from ecosystem_tests.ecosystem_tests_cli.logger import logger
+from ecosystem_tests.nerdl.api import (
+    create_secret,
+    with_client,
+    get_node_instance,
+    list_node_instances,
+    upload_blueprint,
+    create_deployment,
+    wait_for_install,
+    cleanup_on_failure)
+
 
 TEST_ID = environ.get('__ECOSYSTEM_TEST_ID', 'plugin-examples')
 
@@ -40,14 +45,19 @@ def test_update(*_, **__):
     setup_cli()
     try:
         # Upload Cloud Watch Blueprint
-        blueprints_upload(
+        upload_blueprint(
             'examples/file-test.yaml',
             deployment_id)
         # Create Cloud Watch Deployment with Instance ID input
-        deployments_create(
-            deployment_id, {"resource_path": "resources/pod.yaml"})
+        create_deployment(
+            deployment_id,
+            deployment_id,
+            {
+                "resource_path": "resources/pod.yaml"
+            }
+        )
         # Install Cloud Watch Deployment
-        executions_start('install', deployment_id, 300)
+        wait_for_install(deployment_id, 300)
         after_install = get_pod_info()
         update_params = {
             "kind": "Pod",
@@ -67,9 +77,9 @@ def test_update(*_, **__):
         tmp = tempfile.NamedTemporaryFile(delete=false, mode='w', suffix='.yaml')
         yaml.dump(params, tmp)
         tmp.close()
-        executions_start(
-            'update_resource_definition',
+        wait_for_workflow(
             deployment_id,
+            'update_resource_definition',
             300,
             params=tmp
         )
@@ -78,24 +88,31 @@ def test_update(*_, **__):
         assert after_install['spec']['containers'][0]['image'] == 'nginx:stable'
         assert after_update['spec']['containers'][0]['image'] == 'nginx:latest'
         # Uninstall Cloud Watch Deployment
-        executions_start('uninstall', deployment_id, 300)
+        wait_for_uninstall(deployment_id, 300)
     except:
         cleanup_on_failure(deployment_id)
 
 
 def setup_cli():
     cluster_name = runtime_properties(
         node_instance_by_name('kubernetes-cluster')['id'])['name']
-    capabilities = cloudify_exec('cfy deployments capabilities gcp-gke')
-    cloudify_exec('cfy secrets create -u -s {} kubernetes_endpoint'.format(
-        capabilities['endpoint']['value']), get_json=False)
+    capabilities = get_capabilities('gcp-gke')
+    logger.info('capabilities: {}'.format(capabilities))
+    create_secret('kubernetes_endpoint', capabilities['endpoint'])
     with open('gcp.json', 'wb') as outfile:
         creds = base64.b64decode(os.environ['gcp_credentials'])
         outfile.write(creds)
     handle_process('gcloud auth activate-service-account --key-file gcp.json')
     handle_process(
-        'gcloud container clusters get-credentials {} --region us-west1-a'
-        .format(cluster_name))
+        'gcloud container clusters get-credentials {} '
+        '--region us-west1-a --project {}'
+        .format(cluster_name, 'trammell-project'))
+
+
+@with_client
+def get_capabilities(dep_id, client):
+    dep = client.deployments.capabilities.get(dep_id)
+    return dep.capabilities
 
 
 def get_pod_info():
@@ -115,17 +132,9 @@ def node_instance_by_name(name):
     raise Exception('No node instances found.')
 
 
-def nodes():
-    return cloudify_exec('cfy nodes list')
-
-
 def node_instances():
-    return cloudify_exec('cfy node-instances list')
-
-
-def node_instance(node_instance_id):
-    return cloudify_exec('cfy node-instances get {}'.format(node_instance_id))
+    return list_node_instances(TEST_ID)
 
 
 def runtime_properties(node_instance_id):
-    return node_instance(node_instance_id)['runtime_properties']
+    return get_node_instance(node_instance_id)['runtime_properties']

CHANGELOG.txt

@@ -176,4 +176,5 @@
   - Check Drift Workflow
   - Check Status Workflow
 2.13.24: Support Azure Service Account Authentication.
-2.13.25: add __version__.py file in cloudify_kubernetes folder.
+2.13.25: add __version__.py file in cloudify_kubernetes folder.
+2.13.26: check plugin and update circleci config.

cloudify_kubernetes/__version__.py

@@ -1 +1 @@
-version = '2.13.25'
+version = '2.13.26'

cloudify_kubernetes/tasks/shared_cluster.py

@@ -32,7 +32,7 @@
         is_all_executions_finished)
     from cloudify_types.shared_resource.constants import \
         WORKFLOW_EXECUTION_TIMEOUT
-except ImportError:
+except ImportError as e:
     if PY311:
         from mgmtworker.cloudify_types.polling import poll_with_timeout
         from mgmtworker.cloudify_types.component.polling import (
@@ -41,7 +41,7 @@
         from mgmtworker.cloudify_types.shared_resource.constants import \
             WORKFLOW_EXECUTION_TIMEOUT
     elif not PY2:
-        raise
+        raise e
 
 
 @with_rest_client

extra-packaging-instructions.sh

@@ -5,6 +5,12 @@ then
     cat <<EOF > ./pydoc/__init__.py
 def get_doc(*args):
     return ''
+EOF
+    mkdir -p ./webbrowser
+    touch ./webbrowser/__init__.py
+    cat <<EOF > ./webbrowser/__init__.py
+def register(*args, **kwargs):
+    return ''
 EOF
     git apply python311.patch
 fi

plugin.yaml

@@ -3,7 +3,7 @@ plugins:
   kubernetes:
     executor: central_deployment_agent
     package_name: cloudify-kubernetes-plugin
-    package_version: '2.13.25'
+    package_version: '2.13.26'
 
 data_types:
 

plugin_1_4.yaml

@@ -3,7 +3,7 @@ plugins:
   kubernetes:
     executor: central_deployment_agent
     package_name: cloudify-kubernetes-plugin
-    package_version: '2.13.25'
+    package_version: '2.13.26'
 
 data_types:
 

python311.patch

@@ -123,7 +123,7 @@ index f4d4d02..47b9aae 100755
                'cloudify_kubernetes.tasks',
 -              'cloudify_kubernetes.tasks.nested_resources'],
 +              'cloudify_kubernetes.tasks.nested_resources',
-+              'pydoc'],
++              'pydoc', 'webbrowser'],
      license='LICENSE',
      install_requires=install_requires
  )

v2_plugin.yaml

@@ -3,7 +3,7 @@ plugins:
   kubernetes:
     executor: central_deployment_agent
     package_name: cloudify-kubernetes-plugin
-    package_version: '2.13.25'
+    package_version: '2.13.26'
 
 data_types: