Fix “[CI] Tests aren't being mocked correctly on CI reruns” (#60)

* Fix “[CI] Tests aren't being mocked correctly on CI reruns”

* add fix for aem test

Author: derekbekoe

scripts/ci/test_source.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+from __future__ import print_function
+
+import os
+import sys
+import tempfile
+import unittest
+import shutil
+from subprocess import check_call
+
+import mock
+from six import with_metaclass
+
+from util import get_repo_root
+
+
+SOURCES = os.path.join(get_repo_root(), 'src')
+
+ALL_TESTS = []
+
+for src_d in os.listdir(SOURCES):
+    src_d_full = os.path.join(SOURCES, src_d)
+    if os.path.isdir(src_d_full):
+        pkg_name = next((d for d in os.listdir(src_d_full) if d.startswith('azext_')), None)
+        # Find the package and check it has tests
+        if pkg_name and os.path.isdir(os.path.join(src_d_full, pkg_name, 'tests')):
+            ALL_TESTS.append((pkg_name, src_d_full))
+
+
+class TestExtensionSourceMeta(type):
+    def __new__(mcs, name, bases, _dict):
+
+        def gen_test(ext_path):
+            def test(self):
+                pip_args = [sys.executable, '-m', 'pip', 'install', '--upgrade', '--target',
+                            os.path.join(self.ext_dir, 'ext'), ext_path]
+                check_call(pip_args)
+                unittest_args = [sys.executable, '-m', 'unittest', 'discover', '-v', ext_path]
+                check_call(unittest_args)
+            return test
+
+        for tname, ext_path in ALL_TESTS:
+            test_name = "test_%s" % tname
+            _dict[test_name] = gen_test(ext_path)
+        return type.__new__(mcs, name, bases, _dict)
+
+
+class TestExtensionSource(with_metaclass(TestExtensionSourceMeta, unittest.TestCase)):
+
+    def setUp(self):
+        self.ext_dir = tempfile.mkdtemp()
+        self.mock_env = mock.patch.dict(os.environ, {'AZURE_EXTENSION_DIR': self.ext_dir})
+        self.mock_env.start()
+
+    def tearDown(self):
+        self.mock_env.stop()
+        shutil.rmtree(self.ext_dir)
+
+
+if __name__ == '__main__':
+    unittest.main()

scripts/ci/test_source.sh

@@ -9,25 +9,6 @@ pip install "git+https://github.com/Azure/azure-cli@dev#egg=azure-cli-testsdk&su
 pip install knack==0.3.1 -q
 echo "Installed."
 
-
-_AZURE_EXTENSION_DIR="$AZURE_EXTENSION_DIR"
-
-for d in src/*/azext_*/tests;
-    do echo "Running tests for $d";
-    if [ -d $d ]; then
-        export AZURE_EXTENSION_DIR=$(mktemp -d);
-        pip install --upgrade --target $AZURE_EXTENSION_DIR/ext $d/../..;
-        python -m unittest discover -v $d/../..;
-        rm -rf $AZURE_EXTENSION_DIR;
-    else
-        echo "Skipped $d as not a directory."
-    fi
-done;
-
-if ! [ -z "${_AZURE_EXTENSION_DIR+_}" ] ; then
-    AZURE_EXTENSION_DIR="$_AZURE_EXTENSION_DIR"
-    export AZURE_EXTENSION_DIR
-    unset _AZURE_EXTENSION_DIR
-fi
+python ./scripts/ci/test_source.py -v
 
 echo "OK. Completed tests."

src/aem/azext_aem/tests/latest/__init__.py

@@ -0,0 +1,4 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------

src/aem/azext_aem/tests/recordings/latest/test_vm_aem_configure.yaml renamed to src/aem/azext_aem/tests/latest/recordings/test_vm_aem_configure.yaml

@@ -15,7 +15,7 @@ def test_vm_aem_configure(self, resource_group):
         self.kwargs.update({
             'vm': 'vm1',
         })
-        self.cmd('vm create -g {rg} -n {vm} --image centos')
+        self.cmd('vm create -g {rg} -n {vm} --image centos --generate-ssh-keys')
         self.cmd('vm aem set -g {rg} -n {vm}')
         self.cmd('vm aem verify -g {rg} -n {vm}')
         self.cmd('vm aem delete -g {rg} -n {vm}')

src/aem/azext_aem/tests/test_aem_commands.py renamed to src/aem/azext_aem/tests/latest/test_aem_commands.py

@@ -0,0 +1,4 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------

src/subscription/azext_subscription/tests/latest/__init__.py

@@ -10,26 +10,26 @@
 
 class AzureSubscriptionDefinitionScenarioTest(ScenarioTest):
     def test_list_subscription_definitions(self):
-        sub_def_list = self.cmd('account subscriptiondefinition list').get_output_in_json()
+        sub_def_list = self.cmd('account subscription-definition list').get_output_in_json()
         self.assertGreater(len(sub_def_list), 0)
         self.assertIsNotNone(sub_def_list[0])
         self.assertIsNotNone(sub_def_list[0]['name'])
         self.assertIsNotNone(sub_def_list[0]['subscriptionId'])
         self.assertIsNotNone(sub_def_list[0]['subscriptionDisplayName'])
 
     def test_show_subscription_definitions(self):
-        first_sub_def = self.cmd('account subscriptiondefinition list').get_output_in_json()[0]
+        first_sub_def = self.cmd('account subscription-definition list').get_output_in_json()[0]
         self.assertIsNotNone(first_sub_def)
         self.assertIsNotNone(first_sub_def['name'])
-        sub_def = self.cmd('account subscriptiondefinition show -n {}'.format(first_sub_def['name'])).get_output_in_json()
+        sub_def = self.cmd('account subscription-definition show -n {}'.format(first_sub_def['name'])).get_output_in_json()
         self.assertIsNotNone(sub_def)
         self.assertEqual(sub_def['name'], first_sub_def['name'])
 
     def test_create_subscription_definitions(self):
-        sub_def_list_count_before = len(self.cmd('account subscriptiondefinition list').get_output_in_json())
+        sub_def_list_count_before = len(self.cmd('account subscription-definition list').get_output_in_json())
         def_name = self.create_random_name(prefix='cli', length=24)
-        sub_def = self.cmd('account subscriptiondefinition create -n {} --offer-type MS-AZR-0148P'.format(def_name)).get_output_in_json()
-        sub_def_list_count_after = len(self.cmd('account subscriptiondefinition list').get_output_in_json())
+        sub_def = self.cmd('account subscription-definition create -n {} --offer-type MS-AZR-0148P'.format(def_name)).get_output_in_json()
+        sub_def_list_count_after = len(self.cmd('account subscription-definition list').get_output_in_json())
         self.assertIsNotNone(sub_def)
         self.assertEqual(def_name, sub_def['name'])
         self.assertEqual(def_name, sub_def['subscriptionDisplayName'])