Convert ansible-test compile into a sanity test.

Author: mattclay

docs/docsite/rst/dev_guide/testing/sanity/compile.rst

@@ -0,0 +1,4 @@
+Sanity Tests » compile
+======================
+
+See :doc:`../../testing_compile` for more information.

docs/docsite/rst/dev_guide/testing_compile.rst

@@ -15,34 +15,36 @@ Compile tests check source files for valid syntax on all supported python versio
 - 3.5
 - 3.6
 
+NOTE: In Ansible 2.4 and earlier the compile test was provided by a dedicated sub-command ``ansible-test compile`` instead of a sanity test using ``ansible-test sanity --test compile``.
+
 Running compile tests locally
 =============================
 
-Unit tests can be run across the whole code base by doing:
+Compile tests can be run across the whole code base by doing:
 
 .. code:: shell
 
     cd /path/to/ansible/source
     source hacking/env-setup
-    ansible-test compile
+    ansible-test sanity --test compile
 
 Against a single file by doing:
 
 .. code:: shell
 
-   ansible-test compile lineinfile
+   ansible-test sanity --test compile lineinfile
 
 Or against a specific Python version by doing:
 
 .. code:: shell
 
-   ansible-test compile --python 2.7 lineinfile
+   ansible-test sanity --test compile --python 2.7 lineinfile
 
 For advanced usage see the help:
 
 .. code:: shell
 
-   ansible-test units --help
+   ansible-test sanity --help
 
 
 Installing dependencies
@@ -54,7 +56,7 @@ The dependencies can be installed using the ``--requirements`` argument. For exa
 
 .. code:: shell
 
-   ansible-test units --requirements lineinfile
+   ansible-test sanity --test compile --requirements lineinfile
 
 
 
@@ -64,4 +66,4 @@ The full list of requirements can be found at `test/runner/requirements <https:/
 Extending compile tests
 =======================
 
-If you believe changes are needed to the Compile tests please add a comment on the `Testing Working Group Agenda <https://github.com/ansible/community/blob/master/meetings/README.md>`_ so it can be discussed.
+If you believe changes are needed to the compile tests please add a comment on the `Testing Working Group Agenda <https://github.com/ansible/community/blob/master/meetings/README.md>`_ so it can be discussed.

shippable.yml

@@ -8,7 +8,8 @@ matrix:
   exclude:
     - env: T=none
   include:
-    - env: T=other
+    - env: T=sanity/1
+    - env: T=sanity/2
 
     - env: T=units/2.6
     - env: T=units/2.7

test/runner/Makefile

@@ -1,9 +1,6 @@
-.PHONY: all compile sanity units
+.PHONY: all sanity units
 
-all: compile sanity units
-
-compile:
-	./ansible-test compile test/runner/ ${TEST_FLAGS}
+all: sanity units
 
 sanity:
 	./ansible-test sanity test/runner/ ${TEST_FLAGS}

test/runner/lib/classification.py

@@ -44,7 +44,6 @@ def categorize_changes(args, paths, verbose_command=None):
 
     commands = {
         'sanity': set(),
-        'compile': set(),
         'units': set(),
         'integration': set(),
         'windows-integration': set(),
@@ -128,7 +127,6 @@ def __init__(self, args):
         self.sanity_targets = list(walk_sanity_targets())
         self.powershell_targets = [t for t in self.sanity_targets if os.path.splitext(t.path)[1] == '.ps1']
 
-        self.compile_paths = set(t.path for t in self.compile_targets)
         self.units_modules = set(t.module for t in self.units_targets if t.module)
         self.units_paths = set(a for t in self.units_targets for a in t.aliases)
         self.sanity_paths = set(t.path for t in self.sanity_targets)
@@ -228,10 +226,6 @@ def classify(self, path):
         if result is None:
             return None
 
-        # compile path if eligible
-        if path in self.compile_paths:
-            result['compile'] = path
-
         # run sanity on path unless result specified otherwise
         if path in self.sanity_paths and 'sanity' not in result:
             result['sanity'] = path
@@ -393,11 +387,6 @@ def _classify(self, path):
         if path.startswith('test/cache/'):
             return minimal
 
-        if path.startswith('test/compile/'):
-            return {
-                'compile': 'all',
-            }
-
         if path.startswith('test/results/'):
             return minimal
 
@@ -544,7 +533,32 @@ def _classify(self, path):
 
             return all_tests(self.args)  # test infrastructure, run all tests
 
+        if path.startswith('test/utils/shippable/tools/'):
+            return minimal  # not used by tests
+
         if path.startswith('test/utils/shippable/'):
+            if dirname == 'test/utils/shippable':
+                test_map = {
+                    'cloud.sh': 'integration:cloud/',
+                    'freebsd.sh': 'integration:all',
+                    'linux.sh': 'integration:all',
+                    'network.sh': 'network-integration:all',
+                    'osx.sh': 'integration:all',
+                    'rhel.sh': 'integration:all',
+                    'sanity.sh': 'sanity:all',
+                    'units.sh': 'units:all',
+                    'windows.sh': 'windows-integration:all',
+                }
+
+                test_match = test_map.get(filename)
+
+                if test_match:
+                    test_command, test_target = test_match.split(':')
+
+                    return {
+                        test_command: test_target,
+                    }
+
             return all_tests(self.args)  # test infrastructure, run all tests
 
         if path.startswith('test/utils/'):
@@ -602,7 +616,6 @@ def all_tests(args, force=False):
 
     return {
         'sanity': 'all',
-        'compile': 'all',
         'units': 'all',
         'integration': integration_all_target,
         'windows-integration': integration_all_target,

test/runner/lib/config.py

@@ -206,15 +206,6 @@ def __init__(self, args):
         self.collect_only = args.collect_only  # type: bool
 
 
-class CompileConfig(TestConfig):
-    """Configuration for the compile command."""
-    def __init__(self, args):
-        """
-        :type args: any
-        """
-        super(CompileConfig, self).__init__(args, 'compile')
-
-
 class CoverageConfig(EnvironmentConfig):
     """Configuration for the coverage command."""
     def __init__(self, args):

test/runner/lib/executor.py

@@ -63,7 +63,6 @@
     walk_network_integration_targets,
     walk_windows_integration_targets,
     walk_units_targets,
-    walk_compile_targets,
 )
 
 from lib.changes import (
@@ -82,7 +81,6 @@
 from lib.config import (
     TestConfig,
     EnvironmentConfig,
-    CompileConfig,
     IntegrationConfig,
     NetworkIntegrationConfig,
     PosixIntegrationConfig,
@@ -91,13 +89,6 @@
     WindowsIntegrationConfig,
 )
 
-from lib.test import (
-    TestMessage,
-    TestSuccess,
-    TestFailure,
-    TestSkipped,
-)
-
 SUPPORTED_PYTHON_VERSIONS = (
     '2.6',
     '2.7',
@@ -106,8 +97,6 @@
     '3.7',
 )
 
-COMPILE_PYTHON_VERSIONS = SUPPORTED_PYTHON_VERSIONS
-
 
 def check_startup():
     """Checks to perform at startup before running commands."""
@@ -1024,114 +1013,6 @@ def command_units(args):
                 raise
 
 
-def command_compile(args):
-    """
-    :type args: CompileConfig
-    """
-    changes = get_changes_filter(args)
-    require = (args.require or []) + changes
-    include, exclude = walk_external_targets(walk_compile_targets(), args.include, args.exclude, require)
-
-    if not include:
-        raise AllTargetsSkipped()
-
-    if args.delegate:
-        raise Delegate(require=changes)
-
-    install_command_requirements(args)
-
-    total = 0
-    failed = []
-
-    for version in COMPILE_PYTHON_VERSIONS:
-        # run all versions unless version given, in which case run only that version
-        if args.python and version != args.python_version:
-            continue
-
-        display.info('Compile with Python %s' % version)
-
-        result = compile_version(args, version, include, exclude)
-        result.write(args)
-
-        total += 1
-
-        if isinstance(result, TestFailure):
-            failed.append('compile --python %s' % version)
-
-    if failed:
-        message = 'The %d compile test(s) listed below (out of %d) failed. See error output above for details.\n%s' % (
-            len(failed), total, '\n'.join(failed))
-
-        if args.failure_ok:
-            display.error(message)
-        else:
-            raise ApplicationError(message)
-
-
-def compile_version(args, python_version, include, exclude):
-    """
-    :type args: CompileConfig
-    :type python_version: str
-    :type include: tuple[CompletionTarget]
-    :type exclude: tuple[CompletionTarget]
-    :rtype: TestResult
-    """
-    command = 'compile'
-    test = ''
-
-    # optional list of regex patterns to exclude from tests
-    skip_file = 'test/compile/python%s-skip.txt' % python_version
-
-    if os.path.exists(skip_file):
-        with open(skip_file, 'r') as skip_fd:
-            skip_paths = skip_fd.read().splitlines()
-    else:
-        skip_paths = []
-
-    # augment file exclusions
-    skip_paths += [e.path for e in exclude]
-
-    skip_paths = sorted(skip_paths)
-
-    python = 'python%s' % python_version
-    cmd = [python, 'test/compile/compile.py']
-
-    if skip_paths:
-        cmd += ['-x', '|'.join(skip_paths)]
-
-    cmd += [target.path if target.path == '.' else './%s' % target.path for target in include]
-
-    try:
-        stdout, stderr = run_command(args, cmd, capture=True)
-        status = 0
-    except SubprocessError as ex:
-        stdout = ex.stdout
-        stderr = ex.stderr
-        status = ex.status
-
-    if stderr:
-        raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout)
-
-    if args.explain:
-        return TestSkipped(command, test, python_version=python_version)
-
-    pattern = r'^(?P<path>[^:]*):(?P<line>[0-9]+):(?P<column>[0-9]+): (?P<message>.*)$'
-
-    results = [re.search(pattern, line).groupdict() for line in stdout.splitlines()]
-
-    results = [TestMessage(
-        message=r['message'],
-        path=r['path'].replace('./', ''),
-        line=int(r['line']),
-        column=int(r['column']),
-    ) for r in results]
-
-    if results:
-        return TestFailure(command, test, messages=results, python_version=python_version)
-
-    return TestSuccess(command, test, python_version=python_version)
-
-
 def get_changes_filter(args):
     """
     :type args: TestConfig