From 62ae9daa7d7188b2cbd7ba1ba6840bd1b0162075 Mon Sep 17 00:00:00 2001 From: Steve Goldhaber Date: Sat, 8 Nov 2025 00:15:16 +0100 Subject: [PATCH] Add unit tests for SDF reading Fix xmllint checking (some xmllint versions do not return error codes) Update regex usage for fortran_conditional --- scripts/parse_tools/fortran_conditional.py | 7 +- scripts/parse_tools/parse_checkers.py | 5 +- scripts/parse_tools/xml_tools.py | 44 +- .../sample_suite_files/another_suite.xml | 10 + .../sample_suite_files/another_suite2.xml | 16 + .../sample_suite_files/nested_full_suite.xml | 10 + .../sample_suite_files/subsuite1.xml | 7 + .../sample_suite_files/subsuite_inline.xml | 9 + .../suite_bad_v2_duplicate_group.xml | 16 + .../suite_bad_v2_suite_tag.xml | 7 + .../suite_bad_version01.xml | 8 + .../suite_bad_version02.xml | 8 + .../suite_bad_version03.xml | 8 + .../suite_bad_version04.xml | 8 + .../suite_good_v1_test01.xml | 8 + .../suite_good_v1_test02.xml | 11 + .../suite_good_v2_test01.xml | 9 + .../suite_good_v2_test01_exp.xml | 11 + .../suite_good_v2_test02.xml | 10 + .../suite_good_v2_test02_exp.xml | 13 + .../suite_good_v2_test03.xml | 19 + .../suite_good_v2_test03_exp.xml | 30 + .../suite_good_v2_test04.xml | 18 + .../suite_good_v2_test04_exp.xml | 26 + .../sample_suite_files/suite_missing_file.xml | 9 + .../suite_missing_group.xml | 7 + .../suite_missing_loaded_suite.xml | 16 + .../suite_missing_version.xml | 8 + .../suite_recurse_level2.xml | 10 + .../suite_recurse_level2a.xml | 10 + .../suite_recurse_level3.xml | 10 + .../suite_recurse_level3a.xml | 10 + .../sample_suite_files/suite_recurse_top1.xml | 18 + .../sample_suite_files/suite_recurse_top2.xml | 18 + test/unit_tests/test_sdf.py | 525 ++++++++++++++++++ 35 files changed, 947 insertions(+), 12 deletions(-) create mode 100644 test/unit_tests/sample_suite_files/another_suite.xml create mode 100644 test/unit_tests/sample_suite_files/another_suite2.xml create mode 100644 test/unit_tests/sample_suite_files/nested_full_suite.xml create mode 100644 test/unit_tests/sample_suite_files/subsuite1.xml create mode 100644 test/unit_tests/sample_suite_files/subsuite_inline.xml create mode 100644 test/unit_tests/sample_suite_files/suite_bad_v2_duplicate_group.xml create mode 100644 test/unit_tests/sample_suite_files/suite_bad_v2_suite_tag.xml create mode 100644 test/unit_tests/sample_suite_files/suite_bad_version01.xml create mode 100644 test/unit_tests/sample_suite_files/suite_bad_version02.xml create mode 100644 test/unit_tests/sample_suite_files/suite_bad_version03.xml create mode 100644 test/unit_tests/sample_suite_files/suite_bad_version04.xml create mode 100644 test/unit_tests/sample_suite_files/suite_good_v1_test01.xml create mode 100644 test/unit_tests/sample_suite_files/suite_good_v1_test02.xml create mode 100644 test/unit_tests/sample_suite_files/suite_good_v2_test01.xml create mode 100644 test/unit_tests/sample_suite_files/suite_good_v2_test01_exp.xml create mode 100644 test/unit_tests/sample_suite_files/suite_good_v2_test02.xml create mode 100644 test/unit_tests/sample_suite_files/suite_good_v2_test02_exp.xml create mode 100644 test/unit_tests/sample_suite_files/suite_good_v2_test03.xml create mode 100644 test/unit_tests/sample_suite_files/suite_good_v2_test03_exp.xml create mode 100644 test/unit_tests/sample_suite_files/suite_good_v2_test04.xml create mode 100644 test/unit_tests/sample_suite_files/suite_good_v2_test04_exp.xml create mode 100644 test/unit_tests/sample_suite_files/suite_missing_file.xml create mode 100644 test/unit_tests/sample_suite_files/suite_missing_group.xml create mode 100644 test/unit_tests/sample_suite_files/suite_missing_loaded_suite.xml create mode 100644 test/unit_tests/sample_suite_files/suite_missing_version.xml create mode 100644 test/unit_tests/sample_suite_files/suite_recurse_level2.xml create mode 100644 test/unit_tests/sample_suite_files/suite_recurse_level2a.xml create mode 100644 test/unit_tests/sample_suite_files/suite_recurse_level3.xml create mode 100644 test/unit_tests/sample_suite_files/suite_recurse_level3a.xml create mode 100644 test/unit_tests/sample_suite_files/suite_recurse_top1.xml create mode 100644 test/unit_tests/sample_suite_files/suite_recurse_top2.xml create mode 100644 test/unit_tests/test_sdf.py diff --git a/scripts/parse_tools/fortran_conditional.py b/scripts/parse_tools/fortran_conditional.py index b1ec7eeb..ab80da9a 100755 --- a/scripts/parse_tools/fortran_conditional.py +++ b/scripts/parse_tools/fortran_conditional.py @@ -7,7 +7,10 @@ import re -FORTRAN_CONDITIONAL_REGEX_WORDS = [' ', '(', ')', '==', '/=', '<=', '>=', '<', '>', '.eqv.', '.neqv.', +fortran_conditional_regex_tokens = ['==', '/=', '<=', '>=', '<', '>', '.eqv.', '.neqv.', '.true.', '.false.', '.lt.', '.le.', '.eq.', '.ge.', '.gt.', '.ne.', '.not.', '.and.', '.or.', '.xor.'] -FORTRAN_CONDITIONAL_REGEX = re.compile(r"[\w']+|" + "|".join([word.replace('(','\(').replace(')', '\)') for word in FORTRAN_CONDITIONAL_REGEX_WORDS])) + +FORTRAN_CONDITIONAL_REGEX_WORDS = [' ', '[(]', '[)]'] + fortran_conditional_regex_tokens + +FORTRAN_CONDITIONAL_REGEX = re.compile(r"[\w']+|[ ()]|" + "|".join([word for word in FORTRAN_CONDITIONAL_REGEX_WORDS])) diff --git a/scripts/parse_tools/parse_checkers.py b/scripts/parse_tools/parse_checkers.py index 5aaaaeab..2c7184b7 100755 --- a/scripts/parse_tools/parse_checkers.py +++ b/scripts/parse_tools/parse_checkers.py @@ -13,13 +13,14 @@ ######################################################################## _UNITLESS_REGEX = "1" -_NON_LEADING_ZERO_NUM = "[1-9]\d*" +_NON_LEADING_ZERO_NUM = r"[1-9]\d*" _CHAR_WITH_UNDERSCORE = "([a-zA-Z]+_[a-zA-Z]+)+" _NEGATIVE_NON_LEADING_ZERO_NUM = f"[-]{_NON_LEADING_ZERO_NUM}" _POSITIVE_NON_LEADING_ZERO_NUM = f"[+]{_NON_LEADING_ZERO_NUM}" _UNIT_EXPONENT = f"({_NEGATIVE_NON_LEADING_ZERO_NUM}|{_POSITIVE_NON_LEADING_ZERO_NUM}|{_NON_LEADING_ZERO_NUM})" _UNIT_REGEX = f"[a-zA-Z]+{_UNIT_EXPONENT}?" -_UNITS_REGEX = f"^({_CHAR_WITH_UNDERSCORE}|{_UNIT_REGEX}(\s{_UNIT_REGEX})*|{_UNITLESS_REGEX})$" +_UNIT_REGEXES = f"{_UNIT_REGEX}(" r"\s" f"{_UNIT_REGEX})" +_UNITS_REGEX = f"^({_CHAR_WITH_UNDERSCORE}|{_UNIT_REGEXES}*|{_UNITLESS_REGEX})$" _UNITS_RE = re.compile(_UNITS_REGEX) _MAX_MOLAR_MASS = 10000.0 diff --git a/scripts/parse_tools/xml_tools.py b/scripts/parse_tools/xml_tools.py index 7b9b1710..1c8efe17 100644 --- a/scripts/parse_tools/xml_tools.py +++ b/scripts/parse_tools/xml_tools.py @@ -38,10 +38,14 @@ def __init__(self, message): super().__init__(message) ############################################################################### -def call_command(commands, logger, silent=False): +def call_command(commands, logger, silent=False, return_proc=False): ############################################################################### """ - Try a command line and return the output on success (None on failure) + Try a command line and return True only for a zero return code + If silent==True, do not log output and simply return False on an exception + If return_proc==True, return the CompletedProcess instance instead of a boolean + If silent==True and return_proc==True, return None in the case of an exception + >>> _LOGGER = init_log('xml_tools') >>> set_log_to_null(_LOGGER) >>> call_command(['ls', 'really__improbable_fffilename.foo'], _LOGGER) #doctest: +IGNORE_EXCEPTION_DETAIL @@ -50,6 +54,8 @@ def call_command(commands, logger, silent=False): [Errno 2] No such file or directory >>> call_command(['ls', 'really__improbable_fffilename.foo'], _LOGGER, silent=True) False + >>> call_command(['ls'], _LOGGER, silent=True, return_proc=True).returncode + 0 >>> call_command(['ls'], _LOGGER) True >>> try: @@ -75,11 +81,22 @@ def call_command(commands, logger, silent=False): capture_output=True) if not silent: logger.debug(cproc.stdout) + if cproc.stderr: + logger.warning(cproc.stderr) + # end if + # end if + if return_proc: + result = cproc + else: + result = cproc.returncode == 0 # end if - result = cproc.returncode == 0 except (OSError, CCPPError, subprocess.CalledProcessError) as err: if silent: - result = False + if return_proc: + result = None + else: + result = False + # end if else: cmd = ' '.join(commands) outstr = f"Execution of '{cmd}' failed with code: {err.returncode}\n" @@ -213,7 +230,15 @@ def validate_xml_file(filename, schema_root, version, logger, logger.debug("Checking file {} against schema {}".format(filename, schema_file)) cmd = [_XMLLINT, '--noout', '--schema', schema_file, filename] - result = call_command(cmd, logger) + result = call_command(cmd, logger, return_proc=True) + if result.returncode == 0: + ## We got a pass return code but some versions of xmllint do not + ## correctly return an error code on non-validation so double check + ## the result + result = b'validates' in result.stdout or b'validates' in result.stderr + else: + result = True + # end if return result # end if lmsg = "xmllint not found, could not validate file {}" @@ -411,11 +436,14 @@ def replace_nested_suite(element, nested_suite, default_path, logger): suite_name = nested_suite.attrib.get("name") group_name = nested_suite.attrib.get("group") file = nested_suite.attrib.get("file") + if not file: + raise CCPPError("file attribute required for nested_suite tag") + # end if if not os.path.isabs(file): file = os.path.join(default_path, file) referenced_suite = load_suite_by_name(suite_name, group_name, file, logger=logger) - imported_content = [ET.fromstring(ET.tostring(child)) + imported_content = [ET.fromstring(ET.tostring(child)) for child in referenced_suite] # Swap nested suite with imported content for item in imported_content: @@ -581,7 +609,7 @@ def expand_nested_suites(suite, default_path, logger=None): return raise CCPPError("Exceeded number of iterations while expanding nested suites:" + \ "check for inifite recursion or adjust limit max_iterations") - + ############################################################################### def write_xml_file(root, file_path, logger=None): ############################################################################### @@ -598,7 +626,7 @@ def remove_whitespace_nodes(node): # Convert ElementTree to a byte string byte_string = ET.tostring(root, 'us-ascii') - + # Parse string using minidom for pretty printing reparsed = xml.dom.minidom.parseString(byte_string) diff --git a/test/unit_tests/sample_suite_files/another_suite.xml b/test/unit_tests/sample_suite_files/another_suite.xml new file mode 100644 index 00000000..72346933 --- /dev/null +++ b/test/unit_tests/sample_suite_files/another_suite.xml @@ -0,0 +1,10 @@ + + + + + + another_scheme + + more_scheme + + diff --git a/test/unit_tests/sample_suite_files/another_suite2.xml b/test/unit_tests/sample_suite_files/another_suite2.xml new file mode 100644 index 00000000..def97177 --- /dev/null +++ b/test/unit_tests/sample_suite_files/another_suite2.xml @@ -0,0 +1,16 @@ + + + + + + another_scheme + + more_scheme + + + + another_scheme + + more_scheme + + diff --git a/test/unit_tests/sample_suite_files/nested_full_suite.xml b/test/unit_tests/sample_suite_files/nested_full_suite.xml new file mode 100644 index 00000000..2979f3ff --- /dev/null +++ b/test/unit_tests/sample_suite_files/nested_full_suite.xml @@ -0,0 +1,10 @@ + + + + + g1_scheme1 + + + + + diff --git a/test/unit_tests/sample_suite_files/subsuite1.xml b/test/unit_tests/sample_suite_files/subsuite1.xml new file mode 100644 index 00000000..c58ed752 --- /dev/null +++ b/test/unit_tests/sample_suite_files/subsuite1.xml @@ -0,0 +1,7 @@ + + + + + scheme_subsuite1 + + diff --git a/test/unit_tests/sample_suite_files/subsuite_inline.xml b/test/unit_tests/sample_suite_files/subsuite_inline.xml new file mode 100644 index 00000000..706ea801 --- /dev/null +++ b/test/unit_tests/sample_suite_files/subsuite_inline.xml @@ -0,0 +1,9 @@ + + + + + scheme1i + scheme2i + scheme1i + + diff --git a/test/unit_tests/sample_suite_files/suite_bad_v2_duplicate_group.xml b/test/unit_tests/sample_suite_files/suite_bad_v2_duplicate_group.xml new file mode 100644 index 00000000..8ea72077 --- /dev/null +++ b/test/unit_tests/sample_suite_files/suite_bad_v2_duplicate_group.xml @@ -0,0 +1,16 @@ + + + + + + effr_pre + + + scheme9 + + + scheme3 + + + + diff --git a/test/unit_tests/sample_suite_files/suite_bad_v2_suite_tag.xml b/test/unit_tests/sample_suite_files/suite_bad_v2_suite_tag.xml new file mode 100644 index 00000000..6bc4f424 --- /dev/null +++ b/test/unit_tests/sample_suite_files/suite_bad_v2_suite_tag.xml @@ -0,0 +1,7 @@ + + + + + subsuite_inline + + diff --git a/test/unit_tests/sample_suite_files/suite_bad_version01.xml b/test/unit_tests/sample_suite_files/suite_bad_version01.xml new file mode 100644 index 00000000..ecee0c63 --- /dev/null +++ b/test/unit_tests/sample_suite_files/suite_bad_version01.xml @@ -0,0 +1,8 @@ + + + + + scheme1 + scheme2 + + diff --git a/test/unit_tests/sample_suite_files/suite_bad_version02.xml b/test/unit_tests/sample_suite_files/suite_bad_version02.xml new file mode 100644 index 00000000..55aff67a --- /dev/null +++ b/test/unit_tests/sample_suite_files/suite_bad_version02.xml @@ -0,0 +1,8 @@ + + + + + scheme1 + scheme2 + + diff --git a/test/unit_tests/sample_suite_files/suite_bad_version03.xml b/test/unit_tests/sample_suite_files/suite_bad_version03.xml new file mode 100644 index 00000000..794bfe7b --- /dev/null +++ b/test/unit_tests/sample_suite_files/suite_bad_version03.xml @@ -0,0 +1,8 @@ + + + + + scheme1 + scheme2 + + diff --git a/test/unit_tests/sample_suite_files/suite_bad_version04.xml b/test/unit_tests/sample_suite_files/suite_bad_version04.xml new file mode 100644 index 00000000..aaaab154 --- /dev/null +++ b/test/unit_tests/sample_suite_files/suite_bad_version04.xml @@ -0,0 +1,8 @@ + + + + + scheme1 + scheme2 + + diff --git a/test/unit_tests/sample_suite_files/suite_good_v1_test01.xml b/test/unit_tests/sample_suite_files/suite_good_v1_test01.xml new file mode 100644 index 00000000..0eee366b --- /dev/null +++ b/test/unit_tests/sample_suite_files/suite_good_v1_test01.xml @@ -0,0 +1,8 @@ + + + + + scheme1 + scheme2 + + diff --git a/test/unit_tests/sample_suite_files/suite_good_v1_test02.xml b/test/unit_tests/sample_suite_files/suite_good_v1_test02.xml new file mode 100644 index 00000000..3d355cc5 --- /dev/null +++ b/test/unit_tests/sample_suite_files/suite_good_v1_test02.xml @@ -0,0 +1,11 @@ + + + + + scheme1 + scheme2 + scheme3 + scheme2 + scheme1 + + diff --git a/test/unit_tests/sample_suite_files/suite_good_v2_test01.xml b/test/unit_tests/sample_suite_files/suite_good_v2_test01.xml new file mode 100644 index 00000000..73c30732 --- /dev/null +++ b/test/unit_tests/sample_suite_files/suite_good_v2_test01.xml @@ -0,0 +1,9 @@ + + + + + scheme5 + + scheme9 + + diff --git a/test/unit_tests/sample_suite_files/suite_good_v2_test01_exp.xml b/test/unit_tests/sample_suite_files/suite_good_v2_test01_exp.xml new file mode 100644 index 00000000..dcecf218 --- /dev/null +++ b/test/unit_tests/sample_suite_files/suite_good_v2_test01_exp.xml @@ -0,0 +1,11 @@ + + + + + scheme5 + scheme1i + scheme2i + scheme1i + scheme9 + + diff --git a/test/unit_tests/sample_suite_files/suite_good_v2_test02.xml b/test/unit_tests/sample_suite_files/suite_good_v2_test02.xml new file mode 100644 index 00000000..c7b5b8c9 --- /dev/null +++ b/test/unit_tests/sample_suite_files/suite_good_v2_test02.xml @@ -0,0 +1,10 @@ + + + + + + scheme6 + + + + diff --git a/test/unit_tests/sample_suite_files/suite_good_v2_test02_exp.xml b/test/unit_tests/sample_suite_files/suite_good_v2_test02_exp.xml new file mode 100644 index 00000000..6b100283 --- /dev/null +++ b/test/unit_tests/sample_suite_files/suite_good_v2_test02_exp.xml @@ -0,0 +1,13 @@ + + + + + + scheme6 + + + another_scheme + + more_scheme + + diff --git a/test/unit_tests/sample_suite_files/suite_good_v2_test03.xml b/test/unit_tests/sample_suite_files/suite_good_v2_test03.xml new file mode 100644 index 00000000..eff15298 --- /dev/null +++ b/test/unit_tests/sample_suite_files/suite_good_v2_test03.xml @@ -0,0 +1,19 @@ + + + + + scheme13 + + effr_pre + + + main_calc + + + main_post + + + + + + diff --git a/test/unit_tests/sample_suite_files/suite_good_v2_test03_exp.xml b/test/unit_tests/sample_suite_files/suite_good_v2_test03_exp.xml new file mode 100644 index 00000000..5f9a9987 --- /dev/null +++ b/test/unit_tests/sample_suite_files/suite_good_v2_test03_exp.xml @@ -0,0 +1,30 @@ + + + + + scheme13 + + effr_pre + + + main_calc + + + main_post + + + another_scheme + + more_scheme + + another_scheme + + more_scheme + + + g1_scheme1 + + + scheme_subsuite1 + + diff --git a/test/unit_tests/sample_suite_files/suite_good_v2_test04.xml b/test/unit_tests/sample_suite_files/suite_good_v2_test04.xml new file mode 100644 index 00000000..abb87008 --- /dev/null +++ b/test/unit_tests/sample_suite_files/suite_good_v2_test04.xml @@ -0,0 +1,18 @@ + + + + + + effr_pre + + + main_calc + + + main_post + + + + + + diff --git a/test/unit_tests/sample_suite_files/suite_good_v2_test04_exp.xml b/test/unit_tests/sample_suite_files/suite_good_v2_test04_exp.xml new file mode 100644 index 00000000..af103e7d --- /dev/null +++ b/test/unit_tests/sample_suite_files/suite_good_v2_test04_exp.xml @@ -0,0 +1,26 @@ + + + + + + effr_pre + + + main_calc + + + main_post + + + another_scheme + + more_scheme + + another_scheme + + more_scheme + + + scheme_subsuite1 + + diff --git a/test/unit_tests/sample_suite_files/suite_missing_file.xml b/test/unit_tests/sample_suite_files/suite_missing_file.xml new file mode 100644 index 00000000..8c7b85e6 --- /dev/null +++ b/test/unit_tests/sample_suite_files/suite_missing_file.xml @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/test/unit_tests/sample_suite_files/suite_missing_group.xml b/test/unit_tests/sample_suite_files/suite_missing_group.xml new file mode 100644 index 00000000..a33078e6 --- /dev/null +++ b/test/unit_tests/sample_suite_files/suite_missing_group.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/test/unit_tests/sample_suite_files/suite_missing_loaded_suite.xml b/test/unit_tests/sample_suite_files/suite_missing_loaded_suite.xml new file mode 100644 index 00000000..cf21a590 --- /dev/null +++ b/test/unit_tests/sample_suite_files/suite_missing_loaded_suite.xml @@ -0,0 +1,16 @@ + + + + + + scheme23 + + + scheme9 + + + scheme3 + + + + diff --git a/test/unit_tests/sample_suite_files/suite_missing_version.xml b/test/unit_tests/sample_suite_files/suite_missing_version.xml new file mode 100644 index 00000000..463c56d9 --- /dev/null +++ b/test/unit_tests/sample_suite_files/suite_missing_version.xml @@ -0,0 +1,8 @@ + + + + + scheme1 + scheme2 + + diff --git a/test/unit_tests/sample_suite_files/suite_recurse_level2.xml b/test/unit_tests/sample_suite_files/suite_recurse_level2.xml new file mode 100644 index 00000000..35fed8b2 --- /dev/null +++ b/test/unit_tests/sample_suite_files/suite_recurse_level2.xml @@ -0,0 +1,10 @@ + + + + + scheme13 + scheme3 + + scheme43 + + diff --git a/test/unit_tests/sample_suite_files/suite_recurse_level2a.xml b/test/unit_tests/sample_suite_files/suite_recurse_level2a.xml new file mode 100644 index 00000000..2e018e39 --- /dev/null +++ b/test/unit_tests/sample_suite_files/suite_recurse_level2a.xml @@ -0,0 +1,10 @@ + + + + + scheme13 + scheme3 + scheme43 + + + diff --git a/test/unit_tests/sample_suite_files/suite_recurse_level3.xml b/test/unit_tests/sample_suite_files/suite_recurse_level3.xml new file mode 100644 index 00000000..f38a9d8f --- /dev/null +++ b/test/unit_tests/sample_suite_files/suite_recurse_level3.xml @@ -0,0 +1,10 @@ + + + + + scheme13 + scheme3 + + scheme43 + + diff --git a/test/unit_tests/sample_suite_files/suite_recurse_level3a.xml b/test/unit_tests/sample_suite_files/suite_recurse_level3a.xml new file mode 100644 index 00000000..a11182dc --- /dev/null +++ b/test/unit_tests/sample_suite_files/suite_recurse_level3a.xml @@ -0,0 +1,10 @@ + + + + + scheme13 + scheme3 + scheme43 + + + diff --git a/test/unit_tests/sample_suite_files/suite_recurse_top1.xml b/test/unit_tests/sample_suite_files/suite_recurse_top1.xml new file mode 100644 index 00000000..8e8c4f1a --- /dev/null +++ b/test/unit_tests/sample_suite_files/suite_recurse_top1.xml @@ -0,0 +1,18 @@ + + + + + scheme13 + + scheme23 + + + scheme9 + + + scheme3 + + + scheme43 + + diff --git a/test/unit_tests/sample_suite_files/suite_recurse_top2.xml b/test/unit_tests/sample_suite_files/suite_recurse_top2.xml new file mode 100644 index 00000000..87ce7057 --- /dev/null +++ b/test/unit_tests/sample_suite_files/suite_recurse_top2.xml @@ -0,0 +1,18 @@ + + + + + scheme13 + + scheme23 + + + scheme9 + + + scheme3 + + scheme43 + + + diff --git a/test/unit_tests/test_sdf.py b/test/unit_tests/test_sdf.py new file mode 100644 index 00000000..25a49603 --- /dev/null +++ b/test/unit_tests/test_sdf.py @@ -0,0 +1,525 @@ +#! /usr/bin/env python3 +""" +----------------------------------------------------------------------- + Description: Contains unit tests for parsing Suite Definition Files (SDFs) + in scripts/parse_tools/xml_tools.py + + Assumptions: + + Command line arguments: none + + Usage: python3 test_sdf.py # run the unit tests +----------------------------------------------------------------------- +""" + +import filecmp +import glob +import logging +import os +import sys +import unittest +import xml.etree.ElementTree as ET + +_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) +_SCRIPTS_DIR = os.path.abspath(os.path.join(_TEST_DIR, os.pardir, + os.pardir, "scripts")) +_SAMPLE_FILES_DIR = os.path.join(_TEST_DIR, "sample_suite_files") +_PRE_TMP_DIR = os.path.join(_TEST_DIR, "tmp") +_TMP_DIR = os.path.join(_PRE_TMP_DIR, "suite_files") + +if not os.path.exists(_SCRIPTS_DIR): + raise ImportError(f"Cannot find scripts directory, {_SCRIPTS_DIR}") + +sys.path.append(_SCRIPTS_DIR) + +# pylint: disable=wrong-import-position +from parse_tools import init_log +from parse_tools import read_xml_file, validate_xml_file, write_xml_file +from parse_tools import find_schema_version, expand_nested_suites +# pylint: enable=wrong-import-position + +class SDFParseTestCase(unittest.TestCase): + + """Tests for `expand_nested_suites` and related functions.""" + + logger = None + + @classmethod + def setUpClass(cls): + """Clean output directory (tmp) before running tests""" + # Does "tmp" directory exist? If not then create it: + if not os.path.exists(_PRE_TMP_DIR): + os.makedirs(_PRE_TMP_DIR) + # end if + + # We need a logger + cls.logger = init_log(cls.__name__, level=logging.WARNING) + + #Does "tmp" directory exist? If not then create it: + # Ensure the "tmp/suite_files" directory exists and is empty + if os.path.exists(_TMP_DIR): + # Clear out all files: + for fpath in glob.iglob(os.path.join(_TMP_DIR, '*.*')): + if os.path.exists(fpath): + os.remove(fpath) + # End if + # End for + else: + os.makedirs(_TMP_DIR) + # end if + + # Run inherited setup method: + super().setUpClass() + + @classmethod + def get_logger(cls): + return cls.logger + + @classmethod + def compare_text(cls, name, txt1, txt2, typ): + """Compare two XML text or tail items (which may be None). + Return None if items match, otherwise, return an error string""" + res = None + if txt1 and txt2: + if txt1.strip() != txt2.strip(): + res = f"{name} {typ}, '{txt1}', does not match {typ}, '{txt2}'" + # end if + elif txt1: + res = f"{name} {typ} is missing from string2" + elif txt2: + res = f"{name} {typ} is missing from string1" + else: + res = None + # end if + return res + + @classmethod + def xml_diff(cls, xt1, xt2): + """ + Compares two xml etrees, xt1 and xt2 + Return None if the trees match, otherwise, return a difference string + """ + + diffs = [] + # First, compare the XML tags + if xt1.tag != xt2.tag: + diffs.append(f"Tags do not match: {xt1.tag} != {xt2.tag}") + else: + # Compare the attributes + for name, value in xt1.attrib.items(): + if name not in xt2.attrib: + diffs.append(f"xt1 attribute, {name}, is missing in xt2") + else: + xt2v = xt2.attrib.get(name) + if xt2v != value: + diffs.append(f"Attributes for {name} do not match: {str(value)} != {str(xt2v)}") + # end if + # end if + # end for + for name in xt2.attrib.keys(): + if name not in xt1.attrib: + diffs.append(f"xt2 attribute, {name}, is missing in xt1") + # end if + # end for + # Compare the text bodies (if any) + tdiff = cls.compare_text(xt1.tag, xt1.text, xt2.text, "text") + if tdiff: + diffs.append(tdiff) + # end if + tdiff = cls.compare_text(xt1.tag, xt1.tail, xt2.tail, "tail") + if tdiff: + diffs.append(tdiff) + # end if + # Compare children + if len(xt1) != len(xt2): + diffs.append(f"Number of children length differs, {len(xt1)} != {len(xt2)}") + else: + for child1, child2 in zip(xt1, xt2): + kid_diffs = cls.xml_diff(child1, child2) + if kid_diffs: + diffs.extend(kid_diffs) + # end if + # end for + # end if + # end if + return diffs + + def test_xml_diff(self): + """Test that xml_diff catches xml differences""" + root1 = ET.fromstring("item") + root2 = ET.fromstring("item") + diffs = self.xml_diff(root1, root2) + self.assertTrue(diffs) + self.assertEqual(len(diffs), 1) + self.assertTrue("Tags do not match" in diffs[0], + msg="tag1 should not match taga") + root1 = ET.fromstring("item1") + root2 = ET.fromstring("item2") + diffs = self.xml_diff(root1, root2) + self.assertTrue(diffs) + self.assertEqual(len(diffs), 1) + self.assertTrue("does not match" in diffs[0], + msg="item1 should not match item2") + root1 = ET.fromstring('item1') + root2 = ET.fromstring('item1') + diffs = self.xml_diff(root1, root2) + self.assertTrue(diffs) + self.assertEqual(len(diffs), 3) + self.assertTrue("Attributes for" in diffs[0] and "do not match" in diffs[0], + msg="attrib1 values should not match") + self.assertTrue("xt1 attribute, attrib2, is missing in xt2" in diffs[1], + msg=f"attrib2 is missing in root2") + self.assertTrue("xt2 attribute, attrib3, is missing in xt1" in diffs[2], + msg=f"attrib3 is missing in root1") + root1 = ET.fromstring('') + root2 = ET.fromstring('') + diffs = self.xml_diff(root1, root2) + self.assertEqual(len(diffs), 1) + self.assertTrue("Attributes for" in diffs[0] and "do not match" in diffs[0], + msg=f"attrib2 values should not match") + + def test_good_v1_sdf(self): + """Test that the parser recognizes a V1 SDF and parses it correctly + """ + num_tests = 2 + header = "Test of parsing of good V1 SDF" + for test_num in range(num_tests): + # Setup + testname = f"suite_good_v1_test{test_num+1:{0}{2}}" + source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") + compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") + logger = self.get_logger() + # Exercise + _, xml_root = read_xml_file(source, logger) + schema_version = find_schema_version(xml_root) + self.assertEqual(schema_version[0], 1) + self.assertEqual(schema_version[1], 0) + res = validate_xml_file(source, 'suite', schema_version, logger, + error_on_noxmllint=True) + self.assertTrue(res) + write_xml_file(xml_root, compare, logger) + amsg = f"{compare} does not exist" + self.assertTrue(os.path.exists(compare), msg=amsg) + _, compare_root = read_xml_file(compare, logger) + diffs = self.xml_diff(xml_root, compare_root) + lsep = '\n' + amsg = f"{source} does not match {compare}\n{lsep.join(diffs)}" + self.assertFalse(diffs, msg=amsg) + # end for + + def test_good_v2_sdf_01(self): + """Test that the parser recognizes a V2 SDF and parses and + expands it correctly + """ + header = "Test of parsing of good V2 SDF" + # Setup + testname = "suite_good_v2_test01" + source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") + source_exp = os.path.join(_SAMPLE_FILES_DIR, f"{testname}_exp.xml") + compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") + logger = self.get_logger() + # Exercise + _, xml_root = read_xml_file(source, logger) + schema_version = find_schema_version(xml_root) + self.assertEqual(schema_version[0], 2) + self.assertEqual(schema_version[1], 0) + expand_nested_suites(xml_root, _SAMPLE_FILES_DIR, logger=logger) + write_xml_file(xml_root, compare, logger) + res = validate_xml_file(source, 'suite', schema_version, logger, + error_on_noxmllint=True) + self.assertTrue(res) + amsg = f"{compare} does not exist" + self.assertTrue(os.path.exists(compare), msg=amsg) + _, xml_root = read_xml_file(source_exp, logger) + _, compare_root = read_xml_file(compare, logger) + diffs = self.xml_diff(xml_root, compare_root) + lsep = '\n' + amsg = f"{source_exp} does not match {compare}\n{lsep.join(diffs)}" + self.assertFalse(diffs, msg=amsg) + + def test_good_v2_sdf_02(self): + """Test that the parser recognizes a V2 SDF and parses and + expands it correctly + """ + header = "Test of parsing of good V2 SDF" + # Setup + testname = "suite_good_v2_test02" + source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") + source_exp = os.path.join(_SAMPLE_FILES_DIR, f"{testname}_exp.xml") + compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") + logger = self.get_logger() + # Exercise + _, xml_root = read_xml_file(source, logger) + schema_version = find_schema_version(xml_root) + self.assertEqual(schema_version[0], 2) + self.assertEqual(schema_version[1], 0) + expand_nested_suites(xml_root, _SAMPLE_FILES_DIR, logger=logger) + write_xml_file(xml_root, compare, logger) + res = validate_xml_file(source, 'suite', schema_version, logger, + error_on_noxmllint=True) + self.assertTrue(res) + amsg = f"{compare} does not exist" + self.assertTrue(os.path.exists(compare), msg=amsg) + _, xml_root = read_xml_file(source_exp, logger) + _, compare_root = read_xml_file(compare, logger) + diffs = self.xml_diff(xml_root, compare_root) + lsep = '\n' + amsg = f"{source_exp} does not match {compare}\n{lsep.join(diffs)}" + self.assertFalse(diffs, msg=amsg) + + def test_good_v2_sdf_03(self): + """Test that the parser recognizes a V2 SDF and parses and + expands it correctly + """ + header = "Test of parsing of good V2 SDF" + # Setup + testname = "suite_good_v2_test03" + source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") + source_exp = os.path.join(_SAMPLE_FILES_DIR, f"{testname}_exp.xml") + compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") + logger = self.get_logger() + # Exercise + _, xml_root = read_xml_file(source, logger) + schema_version = find_schema_version(xml_root) + self.assertEqual(schema_version[0], 2) + self.assertEqual(schema_version[1], 0) + expand_nested_suites(xml_root, _SAMPLE_FILES_DIR, logger=logger) + write_xml_file(xml_root, compare, logger) + res = validate_xml_file(source, 'suite', schema_version, logger, + error_on_noxmllint=True) + self.assertTrue(res) + amsg = f"{compare} does not exist" + self.assertTrue(os.path.exists(compare), msg=amsg) + _, xml_root = read_xml_file(source_exp, logger) + _, compare_root = read_xml_file(compare, logger) + diffs = self.xml_diff(xml_root, compare_root) + lsep = '\n' + amsg = f"{source_exp} does not match {compare}\n{lsep.join(diffs)}" + self.assertFalse(diffs, msg=amsg) + + def test_good_v2_sdf_04(self): + """Test that the parser recognizes a V2 SDF and parses and + expands it correctly + """ + header = "Test of parsing of good V2 SDF" + # Setup + testname = "suite_good_v2_test04" + source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") + source_exp = os.path.join(_SAMPLE_FILES_DIR, f"{testname}_exp.xml") + compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") + logger = self.get_logger() + # Exercise + _, xml_root = read_xml_file(source, logger) + schema_version = find_schema_version(xml_root) + self.assertEqual(schema_version[0], 2) + self.assertEqual(schema_version[1], 0) + expand_nested_suites(xml_root, _SAMPLE_FILES_DIR, logger=logger) + write_xml_file(xml_root, compare, logger) + res = validate_xml_file(source, 'suite', schema_version, logger, + error_on_noxmllint=True) + self.assertTrue(res) + amsg = f"{compare} does not exist" + self.assertTrue(os.path.exists(compare), msg=amsg) + _, xml_root = read_xml_file(source_exp, logger) + _, compare_root = read_xml_file(compare, logger) + diffs = self.xml_diff(xml_root, compare_root) + lsep = '\n' + amsg = f"{source_exp} does not match {compare}\n{lsep.join(diffs)}" + self.assertFalse(diffs, msg=amsg) + + def test_bad_v2_suite_tag_sdf(self): + """Test that verification system recognizes a misplaced suite tag""" + header = "Test trapping of version attribute on a v2 suite tag" + # Setup + testname = f"suite_bad_v2_suite_tag" + source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") + logger = self.get_logger() + # Exercise + _, xml_root = read_xml_file(source, logger) + schema_version = find_schema_version(xml_root) + res = validate_xml_file(source, 'suite', schema_version, logger, + error_on_noxmllint=True) + self.assertFalse(res, + msg=" tag should not be allowed inside a suite") + + def test_bad_v2_suite_duplicate_group1(self): + """Test that verification system recognizes a duplicate group name""" + header = "Test trapping of expanded suite duplicate group name" + # Setup + testname = f"suite_bad_v2_duplicate_group" + source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") + compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") + logger = self.get_logger() + # Exercise + _, xml_root = read_xml_file(source, logger) + schema_version = find_schema_version(xml_root) + self.assertEqual(schema_version[0], 2) + self.assertEqual(schema_version[1], 0) + res = validate_xml_file(source, 'suite', schema_version, logger, + error_on_noxmllint=True) + self.assertTrue(res, msg="Initial suite file should be valid") + with self.assertRaises(Exception) as context: + expand_nested_suites(xml_root, _SAMPLE_FILES_DIR, logger=logger) + write_xml_file(xml_root, compare, logger) + # end with + emsg = "Duplicate group name, group1, from subsuite_1" + fmsg = str(context.exception) + self.assertTrue(emsg in fmsg, msg=fmsg) + if not emsg in fmsg: + raise context + + def test_bad_v2_suite_missing_group(self): + """Test that verification system recognizes a missing group name""" + header = "Test trapping of expanded suite missing group name" + # Setup + testname = f"suite_missing_group" + source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") + compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") + logger = self.get_logger() + # Exercise + _, xml_root = read_xml_file(source, logger) + schema_version = find_schema_version(xml_root) + self.assertEqual(schema_version[0], 2) + self.assertEqual(schema_version[1], 0) + res = validate_xml_file(source, 'suite', schema_version, logger, + error_on_noxmllint=True) + self.assertTrue(res, msg="Initial suite file should be valid") + with self.assertRaises(Exception) as context: + expand_nested_suites(xml_root, _SAMPLE_FILES_DIR, logger=logger) + write_xml_file(xml_root, compare, logger) + # end with + emsg = "Nested suite subsuite_1, group group2, not found" + fmsg = str(context.exception) + self.assertTrue(emsg in fmsg, msg=fmsg) + + def test_bad_v2_suite_missing_file(self): + """Test that verification system recognizes a missing file argument""" + header = "Test trapping of missing file for nested suite" + # Setup + testname = f"suite_missing_file" + source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") + compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") + logger = self.get_logger() + # Exercise + _, xml_root = read_xml_file(source, logger) + schema_version = find_schema_version(xml_root) + self.assertEqual(schema_version[0], 2) + self.assertEqual(schema_version[1], 0) + res = validate_xml_file(source, 'suite', schema_version, logger, + error_on_noxmllint=True) + self.assertFalse(res, msg="Initial suite file should be invalid") + + def test_bad_v2_suite_missing_loaded_suite(self): + """Test that verification system recognizes a missing suite loaded + from another file""" + header = "Test trapping of expanded suite missing a subsuite in a different file" + # Setup + testname = f"suite_missing_loaded_suite" + source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") + compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") + logger = self.get_logger() + # Exercise + _, xml_root = read_xml_file(source, logger) + schema_version = find_schema_version(xml_root) + self.assertEqual(schema_version[0], 2) + self.assertEqual(schema_version[1], 0) + res = validate_xml_file(source, 'suite', schema_version, logger, + error_on_noxmllint=True) + self.assertTrue(res, msg="Initial suite file should be valid") + with self.assertRaises(Exception) as context: + expand_nested_suites(xml_root, _SAMPLE_FILES_DIR, logger=logger) + write_xml_file(xml_root, compare, logger) + # end with + emsg = "Nested suite v12_suite, group main_group, not found in file" + fmsg = str(context.exception) + self.assertTrue(emsg in fmsg, msg=fmsg) + + def test_bad_v2_suite_infinite_group_recursion(self): + """Test that verification system recognizes infinite recursion when + including at the group level""" + header = "Test trapping of expanded suite with infinite group recursion" + # Setup + testname = f"suite_recurse_top1" + source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") + compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") + logger = self.get_logger() + # Exercise + _, xml_root = read_xml_file(source, logger) + schema_version = find_schema_version(xml_root) + self.assertEqual(schema_version[0], 2) + self.assertEqual(schema_version[1], 0) + res = validate_xml_file(source, 'suite', schema_version, logger, + error_on_noxmllint=True) + self.assertTrue(res, msg="Initial suite file should be valid") + with self.assertRaises(Exception) as context: + expand_nested_suites(xml_root, _SAMPLE_FILES_DIR, logger=logger) + write_xml_file(xml_root, compare, logger) + # end with + emsg = "Nested suite v12_suite, group main_group, not found in file" + fmsg = str(context.exception) + self.assertTrue(emsg in fmsg, msg=fmsg) + + def test_bad_v2_suite_infinite_suite_recursion(self): + """Test that verification system recognizes infinite recursion when + including at the imported suite level""" + header = "Test trapping of expanded suite with infinite suite recursion" + # Setup + testname = f"suite_recurse_top2" + source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") + compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") + logger = self.get_logger() + # Exercise + _, xml_root = read_xml_file(source, logger) + schema_version = find_schema_version(xml_root) + self.assertEqual(schema_version[0], 2) + self.assertEqual(schema_version[1], 0) + res = validate_xml_file(source, 'suite', schema_version, logger, + error_on_noxmllint=True) + self.assertTrue(res, msg="Initial suite file should be valid") + with self.assertRaises(Exception) as context: + expand_nested_suites(xml_root, _SAMPLE_FILES_DIR, logger=logger) + write_xml_file(xml_root, compare, logger) + # end with + emsg = "Duplicate group name, group1, from l2_suite" + fmsg = str(context.exception) + self.assertTrue(emsg in fmsg, msg=fmsg) + + def test_bad_schema_version(self): + """Test that verification system recognizes a bad version entry""" + num_tests = 4 + header = "Test trapping of invalid SDF version" + exc_strings = ["Format must be .", + "Format must be .", + "Major version must be at least 1", + "Minor version must be non-negative"] + for test_num in range(num_tests): + # Setup + testname = f"suite_bad_version{test_num+1:{0}{2}}" + source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") + logger = self.get_logger() + # Exercise + with self.assertRaises(Exception) as context: + _, xml_root = read_xml_file(source, logger) + schema_version = find_schema_version(xml_root) + # end with + # Check exception for expected error messages + exp_str = str(context.exception) + self.assertTrue(exc_strings[test_num] in exp_str, + msg=f"Bad exception in test {test_num + 1}, '{exp_str}'") + # end for + + def test_missing_schema_version(self): + """Test that verification system recognizes a missing version num""" + header = "Test trapping of missing SDF version" + # Setup + testname = f"suite_missing_version" + source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") + logger = self.get_logger() + # Exercise + with self.assertRaises(Exception) as context: + _, xml_root = read_xml_file(source, logger) + schema_version = find_schema_version(xml_root) + # end with + # Check exception for expected error messages + self.assertTrue("version attribute required" in str(context.exception), + msg=f"Bad exception for missing suite version")