Switched tests from stdout to log-file parsing

Signed-off-by: Sebastian Schleemilch <[email protected]>

src/vss_tools/exporters/ddsidl.py

@@ -226,8 +226,8 @@ def export_node(node: VSSNode, generate_all_idl_features: bool) -> None:
                 idl_file_buffer.append("};")
                 allowed_values = str(allowed)
             else:
-                print(
-                    f"Warning: VSS2IDL can only handle allowed values for string type, "
+                log.warning(
+                    f"VSS2IDL can only handle allowed values for string type, "
                     f"signal {node.name} has type {datatype}"
                 )
 

tests/vspec/test_allowed/test_allowed.py

@@ -18,13 +18,14 @@
 def run_exporter(exporter, argument, tmp_path):
     spec = HERE / "test.vspec"
     output = tmp_path / f"out.{exporter}"
-    cmd = f"vspec export {exporter}{argument} --vspec {spec} "
+    log = tmp_path / "log.txt"
+    cmd = f"vspec --log-file {log} export {exporter}{argument} --vspec {spec} "
     if exporter in ["apigear"]:
         cmd += f"--output-dir {output}"
     else:
         cmd += f"--output {output}"
 
-    process = subprocess.run(cmd.split(), capture_output=True, text=True)
+    process = subprocess.run(cmd.split())
     assert process.returncode == 0
     expected = HERE / f"expected.{exporter}"
     if exporter in ["apigear"]:
@@ -37,7 +38,7 @@ def run_exporter(exporter, argument, tmp_path):
     # ddsidl can not handle float and integer
     # Some other tools ignore "allowed" all together
     if exporter in ["ddsidl"]:
-        assert "can only handle allowed values for string type" in process.stdout
+        assert "can only handle allowed values for string type" in log.read_text()
 
 
 def test_allowed(tmp_path):

tests/vspec/test_datatypes_error/test_datatypes_error.py

@@ -6,7 +6,6 @@
 #
 # SPDX-License-Identifier: MPL-2.0
 
-import os
 import subprocess
 from pathlib import Path
 
@@ -18,21 +17,23 @@
 def test_datatype_error(tmp_path):
     spec = HERE / "test.vspec"
     output = tmp_path / "out.json"
-    cmd = f"vspec export json --pretty -u {TEST_UNITS} -q {TEST_QUANT} --vspec {spec} --output {output}"
-    env = os.environ.copy()
-    env["COLUMNS"] = "200"
-    process = subprocess.run(cmd.split(), capture_output=True, text=True, env=env)
+    log = tmp_path / "log.txt"
+    cmd = (
+        f"vspec --log-file {log} export json --pretty -u {TEST_UNITS} -q {TEST_QUANT} --vspec {spec} --output {output}"
+    )
+    process = subprocess.run(cmd.split())
     assert process.returncode != 0
 
     print(process.stdout)
-    assert "'uint7' is not a valid datatype" in process.stdout
+    assert "'uint7' is not a valid datatype" in log.read_text()
 
 
 def test_datatype_branch(tmp_path):
     spec = HERE / "test_datatype_branch.vspec"
     output = tmp_path / "out.json"
-    cmd = f"vspec export json --pretty -u {TEST_UNITS} -q {TEST_QUANT} --vspec {spec} --output {output} --strict"
-    process = subprocess.run(cmd.split(), capture_output=True, text=True)
+    log = tmp_path / "log.txt"
+    cmd = f"vspec --log-file {log} export json --pretty -u {TEST_UNITS} -q {TEST_QUANT}"
+    cmd += f" --vspec {spec} --output {output} --strict"
+    process = subprocess.run(cmd.split())
     assert process.returncode != 0
-    print(process.stdout)
-    assert "Unknown extra attribute: 'A':'datatype'" in process.stdout
+    assert "Unknown extra attribute: 'A':'datatype'" in log.read_text()

tests/vspec/test_default_unit/test_default_units.py

@@ -24,10 +24,11 @@ def run_unit(vspec_file, unit_argument, expected_file, tmp_path):
 
 def run_unit_error(vspec_file, unit_argument, check_message, tmp_path):
     output = tmp_path / "out.json"
-    cmd = f"vspec export json --pretty --vspec {vspec_file} {unit_argument} --output {output}"
+    log = tmp_path / "log.txt"
+    cmd = f"vspec --log-file {log} export json --pretty --vspec {vspec_file} {unit_argument} --output {output}"
     process = subprocess.run(cmd.split(), capture_output=True, text=True)
     assert process.returncode != 0
-    assert check_message in process.stdout
+    assert check_message in log.read_text()
 
 
 def test_default_ok(tmp_path):

tests/vspec/test_extended/test_extended.py

@@ -52,9 +52,10 @@ def test_extended_ok(
     cmd += f" -q {TEST_QUANT} {extended_args} -s {TEST_FILE} -o {output}"
 
     # Make sure there is no line break that affects compare
-    os.environ["COLUMNS"] = "120"
+    env = os.environ.copy()
+    env["COLUMNS"] = "120"
 
-    process = subprocess.run(cmd.split(), capture_output=True, check=True, text=True)
+    process = subprocess.run(cmd.split(), capture_output=True, check=True, text=True, env=env)
 
     if known_extended is not None:
         print(process.stdout)
@@ -84,8 +85,9 @@ def test_extended_error(extended_args: str, tmp_path):
     cmd += f" -q {TEST_QUANT} {extended_args} -s {TEST_FILE} -o {output}"
 
     # Make sure there is no line break that affects compare
-    os.environ["COLUMNS"] = "120"
+    env = os.environ.copy()
+    env["COLUMNS"] = "120"
 
-    process = subprocess.run(cmd.split(), stdout=subprocess.PIPE, text=True, stderr=subprocess.STDOUT)
+    process = subprocess.run(cmd.split(), stdout=subprocess.PIPE, text=True, stderr=subprocess.STDOUT, env=env)
     assert process.returncode != 0
     assert "not allowed" in process.stdout

tests/vspec/test_faulty_type/test_faulty_type.py

@@ -17,9 +17,10 @@
 def test_error(tmp_path):
     spec = HERE / "test.vspec"
     output = tmp_path / "out.json"
-    cmd = f"vspec export json -u {TEST_UNITS} -q {TEST_QUANT} --vspec {spec} --output {output}"
-    process = subprocess.run(cmd.split(), capture_output=True, text=True)
+    log = tmp_path / "log.txt"
+    cmd = f"vspec --log-file {log} export json -u {TEST_UNITS} -q {TEST_QUANT} --vspec {spec} --output {output}"
+    process = subprocess.run(cmd.split())
     assert process.returncode != 0
-    print(process.stdout)
-    assert "input': 'bosch'" in process.stdout
-    assert "Input should be 'branch'" in process.stdout
+    log_content = log.read_text()
+    assert "input': 'bosch'" in log_content
+    assert "Input should be 'branch'" in log_content

tests/vspec/test_overlay/test_overlay.py

@@ -6,7 +6,6 @@
 #
 # SPDX-License-Identifier: MPL-2.0
 import filecmp
-import os
 import subprocess
 from pathlib import Path
 
@@ -48,29 +47,29 @@ def test_no_type(tmp_path):
 def test_overlay_error(tmp_path):
     overlay = HERE / "overlay_error.vspec"
     output = tmp_path / "out.json"
+    log = tmp_path / "log.txt"
     spec = HERE / "test.vspec"
-    cmd = f"vspec export json --pretty -u {TEST_UNITS}"
+    cmd = f"vspec --log-file {log} export json --pretty -u {TEST_UNITS}"
     cmd += f" -q {TEST_QUANT} -l {overlay} --vspec {spec} --output {output}"
-    env = os.environ.copy()
-    env["COLUMNS"] = "300"
-    process = subprocess.run(cmd.split(), capture_output=True, text=True, env=env)
+    process = subprocess.run(cmd.split())
     assert process.returncode != 0
-    print(process.stdout)
-    assert "'A.SignalXXXX' has 1 model error(s)" in process.stdout
-    assert "'type': 'missing'" in process.stdout
-    assert "datatype" in process.stdout
+    log_content = log.read_text()
+    assert "'A.SignalXXXX' has 1 model error(s)" in log_content
+    assert "'type': 'missing'" in log_content
+    assert "datatype" in log_content
 
 
 def test_overlay_branch_error(tmp_path):
     overlay = HERE / "overlay_implicit_branch_no_description.vspec"
     output = tmp_path / "out.json"
+    log = tmp_path / "log.txt"
     spec = HERE / "test.vspec"
-    cmd = f"vspec export json --pretty -u {TEST_UNITS}"
+    cmd = f"vspec --log-file {log} export json --pretty -u {TEST_UNITS}"
     cmd += f" -q {TEST_QUANT} -l {overlay} --vspec {spec} --output {output}"
 
-    process = subprocess.run(cmd.split(), capture_output=True, text=True)
+    process = subprocess.run(cmd.split())
     assert process.returncode != 0
-    print(process.stdout)
-    assert "'A.AB' has 1 model error(s)" in process.stdout
-    assert "'type': 'missing'" in process.stdout
-    assert "description" in process.stdout
+    log_content = log.read_text()
+    assert "'A.AB' has 1 model error(s)" in log_content
+    assert "'type': 'missing'" in log_content
+    assert "description" in log_content

tests/vspec/test_static_uids/test_static_uids.py

@@ -10,7 +10,6 @@
 # Convert vspec files to various other formats
 #
 
-import os
 import shlex
 import subprocess
 from pathlib import Path
@@ -197,77 +196,84 @@ def test_full_script(caplog: pytest.LogCaptureFixture, tmp_path):
 def test_semantic(caplog: pytest.LogCaptureFixture, validation_file: str, tmp_path):
     spec = HERE / "test_vspecs/test.vspec"
     output = tmp_path / "out.vspec"
+    log = tmp_path / "log.txt"
     validation = HERE / validation_file
-    args = f"vspec export id --vspec {spec} --output {output}"
+    args = f"vspec --log-file {log} export id --vspec {spec} --output {output}"
     args += f" --validate-static-uid {validation} -q {TEST_QUANT}"
-    process = subprocess.run(args.split(), capture_output=True, text=True)
-    assert "SEMANTIC NAME CHANGE" in process.stdout
+    subprocess.run(args.split())
+    assert "SEMANTIC NAME CHANGE" in log.read_text()
 
 
 def test_vss_path(caplog: pytest.LogCaptureFixture, tmp_path):
     spec = HERE / "test_vspecs/test_vss_path.vspec"
-    cmd = "vspec export id".split()
+    log = tmp_path / "log.txt"
+    cmd = f"vspec --log-file {log} export id".split()
     clas = shlex.split(get_cla_test(spec, tmp_path))
     cmd += clas
-    env = os.environ.copy()
-    env["COLUMNS"] = "200"
-    process = subprocess.run(cmd, capture_output=True, text=True, env=env)
-    assert "PATH CHANGE" in process.stdout
+    subprocess.run(cmd)
+    assert "PATH CHANGE" in log.read_text()
 
 
 def test_unit(caplog: pytest.LogCaptureFixture, tmp_path):
     spec = HERE / "test_vspecs/test_unit.vspec"
-    cmd = "vspec export id".split()
+    log = tmp_path / "log.txt"
+    cmd = f"vspec --log-file {log} export id".split()
     clas = shlex.split(get_cla_test(spec, tmp_path))
     cmd += clas
-    process = subprocess.run(cmd, capture_output=True, text=True)
-    assert "BREAKING CHANGE" in process.stdout
+    subprocess.run(cmd, capture_output=True, text=True)
+    assert "BREAKING CHANGE" in log.read_text()
 
 
 def test_datatype(caplog: pytest.LogCaptureFixture, tmp_path):
     spec = HERE / "test_vspecs/test_datatype.vspec"
-    cmd = "vspec export id".split()
+    log = tmp_path / "log.txt"
+    cmd = f"vspec --log-file {log} export id".split()
     clas = shlex.split(get_cla_test(spec, tmp_path))
     cmd += clas
-    process = subprocess.run(cmd, capture_output=True, text=True)
-    assert "BREAKING CHANGE" in process.stdout
+    subprocess.run(cmd)
+    assert "BREAKING CHANGE" in log.read_text()
 
 
 def test_name_datatype(caplog: pytest.LogCaptureFixture, tmp_path):
     spec = HERE / "test_vspecs/test_name_datatype.vspec"
-    cmd = "vspec export id".split()
+    log = tmp_path / "log.txt"
+    cmd = f"vspec --log-file {log} export id".split()
     clas = shlex.split(get_cla_test(spec, tmp_path))
     cmd += clas
-    process = subprocess.run(cmd, capture_output=True, text=True)
-    assert "ADDED ATTRIBUTE" in process.stdout
-    assert "DELETED ATTRIBUTE" in process.stdout
+    subprocess.run(cmd)
+    log_content = log.read_text()
+    assert "ADDED ATTRIBUTE" in log_content
+    assert "DELETED ATTRIBUTE" in log_content
 
 
 def test_deprecation(caplog: pytest.LogCaptureFixture, tmp_path):
     spec = HERE / "test_vspecs/test_deprecation.vspec"
-    cmd = "vspec export id".split()
+    log = tmp_path / "log.txt"
+    cmd = f"vspec --log-file {log} export id".split()
     clas = shlex.split(get_cla_test(spec, tmp_path))
     cmd += clas
-    process = subprocess.run(cmd, capture_output=True, text=True)
-    assert "DEPRECATION MSG CHANGE" in process.stdout
+    subprocess.run(cmd)
+    assert "DEPRECATION MSG CHANGE" in log.read_text()
 
 
 def test_description(caplog: pytest.LogCaptureFixture, tmp_path):
     spec = HERE / "test_vspecs/test_description.vspec"
-    cmd = "vspec export id".split()
+    log = tmp_path / "log.txt"
+    cmd = f"vspec --log-file {log} export id".split()
     clas = shlex.split(get_cla_test(spec, tmp_path))
     cmd += clas
-    process = subprocess.run(cmd, capture_output=True, text=True)
-    assert "DESCRIPTION MISMATCH" in process.stdout
+    subprocess.run(cmd)
+    assert "DESCRIPTION MISMATCH" in log.read_text()
 
 
 def test_added_attribute(caplog: pytest.LogCaptureFixture, tmp_path):
     spec = HERE / "test_vspecs/test_added_attribute.vspec"
-    cmd = "vspec export id".split()
+    log = tmp_path / "log.txt"
+    cmd = f"vspec --log-file {log} export id".split()
     clas = shlex.split(get_cla_test(spec, tmp_path))
     cmd += clas
-    process = subprocess.run(cmd, capture_output=True, text=True)
-    assert "ADDED ATTRIBUTE" in process.stdout
+    subprocess.run(cmd)
+    assert "ADDED ATTRIBUTE" in log.read_text()
 
     output = tmp_path / "out.vspec"
     result = yaml.load(open(output), Loader=yaml.FullLoader)
@@ -277,11 +283,12 @@ def test_added_attribute(caplog: pytest.LogCaptureFixture, tmp_path):
 
 def test_deleted_attribute(caplog: pytest.LogCaptureFixture, tmp_path):
     spec = HERE / "test_vspecs/test_deleted_attribute.vspec"
-    cmd = "vspec export id".split()
+    log = tmp_path / "log.txt"
+    cmd = f"vspec --log-file {log} export id".split()
     clas = shlex.split(get_cla_test(spec, tmp_path))
     cmd += clas
-    process = subprocess.run(cmd, capture_output=True, text=True)
-    assert "DELETED ATTRIBUTE" in process.stdout
+    subprocess.run(cmd)
+    assert "DELETED ATTRIBUTE" in log.read_text()
 
     output = tmp_path / "out.vspec"
     result = yaml.load(open(output), Loader=yaml.FullLoader)
@@ -292,12 +299,12 @@ def test_deleted_attribute(caplog: pytest.LogCaptureFixture, tmp_path):
 def test_overlay(caplog: pytest.LogCaptureFixture, tmp_path):
     spec = HERE / "test_vspecs/test.vspec"
     overlay = HERE / "test_vspecs/test_overlay.vspec"
-    cmd = "vspec export id".split()
+    log = tmp_path / "log.txt"
+    cmd = f"vspec --log-file {log} export id".split()
     clas = shlex.split(get_cla_test(spec, tmp_path, overlay))
     cmd += clas
-    process = subprocess.run(cmd, capture_output=True, text=True)
-    print(process.stdout)
-    assert "ADDED ATTRIBUTE" in process.stdout
+    subprocess.run(cmd)
+    assert "ADDED ATTRIBUTE" in log.read_text()
 
     output = tmp_path / "out.vspec"
     result = yaml.load(open(output), Loader=yaml.FullLoader)
@@ -312,7 +319,7 @@ def test_const_id(caplog: pytest.LogCaptureFixture, tmp_path):
     cmd = "vspec export id".split()
     clas = shlex.split(get_cla_test(spec, tmp_path, overlay))
     cmd += clas
-    subprocess.run(cmd, capture_output=True, text=True)
+    subprocess.run(cmd)
 
     output = tmp_path / "out.vspec"
     result = yaml.load(open(output), Loader=yaml.FullLoader)
@@ -326,13 +333,13 @@ def test_iterated_file(caplog: pytest.LogCaptureFixture, tmp_path):
     cmd = "vspec export id".split()
     clas = shlex.split(get_cla_test(spec, tmp_path))
     cmd += clas
-    subprocess.run(cmd, capture_output=True, text=True)
+    subprocess.run(cmd)
     output = tmp_path / "out.vspec"
     result = yaml.load(open(output), Loader=yaml.FullLoader)
 
     # run again on out.vspec to check if it all hashed attributes were exported correctly
     clas = shlex.split(get_cla_test(output, tmp_path))
-    subprocess.run(cmd, capture_output=True, text=True)
+    subprocess.run(cmd)
 
     output = tmp_path / "out.vspec"
     result_iteration = yaml.load(open(output), Loader=yaml.FullLoader)