From 3ab9179e8a987ab0051442a6ca7225ec2161625a Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 6 Feb 2024 17:28:32 -0600 Subject: [PATCH] Add event testing (#51) * add some event tests * code cleanup * put back error * actually run precommit * remove - * add back codecov flags * add back missing coverage * remove dash again * re-undo msg to dict functionality * pr feedback --- pyproject.toml | 1 - tests/unit/test_events.py | 127 ++++++++++++++++++++++++++++++++ tests/unit/test_proto_events.py | 74 +++++++++++++++++++ 3 files changed, 201 insertions(+), 1 deletion(-) create mode 100644 tests/unit/test_events.py create mode 100644 tests/unit/test_proto_events.py diff --git a/pyproject.toml b/pyproject.toml index acacc238..03e046d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,7 +94,6 @@ features = ["test"] [tool.hatch.envs.test.scripts] unit = "python -m pytest --cov=dbt_common --cov-report=xml {args:tests/unit}" - ### Linting settings, envs & scripts [tool.hatch.envs.lint] diff --git a/tests/unit/test_events.py b/tests/unit/test_events.py new file mode 100644 index 00000000..e91b5787 --- /dev/null +++ b/tests/unit/test_events.py @@ -0,0 +1,127 @@ +import re + +import pytest + +from dbt_common.events import types +from dbt_common.events.base_types import msg_from_base_event +from dbt_common.events.base_types import ( + BaseEvent, + DebugLevel, + DynamicLevel, + ErrorLevel, + InfoLevel, + TestLevel, + WarnLevel, +) +from dbt_common.events.functions import msg_to_dict, msg_to_json + + +# takes in a class and finds any subclasses for it +def get_all_subclasses(cls): + all_subclasses = [] + for subclass in cls.__subclasses__(): + if subclass not in [TestLevel, DebugLevel, WarnLevel, InfoLevel, ErrorLevel, DynamicLevel]: + all_subclasses.append(subclass) + all_subclasses.extend(get_all_subclasses(subclass)) + return set(all_subclasses) + + +class TestEventCodes: + # checks to see if event codes are duplicated to keep codes singluar and clear. + # also checks that event codes follow correct namming convention ex. E001 + def test_event_codes(self): + all_concrete = get_all_subclasses(BaseEvent) + all_codes = set() + + for event_cls in all_concrete: + code = event_cls.code(event_cls) + # must be in the form 1 capital letter, 3 digits + assert re.match("^[A-Z][0-9]{3}", code) + # cannot have been used already + assert ( + code not in all_codes + ), f"{code} is assigned more than once. Check types.py for duplicates." + all_codes.add(code) + + +class TestEventJSONSerialization: + """Attempts to test that every event is serializable to json. + + event types that take `Any` are not possible to test in this way since some will serialize + just fine and others won't. + """ + + SAMPLE_VALUES = [ + # N.B. Events instantiated here include the module prefix in order to + # avoid having the entire list twice in the code. + # M - Deps generation ====================== + types.RetryExternalCall(attempt=0, max=0), + types.RecordRetryException(exc=""), + # Z - misc ====================== + types.SystemCouldNotWrite(path="", reason="", exc=""), + types.SystemExecutingCmd(cmd=[""]), + types.SystemStdOut(bmsg=str(b"")), + types.SystemStdErr(bmsg=str(b"")), + types.SystemReportReturnCode(returncode=0), + types.Formatting(), + types.Note(msg="This is a note."), + ] + + def test_all_serializable(self): + all_non_abstract_events = set( + get_all_subclasses(BaseEvent), + ) + all_event_values_list = list(map(lambda x: x.__class__, self.SAMPLE_VALUES)) + diff = all_non_abstract_events.difference(set(all_event_values_list)) + assert ( + not diff + ), f"{diff}test is missing concrete values in `SAMPLE_VALUES`. Please add the values for the aforementioned event classes" + + # make sure everything in the list is a value not a type + for event in self.SAMPLE_VALUES: + assert not isinstance(event, type) + + # if we have everything we need to test, try to serialize everything + count = 0 + for event in self.SAMPLE_VALUES: + msg = msg_from_base_event(event) + print(f"--- msg: {msg.info.name}") + # Serialize to dictionary + try: + msg_to_dict(msg) + except Exception as e: + raise Exception( + f"{event} can not be converted to a dict. Originating exception: {e}" + ) + # Serialize to json + try: + msg_to_json(msg) + except Exception as e: + raise Exception(f"{event} is not serializable to json. Originating exception: {e}") + # Serialize to binary + try: + msg.SerializeToString() + except Exception as e: + raise Exception( + f"{event} is not serializable to binary protobuf. Originating exception: {e}" + ) + count += 1 + print(f"--- Found {count} events") + + +def test_bad_serialization(): + """Tests that bad serialization enters the proper exception handling + + When pytest is in use the exception handling of `BaseEvent` raises an + exception. When pytest isn't present, it fires a Note event. Thus to test + that bad serializations are properly handled, the best we can do is test + that the exception handling path is used. + """ + + with pytest.raises(Exception) as excinfo: + types.Note(param_event_doesnt_have="This should break") + + assert ( + str(excinfo.value) + == "[Note]: Unable to parse dict {'param_event_doesnt_have': 'This should break'}" + ) diff --git a/tests/unit/test_proto_events.py b/tests/unit/test_proto_events.py new file mode 100644 index 00000000..32eb08ae --- /dev/null +++ b/tests/unit/test_proto_events.py @@ -0,0 +1,74 @@ +from dbt_common.events.functions import msg_to_dict, msg_to_json, reset_metadata_vars +from dbt_common.events import types_pb2 +from dbt_common.events.base_types import msg_from_base_event +from dbt_common.events.types import RetryExternalCall +from google.protobuf.json_format import MessageToDict + +info_keys = { + "name", + "code", + "msg", + "level", + "invocation_id", + "pid", + "thread", + "ts", + "extra", + "category", +} + + +def test_events(): + # M020 event + event_code = "M020" + event = RetryExternalCall(attempt=3, max=5) + msg = msg_from_base_event(event) + msg_dict = msg_to_dict(msg) + msg_json = msg_to_json(msg) + serialized = msg.SerializeToString() + assert "Retrying external call. Attempt: 3" in str(serialized) + assert set(msg_dict.keys()) == {"info", "data"} + assert set(msg_dict["data"].keys()) == {"attempt", "max"} + assert set(msg_dict["info"].keys()) == info_keys + assert msg_json + assert msg.info.code == event_code + + # Extract EventInfo from serialized message + generic_msg = types_pb2.GenericMessage() + generic_msg.ParseFromString(serialized) + assert generic_msg.info.code == event_code + # get the message class for the real message from the generic message + message_class = getattr(types_pb2, f"{generic_msg.info.name}Msg") + new_msg = message_class() + new_msg.ParseFromString(serialized) + assert new_msg.info.code == msg.info.code + assert new_msg.data.attempt == msg.data.attempt + + +def test_extra_dict_on_event(monkeypatch): + monkeypatch.setenv("DBT_ENV_CUSTOM_ENV_env_key", "env_value") + + reset_metadata_vars() + + event_code = "M020" + event = RetryExternalCall(attempt=3, max=5) + msg = msg_from_base_event(event) + msg_dict = msg_to_dict(msg) + assert set(msg_dict["info"].keys()) == info_keys + extra_dict = {"env_key": "env_value"} + assert msg.info.extra == extra_dict + serialized = msg.SerializeToString() + + # Extract EventInfo from serialized message + generic_msg = types_pb2.GenericMessage() + generic_msg.ParseFromString(serialized) + assert generic_msg.info.code == event_code + # get the message class for the real message from the generic message + message_class = getattr(types_pb2, f"{generic_msg.info.name}Msg") + new_msg = message_class() + new_msg.ParseFromString(serialized) + new_msg_dict = MessageToDict(new_msg) + assert new_msg_dict["info"]["extra"] == msg.info.extra + + # clean up + reset_metadata_vars()