From 8e0175b7513280240f6a4d86713a2faaab4f0ee3 Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Mon, 5 Dec 2022 16:07:22 -0800 Subject: [PATCH 01/54] Add tree model function traces (#691) * Add sklearn to tox * Add function traces around model methods * Support Python 2.7 & 3.7 sklearn * Add test for multiple calls to model method * Fixup: add comments & organize * Refactor * Follow two digit convention * Make if-else a one-liner * Abstract to re-usable instrumentation function * Use wrap_method_trace & change to Function group Co-authored-by: Timothy Pansino Co-authored-by: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Co-authored-by: Uma Annamalai * Fixup: use if-else one-liner * Use hasattr instead of model name check * Change component_sklearn to mlmodel_sklearn * Fixup: replace in model names with hasattr Co-authored-by: Timothy Pansino Co-authored-by: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Co-authored-by: Uma Annamalai --- newrelic/config.py | 12 ++ newrelic/hooks/mlmodel_sklearn.py | 76 +++++++++++ tests/mlmodel_sklearn/conftest.py | 38 ++++++ tests/mlmodel_sklearn/test_tree_models.py | 158 ++++++++++++++++++++++ tox.ini | 7 + 5 files changed, 291 insertions(+) create mode 100644 newrelic/hooks/mlmodel_sklearn.py create mode 100644 tests/mlmodel_sklearn/conftest.py create mode 100644 tests/mlmodel_sklearn/test_tree_models.py diff --git a/newrelic/config.py b/newrelic/config.py index f0b638cd4..6961d5f05 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2790,6 +2790,18 @@ def _process_module_builtin_defaults(): ) _process_module_definition("tastypie.api", "newrelic.hooks.component_tastypie", "instrument_tastypie_api") + _process_module_definition( + "sklearn.tree._classes", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_tree_models", + ) + # In scikit-learn < 0.21 the model classes are in tree.py instead of _classes.py. + _process_module_definition( + "sklearn.tree.tree", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_tree_models", + ) + _process_module_definition( "rest_framework.views", "newrelic.hooks.component_djangorestframework", diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py new file mode 100644 index 000000000..4034b5dfd --- /dev/null +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -0,0 +1,76 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from newrelic.api.function_trace import FunctionTrace +from newrelic.api.time_trace import current_trace +from newrelic.api.transaction import current_transaction +from newrelic.common.object_wrapper import wrap_function_wrapper + +METHODS_TO_WRAP = ("predict", "fit", "fit_predict", "predict_log_proba", "predict_proba", "transform", "score") + + +def _wrap_method_trace(module, _class, method, name=None, group=None): + def _nr_wrapper_method(wrapped, instance, args, kwargs): + transaction = current_transaction() + trace = current_trace() + + if transaction is None: + return wrapped(*args, **kwargs) + + wrapped_attr_name = "_nr_wrapped_%s" % method + + # If the method has already been wrapped do not wrap it again. This happens + # when one class inherits from another and they both implement the method. + if getattr(trace, wrapped_attr_name, False): + return wrapped(*args, **kwargs) + + trace = FunctionTrace(name=name, group=group, source=wrapped) + + try: + # Set the _nr_wrapped attribute to denote that this method is being wrapped. + setattr(trace, wrapped_attr_name, True) + + with trace: + return_val = wrapped(*args, **kwargs) + finally: + # Set the _nr_wrapped attribute to denote that this method is no longer wrapped. + setattr(trace, wrapped_attr_name, False) + + return return_val + + wrap_function_wrapper(module, "%s.%s" % (_class, method), _nr_wrapper_method) + + +def _nr_instrument_model(module, model_class): + for method_name in METHODS_TO_WRAP: + if hasattr(getattr(module, model_class), method_name): + # Function/MLModel/Sklearn/Named/. + name = "MLModel/Sklearn/Named/%s.%s" % (model_class, method_name) + _wrap_method_trace(module, model_class, method_name, name=name) + + +def _instrument_sklearn_models(module, model_classes): + for model_cls in model_classes: + if hasattr(module, model_cls): + _nr_instrument_model(module, model_cls) + + +def instrument_sklearn_tree_models(module): + model_classes = ( + "DecisionTreeClassifier", + "DecisionTreeRegressor", + "ExtraTreeClassifier", + "ExtraTreeRegressor", + ) + _instrument_sklearn_models(module, model_classes) diff --git a/tests/mlmodel_sklearn/conftest.py b/tests/mlmodel_sklearn/conftest.py new file mode 100644 index 000000000..e48bc52a9 --- /dev/null +++ b/tests/mlmodel_sklearn/conftest.py @@ -0,0 +1,38 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from testing_support.fixtures import ( # noqa: F401, pylint: disable=W0611 + code_coverage_fixture, + collector_agent_registration_fixture, + collector_available_fixture, +) + +_coverage_source = [ + "newrelic.hooks.mlmodel_sklearn", +] + +code_coverage = code_coverage_fixture(source=_coverage_source) + +_default_settings = { + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, +} +collector_agent_registration = collector_agent_registration_fixture( + app_name="Python Agent Test (mlmodel_sklearn)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (mlmodel_sklearn)"], +) diff --git a/tests/mlmodel_sklearn/test_tree_models.py b/tests/mlmodel_sklearn/test_tree_models.py new file mode 100644 index 000000000..b30b7e2ea --- /dev/null +++ b/tests/mlmodel_sklearn/test_tree_models.py @@ -0,0 +1,158 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.packages import six + + +def test_model_methods_wrapped_in_function_trace(tree_model_name, run_tree_model): + # Note: in the following expected metrics, predict and predict_proba are called by + # score and predict_log_proba so they are expected to be called twice instead of + # once like the rest of the methods. + expected_scoped_metrics = { + "ExtraTreeRegressor": [ + ("Function/MLModel/Sklearn/Named/ExtraTreeRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/ExtraTreeRegressor.predict", 2), + ("Function/MLModel/Sklearn/Named/ExtraTreeRegressor.score", 1), + ], + "DecisionTreeClassifier": [ + ("Function/MLModel/Sklearn/Named/DecisionTreeClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/DecisionTreeClassifier.predict", 2), + ("Function/MLModel/Sklearn/Named/DecisionTreeClassifier.score", 1), + ("Function/MLModel/Sklearn/Named/DecisionTreeClassifier.predict_log_proba", 1), + ("Function/MLModel/Sklearn/Named/DecisionTreeClassifier.predict_proba", 2), + ], + "ExtraTreeClassifier": [ + ("Function/MLModel/Sklearn/Named/ExtraTreeClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/ExtraTreeClassifier.predict", 2), + ("Function/MLModel/Sklearn/Named/ExtraTreeClassifier.score", 1), + ("Function/MLModel/Sklearn/Named/ExtraTreeClassifier.predict_log_proba", 1), + ("Function/MLModel/Sklearn/Named/ExtraTreeClassifier.predict_proba", 2), + ], + "DecisionTreeRegressor": [ + ("Function/MLModel/Sklearn/Named/DecisionTreeRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/DecisionTreeRegressor.predict", 2), + ("Function/MLModel/Sklearn/Named/DecisionTreeRegressor.score", 1), + ], + } + expected_transaction_name = ( + "test_tree_models:test_model_methods_wrapped_in_function_trace.._test" + if six.PY3 + else "test_tree_models:_test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[tree_model_name], + rollup_metrics=expected_scoped_metrics[tree_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_tree_model() + + _test() + + +def test_multiple_calls_to_model_methods(tree_model_name, run_tree_model): + # Note: in the following expected metrics, predict and predict_proba are called by + # score and predict_log_proba so they are expected to be called twice as often as + # the other methods. + expected_scoped_metrics = { + "ExtraTreeRegressor": [ + ("Function/MLModel/Sklearn/Named/ExtraTreeRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/ExtraTreeRegressor.predict", 4), + ("Function/MLModel/Sklearn/Named/ExtraTreeRegressor.score", 2), + ], + "DecisionTreeClassifier": [ + ("Function/MLModel/Sklearn/Named/DecisionTreeClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/DecisionTreeClassifier.predict", 4), + ("Function/MLModel/Sklearn/Named/DecisionTreeClassifier.score", 2), + ("Function/MLModel/Sklearn/Named/DecisionTreeClassifier.predict_log_proba", 2), + ("Function/MLModel/Sklearn/Named/DecisionTreeClassifier.predict_proba", 4), + ], + "ExtraTreeClassifier": [ + ("Function/MLModel/Sklearn/Named/ExtraTreeClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/ExtraTreeClassifier.predict", 4), + ("Function/MLModel/Sklearn/Named/ExtraTreeClassifier.score", 2), + ("Function/MLModel/Sklearn/Named/ExtraTreeClassifier.predict_log_proba", 2), + ("Function/MLModel/Sklearn/Named/ExtraTreeClassifier.predict_proba", 4), + ], + "DecisionTreeRegressor": [ + ("Function/MLModel/Sklearn/Named/DecisionTreeRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/DecisionTreeRegressor.predict", 4), + ("Function/MLModel/Sklearn/Named/DecisionTreeRegressor.score", 2), + ], + } + expected_transaction_name = ( + "test_tree_models:test_multiple_calls_to_model_methods.._test" if six.PY3 else "test_tree_models:_test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[tree_model_name], + rollup_metrics=expected_scoped_metrics[tree_model_name], + background_task=True, + ) + @background_task() + def _test(): + x_test = [[2.0, 2.0], [2.0, 1.0]] + y_test = [1, 1] + + model = run_tree_model() + + model.predict(x_test) + model.score(x_test, y_test) + # Some models don't have these methods. + if hasattr(model, "predict_log_proba"): + model.predict_log_proba(x_test) + if hasattr(model, "predict_proba"): + model.predict_proba(x_test) + + _test() + + +@pytest.fixture(params=["ExtraTreeRegressor", "DecisionTreeClassifier", "ExtraTreeClassifier", "DecisionTreeRegressor"]) +def tree_model_name(request): + return request.param + + +@pytest.fixture +def run_tree_model(tree_model_name): + def _run(): + import sklearn.tree + + x_train = [[0, 0], [1, 1]] + y_train = [0, 1] + x_test = [[2.0, 2.0], [2.0, 1.0]] + y_test = [1, 1] + + clf = getattr(sklearn.tree, tree_model_name)(random_state=0) + model = clf.fit(x_train, y_train) + + labels = model.predict(x_test) + model.score(x_test, y_test) + # Some models don't have these methods. + if hasattr(model, "predict_log_proba"): + model.predict_log_proba(x_test) + if hasattr(model, "predict_proba"): + model.predict_proba(x_test) + return model + + return _run diff --git a/tox.ini b/tox.ini index c815defc2..11a1eea4d 100644 --- a/tox.ini +++ b/tox.ini @@ -60,6 +60,9 @@ envlist = python-agent_unittests-{pypy,pypy37}-without_extensions, python-application_celery-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, gearman-application_gearman-{py27,pypy}, + python-mlmodel_sklearn-{py38,py39,py310,py311}-scikitlearnlatest, + python-mlmodel_sklearn-{py37,pypy37}-scikitlearn0101, + python-mlmodel_sklearn-{py27,pypy27}-scikitlearn0020, python-component_djangorestframework-py27-djangorestframework0300, python-component_djangorestframework-{py37,py38,py39,py310,py311}-djangorestframeworklatest, python-component_flask_rest-{py37,py38,py39,pypy37}-flaskrestxlatest, @@ -203,6 +206,9 @@ deps = application_celery: celery<6.0 application_celery-py{py37,37}: importlib-metadata<5.0 application_gearman: gearman<3.0.0 + mlmodel_sklearn-scikitlearnlatest: scikit-learn + mlmodel_sklearn-scikitlearn0101: scikit-learn < 1.1 + mlmodel_sklearn-scikitlearn0020: scikit-learn < 0.21 component_djangorestframework-djangorestframework0300: Django < 1.9 component_djangorestframework-djangorestframework0300: djangorestframework < 3.1 component_djangorestframework-djangorestframeworklatest: Django @@ -431,6 +437,7 @@ changedir = agent_unittests: tests/agent_unittests application_celery: tests/application_celery application_gearman: tests/application_gearman + mlmodel_sklearn: tests/mlmodel_sklearn component_djangorestframework: tests/component_djangorestframework component_flask_rest: tests/component_flask_rest component_graphqlserver: tests/component_graphqlserver From 2e9a83bcc077bf1f3741153edb51d2f2baaead0c Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Wed, 7 Dec 2022 12:28:18 -0800 Subject: [PATCH 02/54] Add config setting for sklearn inference event capture. (#706) * Add sklearn to tox * Add function traces around model methods * Support Python 2.7 & 3.7 sklearn * Add test for multiple calls to model method * Fixup: add comments & organize * Refactor * Follow two digit convention * Make if-else a one-liner * Abstract to re-usable instrumentation function * Add ML inference event capture config setting. * [Mega-Linter] Apply linters fixes * Fixup: remove component_sklearn files * Add high security mode testing for ML events setting. * [Mega-Linter] Apply linters fixes Co-authored-by: Hannah Stepanek Co-authored-by: umaannamalai --- newrelic/config.py | 6 ++++ newrelic/core/agent_protocol.py | 1 + newrelic/core/config.py | 13 ++++++++ .../agent_features/test_high_security_mode.py | 31 +++++++++++++++++++ 4 files changed, 51 insertions(+) diff --git a/newrelic/config.py b/newrelic/config.py index f270a0335..fc43ee65d 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -539,6 +539,8 @@ def _process_configuration(section): _process_setting(section, "application_logging.metrics.enabled", "getboolean", None) _process_setting(section, "application_logging.local_decorating.enabled", "getboolean", None) + _process_setting(section, "machine_learning.inference_event_value.enabled", "getboolean", None) + # Loading of configuration from specified file and for specified # deployment environment. Can also indicate whether configuration @@ -875,6 +877,10 @@ def apply_local_high_security_mode_setting(settings): settings.application_logging.forwarding.enabled = False _logger.info(log_template, "application_logging.forwarding.enabled", True, False) + if settings.machine_learning.inference_event_value.enabled: + settings.machine_learning.inference_event_value.enabled = False + _logger.info(log_template, "machine_learning.inference_event_value.enabled", True, False) + return settings diff --git a/newrelic/core/agent_protocol.py b/newrelic/core/agent_protocol.py index ba277d4de..c5ac95e23 100644 --- a/newrelic/core/agent_protocol.py +++ b/newrelic/core/agent_protocol.py @@ -144,6 +144,7 @@ class AgentProtocol(object): "strip_exception_messages.enabled", "custom_insights_events.enabled", "application_logging.forwarding.enabled", + "machine_learning.inference_event_value.enabled", ) LOGGER_FUNC_MAPPING = { diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 4111c7149..edc7e820d 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -121,6 +121,14 @@ class GCRuntimeMetricsSettings(Settings): enabled = False +class MachineLearningSettings(Settings): + pass + + +class MachineLearningInferenceEventValueSettings(Settings): + pass + + class CodeLevelMetricsSettings(Settings): pass @@ -359,6 +367,8 @@ class EventHarvestConfigHarvestLimitSettings(Settings): _settings.application_logging.forwarding = ApplicationLoggingForwardingSettings() _settings.application_logging.metrics = ApplicationLoggingMetricsSettings() _settings.application_logging.local_decorating = ApplicationLoggingLocalDecoratingSettings() +_settings.machine_learning = MachineLearningSettings() +_settings.machine_learning.inference_event_value = MachineLearningInferenceEventValueSettings() _settings.attributes = AttributesSettings() _settings.gc_runtime_metrics = GCRuntimeMetricsSettings() _settings.code_level_metrics = CodeLevelMetricsSettings() @@ -821,6 +831,9 @@ def default_host(license_key): _settings.application_logging.local_decorating.enabled = _environ_as_bool( "NEW_RELIC_APPLICATION_LOGGING_LOCAL_DECORATING_ENABLED", default=False ) +_settings.machine_learning.inference_event_value.enabled = _environ_as_bool( + "NEW_RELIC_MACHINE_LEARNING_INFERENCE_EVENT_VALUE_ENABLED", default=True +) def global_settings(): diff --git a/tests/agent_features/test_high_security_mode.py b/tests/agent_features/test_high_security_mode.py index dad7edc29..e37caac43 100644 --- a/tests/agent_features/test_high_security_mode.py +++ b/tests/agent_features/test_high_security_mode.py @@ -79,6 +79,7 @@ def test_hsm_configuration_default(): "custom_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": True, "application_logging.forwarding.enabled": True, + "machine_learning.inference_event_value.enabled": True, }, { "high_security": False, @@ -88,6 +89,7 @@ def test_hsm_configuration_default(): "custom_insights_events.enabled": False, "message_tracer.segment_parameters_enabled": True, "application_logging.forwarding.enabled": True, + "machine_learning.inference_event_value.enabled": True, }, { "high_security": False, @@ -97,6 +99,7 @@ def test_hsm_configuration_default(): "custom_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": False, "application_logging.forwarding.enabled": False, + "machine_learning.inference_event_value.enabled": False, }, { "high_security": False, @@ -106,6 +109,7 @@ def test_hsm_configuration_default(): "custom_insights_events.enabled": False, "message_tracer.segment_parameters_enabled": False, "application_logging.forwarding.enabled": False, + "machine_learning.inference_event_value.enabled": False, }, ] @@ -118,6 +122,7 @@ def test_hsm_configuration_default(): "custom_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": True, "application_logging.forwarding.enabled": False, + "machine_learning.inference_event_value.enabled": False, }, { "high_security": True, @@ -127,6 +132,7 @@ def test_hsm_configuration_default(): "custom_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": True, "application_logging.forwarding.enabled": False, + "machine_learning.inference_event_value.enabled": False, }, { "high_security": True, @@ -136,6 +142,7 @@ def test_hsm_configuration_default(): "custom_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": True, "application_logging.forwarding.enabled": False, + "machine_learning.inference_event_value.enabled": False, }, { "high_security": True, @@ -145,6 +152,7 @@ def test_hsm_configuration_default(): "custom_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": True, "application_logging.forwarding.enabled": True, + "machine_learning.inference_event_value.enabled": True, }, { "high_security": True, @@ -154,6 +162,7 @@ def test_hsm_configuration_default(): "custom_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": True, "application_logging.forwarding.enabled": True, + "machine_learning.inference_event_value.enabled": True, }, { "high_security": True, @@ -163,6 +172,7 @@ def test_hsm_configuration_default(): "custom_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": False, "application_logging.forwarding.enabled": True, + "machine_learning.inference_event_value.enabled": True, }, { "high_security": True, @@ -172,6 +182,7 @@ def test_hsm_configuration_default(): "custom_insights_events.enabled": False, "message_tracer.segment_parameters_enabled": False, "application_logging.forwarding.enabled": True, + "machine_learning.inference_event_value.enabled": True, }, ] @@ -196,6 +207,7 @@ def test_local_config_file_override_hsm_disabled(settings): original_custom_events = settings.custom_insights_events.enabled original_message_segment_params_enabled = settings.message_tracer.segment_parameters_enabled original_application_logging_forwarding_enabled = settings.application_logging.forwarding.enabled + original_machine_learning_inference_event_value_enabled = settings.machine_learning.inference_event_value.enabled apply_local_high_security_mode_setting(settings) @@ -205,6 +217,10 @@ def test_local_config_file_override_hsm_disabled(settings): assert settings.custom_insights_events.enabled == original_custom_events assert settings.message_tracer.segment_parameters_enabled == original_message_segment_params_enabled assert settings.application_logging.forwarding.enabled == original_application_logging_forwarding_enabled + assert ( + settings.machine_learning.inference_event_value.enabled + == original_machine_learning_inference_event_value_enabled + ) @parameterize_hsm_local_config(_hsm_local_config_file_settings_enabled) @@ -217,6 +233,7 @@ def test_local_config_file_override_hsm_enabled(settings): assert settings.custom_insights_events.enabled is False assert settings.message_tracer.segment_parameters_enabled is False assert settings.application_logging.forwarding.enabled is False + assert settings.machine_learning.inference_event_value.enabled is False _server_side_config_settings_hsm_disabled = [ @@ -228,6 +245,7 @@ def test_local_config_file_override_hsm_enabled(settings): "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, "application_logging.forwarding.enabled": False, + "machine_learning.inference_event_value.enabled": False, }, { "agent_config": { @@ -236,6 +254,7 @@ def test_local_config_file_override_hsm_enabled(settings): "strip_exception_messages.enabled": False, "custom_insights_events.enabled": True, "application_logging.forwarding.enabled": True, + "machine_learning.inference_event_value.enabled": True, }, }, ), @@ -247,6 +266,7 @@ def test_local_config_file_override_hsm_enabled(settings): "strip_exception_messages.enabled": False, "custom_insights_events.enabled": True, "application_logging.forwarding.enabled": True, + "machine_learning.inference_event_value.enabled": True, }, { "agent_config": { @@ -255,6 +275,7 @@ def test_local_config_file_override_hsm_enabled(settings): "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, "application_logging.forwarding.enabled": False, + "machine_learning.inference_event_value.enabled": False, }, }, ), @@ -269,6 +290,7 @@ def test_local_config_file_override_hsm_enabled(settings): "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, "application_logging.forwarding.enabled": False, + "machine_learning.inference_event_value.enabled": False, }, { "high_security": True, @@ -277,12 +299,14 @@ def test_local_config_file_override_hsm_enabled(settings): "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, "application_logging.forwarding.enabled": False, + "machine_learning.inference_event_value.enabled": False, "agent_config": { "capture_params": False, "transaction_tracer.record_sql": "obfuscated", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, "application_logging.forwarding.enabled": False, + "machine_learning.inference_event_value.enabled": False, }, }, ), @@ -294,6 +318,7 @@ def test_local_config_file_override_hsm_enabled(settings): "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, "application_logging.forwarding.enabled": False, + "machine_learning.inference_event_value.enabled": False, }, { "high_security": True, @@ -302,12 +327,14 @@ def test_local_config_file_override_hsm_enabled(settings): "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, "application_logging.forwarding.enabled": False, + "machine_learning.inference_event_value.enabled": False, "agent_config": { "capture_params": True, "transaction_tracer.record_sql": "raw", "strip_exception_messages.enabled": False, "custom_insights_events.enabled": True, "application_logging.forwarding.enabled": True, + "machine_learning.inference_event_value.enabled": True, }, }, ), @@ -328,6 +355,7 @@ def test_remote_config_fixups_hsm_disabled(local_settings, server_settings): original_strip_messages = agent_config["strip_exception_messages.enabled"] original_custom_events = agent_config["custom_insights_events.enabled"] original_log_forwarding = agent_config["application_logging.forwarding.enabled"] + original_machine_learning_events = agent_config["machine_learning.inference_event_value.enabled"] _settings = global_settings() settings = override_generic_settings(_settings, local_settings)(AgentProtocol._apply_high_security_mode_fixups)( @@ -343,6 +371,7 @@ def test_remote_config_fixups_hsm_disabled(local_settings, server_settings): assert agent_config["strip_exception_messages.enabled"] == original_strip_messages assert agent_config["custom_insights_events.enabled"] == original_custom_events assert agent_config["application_logging.forwarding.enabled"] == original_log_forwarding + assert agent_config["machine_learning.inference_event_value.enabled"] == original_machine_learning_events @pytest.mark.parametrize("local_settings,server_settings", _server_side_config_settings_hsm_enabled) @@ -365,12 +394,14 @@ def test_remote_config_fixups_hsm_enabled(local_settings, server_settings): assert "strip_exception_messages.enabled" not in settings assert "custom_insights_events.enabled" not in settings assert "application_logging.forwarding.enabled" not in settings + assert "machine_learning.inference_event_value.enabled" not in settings assert "capture_params" not in agent_config assert "transaction_tracer.record_sql" not in agent_config assert "strip_exception_messages.enabled" not in agent_config assert "custom_insights_events.enabled" not in agent_config assert "application_logging.forwarding.enabled" not in agent_config + assert "machine_learning.inference_event_value.enabled" not in agent_config def test_remote_config_hsm_fixups_server_side_disabled(): From 9df29a950e1c078ca9faf564345a58cecb17324b Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Wed, 7 Dec 2022 17:05:12 -0800 Subject: [PATCH 03/54] Capture scorer results (#694) * Add score results attributes to metric scorers * Test un-subclassable types * [Mega-Linter] Apply linters fixes * [Mega-Linter] Apply linters fixes * Trigger tests * Remove custom subclassing code. * [Mega-Linter] Apply linters fixes * Remove unused function * Add test for iterable score results * Change name of object proxy * Fixup: rename proxy in tests too Co-authored-by: hmstepanek Co-authored-by: Tim Pansino Co-authored-by: TimPansino --- newrelic/config.py | 5 + newrelic/hooks/mlmodel_sklearn.py | 57 ++++++++- tests/mlmodel_sklearn/test_metric_scorers.py | 118 +++++++++++++++++++ 3 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 tests/mlmodel_sklearn/test_metric_scorers.py diff --git a/newrelic/config.py b/newrelic/config.py index fc43ee65d..c40abc3c5 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2812,6 +2812,11 @@ def _process_module_builtin_defaults(): "newrelic.hooks.mlmodel_sklearn", "instrument_sklearn_tree_models", ) + _process_module_definition( + "sklearn.metrics", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_metrics", + ) _process_module_definition( "rest_framework.views", diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index 4034b5dfd..01ae6d8b6 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -12,12 +12,30 @@ # See the License for the specific language governing permissions and # limitations under the License. +import sys + from newrelic.api.function_trace import FunctionTrace from newrelic.api.time_trace import current_trace from newrelic.api.transaction import current_transaction -from newrelic.common.object_wrapper import wrap_function_wrapper +from newrelic.common.object_wrapper import ObjectProxy, wrap_function_wrapper METHODS_TO_WRAP = ("predict", "fit", "fit_predict", "predict_log_proba", "predict_proba", "transform", "score") +METRIC_SCORERS = ( + "accuracy_score", + "balanced_accuracy_score", + "f1_score", + "precision_score", + "recall_score", + "roc_auc_score", + "r2_score", +) +PY2 = sys.version_info[0] == 2 + + +class PredictReturnTypeProxy(ObjectProxy): + def __init__(self, wrapped, model_name): + super(ObjectProxy, self).__init__(wrapped) + self._nr_model_name = model_name def _wrap_method_trace(module, _class, method, name=None, group=None): @@ -47,6 +65,10 @@ def _nr_wrapper_method(wrapped, instance, args, kwargs): # Set the _nr_wrapped attribute to denote that this method is no longer wrapped. setattr(trace, wrapped_attr_name, False) + # If this is the predict method, wrap the return type in an nr type with + # _nr_wrapped attrs that will attach model info to the data. + if method == "predict": + return PredictReturnTypeProxy(return_val, model_name=_class) return return_val wrap_function_wrapper(module, "%s.%s" % (_class, method), _nr_wrapper_method) @@ -66,6 +88,33 @@ def _instrument_sklearn_models(module, model_classes): _nr_instrument_model(module, model_cls) +def _bind_scorer(y_true, y_pred, *args, **kwargs): + return y_true, y_pred, args, kwargs + + +def wrap_metric_scorer(wrapped, instance, args, kwargs): + transaction = current_transaction() + # If there is no transaction, do not wrap anything. + if not transaction: + return wrapped(*args, **kwargs) + + score = wrapped(*args, **kwargs) + + y_true, y_pred, args, kwargs = _bind_scorer(*args, **kwargs) + model_name = "Unknown" + if hasattr(y_pred, "_nr_model_name"): + model_name = y_pred._nr_model_name + # Attribute values must be int, float, str, or boolean. If it's not one of these + # types and an iterable add the values as separate attributes. + if not isinstance(score, (str, int, float, bool)): + if hasattr(score, "__iter__"): + for i, s in enumerate(score): + transaction._add_agent_attribute("%s.%s[%s]" % (model_name, wrapped.__name__, i), s) + else: + transaction._add_agent_attribute("%s.%s" % (model_name, wrapped.__name__), score) + return score + + def instrument_sklearn_tree_models(module): model_classes = ( "DecisionTreeClassifier", @@ -74,3 +123,9 @@ def instrument_sklearn_tree_models(module): "ExtraTreeRegressor", ) _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_metrics(module): + for scorer in METRIC_SCORERS: + if hasattr(module, scorer): + wrap_function_wrapper(module, scorer, wrap_metric_scorer) diff --git a/tests/mlmodel_sklearn/test_metric_scorers.py b/tests/mlmodel_sklearn/test_metric_scorers.py new file mode 100644 index 000000000..7ee7a2ebe --- /dev/null +++ b/tests/mlmodel_sklearn/test_metric_scorers.py @@ -0,0 +1,118 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import pytest +from testing_support.fixtures import validate_attributes + +from newrelic.api.background_task import background_task +from newrelic.hooks.mlmodel_sklearn import PredictReturnTypeProxy + + +@pytest.mark.parametrize( + "metric_scorer_name", + ( + "accuracy_score", + "balanced_accuracy_score", + "f1_score", + "precision_score", + "recall_score", + "roc_auc_score", + "r2_score", + ), +) +def test_metric_scorer_attributes(metric_scorer_name, run_metric_scorer): + @validate_attributes("agent", ["DecisionTreeClassifier.%s" % metric_scorer_name]) + @background_task() + def _test(): + run_metric_scorer(metric_scorer_name) + + _test() + + +@pytest.mark.parametrize( + "metric_scorer_name,kwargs", + [ + ("f1_score", {"average": None}), + ("precision_score", {"average": None}), + ("recall_score", {"average": None}), + ], +) +def test_metric_scorer_iterable_score_attributes(metric_scorer_name, kwargs, run_metric_scorer): + @validate_attributes( + "agent", + [ + "DecisionTreeClassifier.%s[0]" % metric_scorer_name, + "DecisionTreeClassifier.%s[1]" % metric_scorer_name, + ], + ) + @background_task() + def _test(): + run_metric_scorer(metric_scorer_name, kwargs) + + _test() + + +@pytest.mark.parametrize( + "metric_scorer_name", + [ + "accuracy_score", + "balanced_accuracy_score", + "f1_score", + "precision_score", + "recall_score", + "roc_auc_score", + "r2_score", + ], +) +def test_metric_scorer_attributes_unknown_model(metric_scorer_name): + @validate_attributes("agent", ["Unknown.%s" % metric_scorer_name]) + @background_task() + def _test(): + from sklearn import metrics + + y_pred = [1, 0] + y_test = [1, 0] + + getattr(metrics, metric_scorer_name)(y_test, y_pred) + + _test() + + +@pytest.mark.parametrize("data", (np.array([0, 1]), "foo", 1, 1.0, True, [0, 1], {"foo": "bar"}, (0, 1), np.str_("F"))) +def test_PredictReturnTypeProxy(data): + wrapped_data = PredictReturnTypeProxy(data, "ModelName") + + assert wrapped_data._nr_model_name == "ModelName" + + +@pytest.fixture +def run_metric_scorer(): + def _run(metric_scorer_name, metric_scorer_kwargs=None): + from sklearn import metrics, tree + + x_train = [[0, 0], [1, 1]] + y_train = [0, 1] + x_test = [[2.0, 2.0], [0, 0.5]] + y_test = [1, 0] + + clf = tree.DecisionTreeClassifier(random_state=0) + model = clf.fit(x_train, y_train) + + labels = model.predict(x_test) + + metric_scorer_kwargs = metric_scorer_kwargs or {} + return getattr(metrics, metric_scorer_name)(y_test, labels, **metric_scorer_kwargs) + + return _run From c6a9d4c9c18820ff2223f68587d6db8bc0fbc5e5 Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Tue, 13 Dec 2022 13:55:27 -0800 Subject: [PATCH 04/54] Add ensemble model function traces (#697) * Add sklearn to tox * Add function traces around model methods * Support Python 2.7 & 3.7 sklearn * Add test for multiple calls to model method * Fixup: add comments & organize * Add ensemble models * Add ensemble model tests * Edit tests * Add ensemble library models from sklearn * Start tests with empty commit * Clean up tests * Fix tests for various versions of sklearn * Fix ensemble tests with changes from tree PR * [Mega-Linter] Apply linters fixes * Remove breakpoints * Remove commited config files * Group tests into more readable format * Pin startlette latest < 0.23.1 * Convert PY3 checks to one-liners * Use tuple checks for sklearn version Use tuple checks for sklearn version, string checks can result in unexpected out of order comparisons. Also use direct comparisons for easier readability. * Fix VotingRegressor test Co-authored-by: Hannah Stepanek Co-authored-by: lrafeei --- newrelic/config.py | 84 +++++ newrelic/hooks/mlmodel_sklearn.py | 64 ++++ tests/mlmodel_sklearn/test_ensemble_models.py | 301 ++++++++++++++++++ tox.ini | 3 +- 4 files changed, 451 insertions(+), 1 deletion(-) create mode 100644 tests/mlmodel_sklearn/test_ensemble_models.py diff --git a/newrelic/config.py b/newrelic/config.py index c40abc3c5..9a22a7a83 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2818,6 +2818,90 @@ def _process_module_builtin_defaults(): "instrument_sklearn_metrics", ) + _process_module_definition( + "sklearn.ensemble._bagging", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_ensemble_bagging_models", + ) + + _process_module_definition( + "sklearn.ensemble.bagging", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_ensemble_bagging_models", + ) + + _process_module_definition( + "sklearn.ensemble._forest", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_ensemble_forest_models", + ) + + _process_module_definition( + "sklearn.ensemble.forest", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_ensemble_forest_models", + ) + + _process_module_definition( + "sklearn.ensemble._iforest", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_ensemble_iforest_models", + ) + + _process_module_definition( + "sklearn.ensemble.iforest", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_ensemble_iforest_models", + ) + + _process_module_definition( + "sklearn.ensemble._weight_boosting", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_ensemble_weight_boosting_models", + ) + + _process_module_definition( + "sklearn.ensemble.weight_boosting", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_ensemble_weight_boosting_models", + ) + + _process_module_definition( + "sklearn.ensemble._gb", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_ensemble_gradient_boosting_models", + ) + + _process_module_definition( + "sklearn.ensemble.gradient_boosting", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_ensemble_gradient_boosting_models", + ) + + _process_module_definition( + "sklearn.ensemble._voting", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_ensemble_voting_models", + ) + + _process_module_definition( + "sklearn.ensemble.voting_classifier", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_ensemble_voting_models", + ) + + _process_module_definition( + "sklearn.ensemble._stacking", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_ensemble_stacking_models", + ) + + _process_module_definition( + "sklearn.ensemble._hist_gradient_boosting.gradient_boosting", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_ensemble_hist_models", + ) + _process_module_definition( "rest_framework.views", "newrelic.hooks.component_djangorestframework", diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index 01ae6d8b6..2426f5813 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -125,6 +125,70 @@ def instrument_sklearn_tree_models(module): _instrument_sklearn_models(module, model_classes) +def instrument_sklearn_ensemble_bagging_models(module): + model_classes = ( + "BaggingClassifier", + "BaggingRegressor", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_ensemble_forest_models(module): + model_classes = ( + "ExtraTreesClassifier", + "ExtraTreesRegressor", + "RandomForestClassifier", + "RandomForestRegressor", + "RandomTreesEmbedding", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_ensemble_iforest_models(module): + model_classes = ("IsolationForest",) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_ensemble_weight_boosting_models(module): + model_classes = ( + "AdaBoostClassifier", + "AdaBoostRegressor", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_ensemble_gradient_boosting_models(module): + model_classes = ( + "GradientBoostingClassifier", + "GradientBoostingRegressor", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_ensemble_voting_models(module): + model_classes = ( + "VotingClassifier", + "VotingRegressor", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_ensemble_stacking_models(module): + module_classes = ( + "StackingClassifier", + "StackingRegressor", + ) + _instrument_sklearn_models(module, module_classes) + + +def instrument_sklearn_ensemble_hist_models(module): + model_classes = ( + "HistGradientBoostingClassifier", + "HistGradientBoostingRegressor", + ) + _instrument_sklearn_models(module, model_classes) + + def instrument_sklearn_metrics(module): for scorer in METRIC_SCORERS: if hasattr(module, scorer): diff --git a/tests/mlmodel_sklearn/test_ensemble_models.py b/tests/mlmodel_sklearn/test_ensemble_models.py new file mode 100644 index 000000000..29ade4cee --- /dev/null +++ b/tests/mlmodel_sklearn/test_ensemble_models.py @@ -0,0 +1,301 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor +from sklearn.linear_model import LinearRegression +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version +from newrelic.packages import six + +SKLEARN_VERSION = tuple(map(int, get_package_version("sklearn").split("."))) + + +@pytest.mark.parametrize( + "ensemble_model_name", + [ + "AdaBoostClassifier", + "AdaBoostRegressor", + "BaggingClassifier", + "BaggingRegressor", + "ExtraTreesClassifier", + "ExtraTreesRegressor", + "GradientBoostingClassifier", + "GradientBoostingRegressor", + "IsolationForest", + "RandomForestClassifier", + "RandomForestRegressor", + "RandomTreesEmbedding", + "VotingClassifier", + ], +) +def test_below_v1_0_model_methods_wrapped_in_function_trace(ensemble_model_name, run_ensemble_model): + expected_scoped_metrics = { + "AdaBoostClassifier": [ + ("Function/MLModel/Sklearn/Named/AdaBoostClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/AdaBoostClassifier.predict", 2), + ("Function/MLModel/Sklearn/Named/AdaBoostClassifier.predict_log_proba", 1), + ("Function/MLModel/Sklearn/Named/AdaBoostClassifier.predict_proba", 2), + ("Function/MLModel/Sklearn/Named/AdaBoostClassifier.score", 1), + ], + "AdaBoostRegressor": [ + ("Function/MLModel/Sklearn/Named/AdaBoostRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/AdaBoostRegressor.predict", 2), + ("Function/MLModel/Sklearn/Named/AdaBoostRegressor.score", 1), + ], + "BaggingClassifier": [ + ("Function/MLModel/Sklearn/Named/BaggingClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/BaggingClassifier.predict", 2), + ("Function/MLModel/Sklearn/Named/BaggingClassifier.score", 1), + ("Function/MLModel/Sklearn/Named/BaggingClassifier.predict_log_proba", 1), + ("Function/MLModel/Sklearn/Named/BaggingClassifier.predict_proba", 3), + ], + "BaggingRegressor": [ + ("Function/MLModel/Sklearn/Named/BaggingRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/BaggingRegressor.predict", 2), + ("Function/MLModel/Sklearn/Named/BaggingRegressor.score", 1), + ], + "ExtraTreesClassifier": [ + ("Function/MLModel/Sklearn/Named/ExtraTreesClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/ExtraTreesClassifier.predict", 2), + ("Function/MLModel/Sklearn/Named/ExtraTreesClassifier.score", 1), + ("Function/MLModel/Sklearn/Named/ExtraTreesClassifier.predict_log_proba", 1), + ("Function/MLModel/Sklearn/Named/ExtraTreesClassifier.predict_proba", 4), + ], + "ExtraTreesRegressor": [ + ("Function/MLModel/Sklearn/Named/ExtraTreesRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/ExtraTreesRegressor.predict", 2), + ("Function/MLModel/Sklearn/Named/ExtraTreesRegressor.score", 1), + ], + "GradientBoostingClassifier": [ + ("Function/MLModel/Sklearn/Named/GradientBoostingClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/GradientBoostingClassifier.predict", 2), + ("Function/MLModel/Sklearn/Named/GradientBoostingClassifier.score", 1), + ("Function/MLModel/Sklearn/Named/GradientBoostingClassifier.predict_log_proba", 1), + ("Function/MLModel/Sklearn/Named/GradientBoostingClassifier.predict_proba", 2), + ], + "GradientBoostingRegressor": [ + ("Function/MLModel/Sklearn/Named/GradientBoostingRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/GradientBoostingRegressor.predict", 2), + ("Function/MLModel/Sklearn/Named/GradientBoostingRegressor.score", 1), + ], + "IsolationForest": [ + ("Function/MLModel/Sklearn/Named/IsolationForest.fit", 1), + ("Function/MLModel/Sklearn/Named/IsolationForest.predict", 1), + ], + "RandomForestClassifier": [ + ("Function/MLModel/Sklearn/Named/RandomForestClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/RandomForestClassifier.predict", 2), + ("Function/MLModel/Sklearn/Named/RandomForestClassifier.score", 1), + ("Function/MLModel/Sklearn/Named/RandomForestClassifier.predict_log_proba", 1), + ("Function/MLModel/Sklearn/Named/RandomForestClassifier.predict_proba", 4), + ], + "RandomForestRegressor": [ + ("Function/MLModel/Sklearn/Named/RandomForestRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/RandomForestRegressor.predict", 2), + ("Function/MLModel/Sklearn/Named/RandomForestRegressor.score", 1), + ], + "RandomTreesEmbedding": [ + ("Function/MLModel/Sklearn/Named/RandomTreesEmbedding.fit", 1), + ("Function/MLModel/Sklearn/Named/RandomTreesEmbedding.transform", 1), + ], + "VotingClassifier": [ + ("Function/MLModel/Sklearn/Named/VotingClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/VotingClassifier.predict", 2), + ("Function/MLModel/Sklearn/Named/VotingClassifier.score", 1), + ("Function/MLModel/Sklearn/Named/VotingClassifier.transform", 1), + ("Function/MLModel/Sklearn/Named/VotingClassifier.predict_proba", 3), + ], + } + + expected_transaction_name = ( + "test_ensemble_models:test_below_v1_0_model_methods_wrapped_in_function_trace.._test" + if six.PY3 + else "test_ensemble_models:_test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[ensemble_model_name], + rollup_metrics=expected_scoped_metrics[ensemble_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_ensemble_model(ensemble_model_name) + + _test() + + +@pytest.mark.skipif(SKLEARN_VERSION < (1, 0, 0) or SKLEARN_VERSION >= (1, 1, 0), reason="Requires 1.0 <= sklearn < 1.1") +@pytest.mark.parametrize( + "ensemble_model_name", + [ + "HistGradientBoostingClassifier", + "HistGradientBoostingRegressor", + "StackingClassifier", + "StackingRegressor", + "VotingRegressor", + ], +) +def test_between_v1_0_and_v1_1_model_methods_wrapped_in_function_trace(ensemble_model_name, run_ensemble_model): + expected_scoped_metrics = { + "HistGradientBoostingClassifier": [ + ("Function/MLModel/Sklearn/Named/HistGradientBoostingClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/HistGradientBoostingClassifier.predict", 2), + ("Function/MLModel/Sklearn/Named/HistGradientBoostingClassifier.score", 1), + ("Function/MLModel/Sklearn/Named/HistGradientBoostingClassifier.predict_proba", 3), + ], + "HistGradientBoostingRegressor": [ + ("Function/MLModel/Sklearn/Named/HistGradientBoostingRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/HistGradientBoostingRegressor.predict", 2), + ("Function/MLModel/Sklearn/Named/HistGradientBoostingRegressor.score", 1), + ], + "StackingClassifier": [ + ("Function/MLModel/Sklearn/Named/StackingClassifier.fit", 1), + ], + "StackingRegressor": [ + ("Function/MLModel/Sklearn/Named/StackingRegressor.fit", 1), + ], + "VotingRegressor": [ + ("Function/MLModel/Sklearn/Named/VotingRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/VotingRegressor.predict", 2), + ("Function/MLModel/Sklearn/Named/VotingRegressor.score", 1), + ("Function/MLModel/Sklearn/Named/VotingRegressor.transform", 1), + ], + } + expected_transaction_name = ( + "test_ensemble_models:test_between_v1_0_and_v1_1_model_methods_wrapped_in_function_trace.._test" + if six.PY3 + else "test_ensemble_models:_test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[ensemble_model_name], + rollup_metrics=expected_scoped_metrics[ensemble_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_ensemble_model(ensemble_model_name) + + _test() + + +@pytest.mark.skipif(SKLEARN_VERSION < (1, 1, 0), reason="Requires sklearn >= 1.1") +@pytest.mark.parametrize( + "ensemble_model_name", + [ + "HistGradientBoostingClassifier", + "HistGradientBoostingRegressor", + "StackingClassifier", + "StackingRegressor", + "VotingRegressor", + ], +) +def test_above_v1_1_model_methods_wrapped_in_function_trace(ensemble_model_name, run_ensemble_model): + expected_scoped_metrics = { + "StackingClassifier": [ + ("Function/MLModel/Sklearn/Named/StackingClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/StackingClassifier.predict", 2), + ("Function/MLModel/Sklearn/Named/StackingClassifier.score", 1), + ("Function/MLModel/Sklearn/Named/StackingClassifier.predict_proba", 1), + ("Function/MLModel/Sklearn/Named/StackingClassifier.transform", 4), + ], + "StackingRegressor": [ + ("Function/MLModel/Sklearn/Named/StackingRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/StackingRegressor.predict", 2), + ("Function/MLModel/Sklearn/Named/StackingRegressor.score", 1), + ], + "VotingRegressor": [ + ("Function/MLModel/Sklearn/Named/VotingRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/VotingRegressor.predict", 2), + ("Function/MLModel/Sklearn/Named/VotingRegressor.score", 1), + ("Function/MLModel/Sklearn/Named/VotingRegressor.transform", 1), + ], + "HistGradientBoostingClassifier": [ + ("Function/MLModel/Sklearn/Named/HistGradientBoostingClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/HistGradientBoostingClassifier.predict", 2), + ("Function/MLModel/Sklearn/Named/HistGradientBoostingClassifier.score", 1), + ("Function/MLModel/Sklearn/Named/HistGradientBoostingClassifier.predict_proba", 3), + ], + "HistGradientBoostingRegressor": [ + ("Function/MLModel/Sklearn/Named/HistGradientBoostingRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/HistGradientBoostingRegressor.predict", 2), + ("Function/MLModel/Sklearn/Named/HistGradientBoostingRegressor.score", 1), + ], + } + expected_transaction_name = ( + "test_ensemble_models:test_above_v1_1_model_methods_wrapped_in_function_trace.._test" + if six.PY3 + else "test_ensemble_models:_test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[ensemble_model_name], + rollup_metrics=expected_scoped_metrics[ensemble_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_ensemble_model(ensemble_model_name) + + _test() + + +@pytest.fixture +def run_ensemble_model(): + def _run(ensemble_model_name): + import sklearn.ensemble + from sklearn.datasets import load_iris + from sklearn.model_selection import train_test_split + + X, y = load_iris(return_X_y=True) + x_train, x_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) + + kwargs = {"random_state": 0} + if ensemble_model_name == "StackingClassifier": + kwargs = {"estimators": [("rf", RandomForestClassifier())], "final_estimator": RandomForestClassifier()} + elif ensemble_model_name == "VotingClassifier": + kwargs = { + "estimators": [("rf", RandomForestClassifier())], + "voting": "soft", + } + elif ensemble_model_name == "VotingRegressor": + kwargs = {"estimators": [("rf", RandomForestRegressor()), ("lr", LinearRegression())]} + elif ensemble_model_name == "StackingRegressor": + kwargs = {"estimators": [("rf", RandomForestRegressor())]} + clf = getattr(sklearn.ensemble, ensemble_model_name)(**kwargs) + + model = clf.fit(x_train, y_train) + if hasattr(model, "predict"): + model.predict(x_test) + if hasattr(model, "score"): + model.score(x_test, y_test) + if hasattr(model, "predict_log_proba"): + model.predict_log_proba(x_test) + if hasattr(model, "predict_proba"): + model.predict_proba(x_test) + if hasattr(model, "transform"): + model.transform(x_test) + + return model + + return _run diff --git a/tox.ini b/tox.ini index 11a1eea4d..a2d3366e9 100644 --- a/tox.ini +++ b/tox.ini @@ -363,7 +363,8 @@ deps = framework_starlette-starlette0015: starlette<0.16 framework_starlette-starlette0019: starlette<0.20 framework_starlette-starlette002001: starlette==0.20.1 - framework_starlette-starlettelatest: starlette + # Temporarily pin startlette latest until we fix the failures. + framework_starlette-starlettelatest: starlette<0.23.1 framework_strawberry: starlette framework_strawberry-strawberrylatest: strawberry-graphql framework_tornado: pycurl From d9d5636e115a552eac582c6a41105515474c8a7b Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Fri, 16 Dec 2022 10:32:42 -0800 Subject: [PATCH 05/54] Include training step in metric scorer name (#712) * Include training step in scorer name * Add fit_predict data proxying * Remove name comments * Fix predict being called before fit * Re-use existing fixture --- newrelic/hooks/mlmodel_sklearn.py | 22 +++++++-- tests/mlmodel_sklearn/test_metric_scorers.py | 52 ++++++++++++++++---- 2 files changed, 59 insertions(+), 15 deletions(-) diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index 2426f5813..de5a8d6f7 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -33,9 +33,10 @@ class PredictReturnTypeProxy(ObjectProxy): - def __init__(self, wrapped, model_name): + def __init__(self, wrapped, model_name, training_step): super(ObjectProxy, self).__init__(wrapped) self._nr_model_name = model_name + self._nr_training_step = training_step def _wrap_method_trace(module, _class, method, name=None, group=None): @@ -65,10 +66,16 @@ def _nr_wrapper_method(wrapped, instance, args, kwargs): # Set the _nr_wrapped attribute to denote that this method is no longer wrapped. setattr(trace, wrapped_attr_name, False) + # If this is the fit method, increment the training_step counter. + if method in ("fit", "fit_predict"): + training_step = getattr(instance, "_nr_wrapped_training_step", -1) + setattr(instance, "_nr_wrapped_training_step", training_step + 1) + # If this is the predict method, wrap the return type in an nr type with # _nr_wrapped attrs that will attach model info to the data. - if method == "predict": - return PredictReturnTypeProxy(return_val, model_name=_class) + if method in ("predict", "fit_predict"): + training_step = getattr(instance, "_nr_wrapped_training_step", "Unknown") + return PredictReturnTypeProxy(return_val, model_name=_class, training_step=training_step) return return_val wrap_function_wrapper(module, "%s.%s" % (_class, method), _nr_wrapper_method) @@ -102,16 +109,21 @@ def wrap_metric_scorer(wrapped, instance, args, kwargs): y_true, y_pred, args, kwargs = _bind_scorer(*args, **kwargs) model_name = "Unknown" + training_step = "Unknown" if hasattr(y_pred, "_nr_model_name"): model_name = y_pred._nr_model_name + if hasattr(y_pred, "_nr_training_step"): + training_step = y_pred._nr_training_step # Attribute values must be int, float, str, or boolean. If it's not one of these # types and an iterable add the values as separate attributes. if not isinstance(score, (str, int, float, bool)): if hasattr(score, "__iter__"): for i, s in enumerate(score): - transaction._add_agent_attribute("%s.%s[%s]" % (model_name, wrapped.__name__, i), s) + transaction._add_agent_attribute( + "%s/TrainingStep/%s/%s[%s]" % (model_name, training_step, wrapped.__name__, i), s + ) else: - transaction._add_agent_attribute("%s.%s" % (model_name, wrapped.__name__), score) + transaction._add_agent_attribute("%s/TrainingStep/%s/%s" % (model_name, training_step, wrapped.__name__), score) return score diff --git a/tests/mlmodel_sklearn/test_metric_scorers.py b/tests/mlmodel_sklearn/test_metric_scorers.py index 7ee7a2ebe..50557b882 100644 --- a/tests/mlmodel_sklearn/test_metric_scorers.py +++ b/tests/mlmodel_sklearn/test_metric_scorers.py @@ -33,7 +33,7 @@ ), ) def test_metric_scorer_attributes(metric_scorer_name, run_metric_scorer): - @validate_attributes("agent", ["DecisionTreeClassifier.%s" % metric_scorer_name]) + @validate_attributes("agent", ["DecisionTreeClassifier/TrainingStep/0/%s" % metric_scorer_name]) @background_task() def _test(): run_metric_scorer(metric_scorer_name) @@ -41,6 +41,33 @@ def _test(): _test() +@pytest.mark.parametrize( + "metric_scorer_name", + ( + "accuracy_score", + "balanced_accuracy_score", + "f1_score", + "precision_score", + "recall_score", + "roc_auc_score", + "r2_score", + ), +) +def test_metric_scorer_training_steps_attributes(metric_scorer_name, run_metric_scorer): + @validate_attributes( + "agent", + [ + "DecisionTreeClassifier/TrainingStep/0/%s" % metric_scorer_name, + "DecisionTreeClassifier/TrainingStep/1/%s" % metric_scorer_name, + ], + ) + @background_task() + def _test(): + run_metric_scorer(metric_scorer_name, training_steps=[0, 1]) + + _test() + + @pytest.mark.parametrize( "metric_scorer_name,kwargs", [ @@ -53,8 +80,8 @@ def test_metric_scorer_iterable_score_attributes(metric_scorer_name, kwargs, run @validate_attributes( "agent", [ - "DecisionTreeClassifier.%s[0]" % metric_scorer_name, - "DecisionTreeClassifier.%s[1]" % metric_scorer_name, + "DecisionTreeClassifier/TrainingStep/0/%s[0]" % metric_scorer_name, + "DecisionTreeClassifier/TrainingStep/0/%s[1]" % metric_scorer_name, ], ) @background_task() @@ -77,7 +104,7 @@ def _test(): ], ) def test_metric_scorer_attributes_unknown_model(metric_scorer_name): - @validate_attributes("agent", ["Unknown.%s" % metric_scorer_name]) + @validate_attributes("agent", ["Unknown/TrainingStep/Unknown/%s" % metric_scorer_name]) @background_task() def _test(): from sklearn import metrics @@ -92,14 +119,15 @@ def _test(): @pytest.mark.parametrize("data", (np.array([0, 1]), "foo", 1, 1.0, True, [0, 1], {"foo": "bar"}, (0, 1), np.str_("F"))) def test_PredictReturnTypeProxy(data): - wrapped_data = PredictReturnTypeProxy(data, "ModelName") + wrapped_data = PredictReturnTypeProxy(data, "ModelName", 0) assert wrapped_data._nr_model_name == "ModelName" + assert wrapped_data._nr_training_step == 0 @pytest.fixture def run_metric_scorer(): - def _run(metric_scorer_name, metric_scorer_kwargs=None): + def _run(metric_scorer_name, metric_scorer_kwargs=None, training_steps=None): from sklearn import metrics, tree x_train = [[0, 0], [1, 1]] @@ -107,12 +135,16 @@ def _run(metric_scorer_name, metric_scorer_kwargs=None): x_test = [[2.0, 2.0], [0, 0.5]] y_test = [1, 0] + if not training_steps: + training_steps = [0] + clf = tree.DecisionTreeClassifier(random_state=0) - model = clf.fit(x_train, y_train) + for step in training_steps: + model = clf.fit(x_train, y_train) - labels = model.predict(x_test) + labels = model.predict(x_test) - metric_scorer_kwargs = metric_scorer_kwargs or {} - return getattr(metrics, metric_scorer_name)(y_test, labels, **metric_scorer_kwargs) + metric_scorer_kwargs = metric_scorer_kwargs or {} + getattr(metrics, metric_scorer_name)(y_test, labels, **metric_scorer_kwargs) return _run From f33d21e970cf617d1d771ed1a6db61c23d4e8643 Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Tue, 20 Dec 2022 10:34:45 -0800 Subject: [PATCH 06/54] Add cluster model function traces (#700) * Add sklearn to tox * Add function traces around model methods * Support Python 2.7 & 3.7 sklearn * Add test for multiple calls to model method * Fixup: add comments & organize * Add ensemble models * Add ensemble model tests * Edit tests * Add ensemble library models from sklearn * Start tests with empty commit * Clean up tests * Add cluster model instrumentaton * Fix tests for various versions of sklearn * Fix ensemble tests with changes from tree PR * [Mega-Linter] Apply linters fixes * Fix some cluster model tests * Fix tests after ensemble PR merge * Add transform to tests * Remove accidental commits * Modify cluster tests to be more readable * Break up instrumentation models * Remove duplicate ensemble module defs * Modify VotingRegressor test Co-authored-by: Hannah Stepanek Co-authored-by: lrafeei --- newrelic/config.py | 114 +++++++++++ newrelic/hooks/mlmodel_sklearn.py | 37 ++++ tests/mlmodel_sklearn/test_cluster_models.py | 186 ++++++++++++++++++ tests/mlmodel_sklearn/test_ensemble_models.py | 2 +- 4 files changed, 338 insertions(+), 1 deletion(-) create mode 100644 tests/mlmodel_sklearn/test_cluster_models.py diff --git a/newrelic/config.py b/newrelic/config.py index 9a22a7a83..6bfc2c22e 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2902,6 +2902,120 @@ def _process_module_builtin_defaults(): "instrument_sklearn_ensemble_hist_models", ) + _process_module_definition( + "sklearn.cluster._affinity_propagation", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_models", + ) + + _process_module_definition( + "sklearn.cluster.affinity_propagation_", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_models", + ) + + _process_module_definition( + "sklearn.cluster._agglomerative", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_agglomerative_models", + ) + + _process_module_definition( + "sklearn.cluster.hierarchical", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_agglomerative_models", + ) + + _process_module_definition( + "sklearn.cluster._birch", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_models", + ) + + _process_module_definition( + "sklearn.cluster.birch", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_models", + ) + + _process_module_definition( + "sklearn.cluster._bisect_k_means", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_kmeans_models", + ) + + _process_module_definition( + "sklearn.cluster._dbscan", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_models", + ) + + _process_module_definition( + "sklearn.cluster.dbscan_", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_models", + ) + + _process_module_definition( + "sklearn.cluster._feature_agglomeration", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_models", + ) + + _process_module_definition( + "sklearn.cluster._kmeans", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_kmeans_models", + ) + + _process_module_definition( + "sklearn.cluster.k_means_", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_kmeans_models", + ) + + _process_module_definition( + "sklearn.cluster._mean_shift", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_models", + ) + + _process_module_definition( + "sklearn.cluster.mean_shift_", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_models", + ) + + _process_module_definition( + "sklearn.cluster._optics", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_models", + ) + + _process_module_definition( + "sklearn.cluster._spectral", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_clustering_models", + ) + + _process_module_definition( + "sklearn.cluster.spectral", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_clustering_models", + ) + + _process_module_definition( + "sklearn.cluster._bicluster", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_clustering_models", + ) + + _process_module_definition( + "sklearn.cluster.bicluster", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_clustering_models", + ) + _process_module_definition( "rest_framework.views", "newrelic.hooks.component_djangorestframework", diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index de5a8d6f7..d7d5a9b9b 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -201,6 +201,43 @@ def instrument_sklearn_ensemble_hist_models(module): _instrument_sklearn_models(module, model_classes) +def instrument_sklearn_cluster_models(module): + model_classes = ( + "AffinityPropagation", + "Birch", + "DBSCAN", + "MeanShift", + "OPTICS", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_cluster_agglomerative_models(module): + model_classes = ( + "AgglomerativeClustering", + "FeatureAgglomeration", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_cluster_clustering_models(module): + model_classes = ( + "SpectralBiclustering", + "SpectralCoclustering", + "SpectralClustering", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_cluster_kmeans_models(module): + model_classes = ( + "BisectingKMeans", + "KMeans", + "MiniBatchKMeans", + ) + _instrument_sklearn_models(module, model_classes) + + def instrument_sklearn_metrics(module): for scorer in METRIC_SCORERS: if hasattr(module, scorer): diff --git a/tests/mlmodel_sklearn/test_cluster_models.py b/tests/mlmodel_sklearn/test_cluster_models.py new file mode 100644 index 000000000..906995c22 --- /dev/null +++ b/tests/mlmodel_sklearn/test_cluster_models.py @@ -0,0 +1,186 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from sklearn import __version__ # noqa: this is needed for get_package_version +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version +from newrelic.packages import six + +SKLEARN_VERSION = tuple(map(int, get_package_version("sklearn").split("."))) + + +@pytest.mark.parametrize( + "cluster_model_name", + [ + "AffinityPropagation", + "AgglomerativeClustering", + "Birch", + "DBSCAN", + "FeatureAgglomeration", + "KMeans", + "MeanShift", + "MiniBatchKMeans", + "SpectralBiclustering", + "SpectralCoclustering", + "SpectralClustering", + ], +) +def test_below_v1_1_model_methods_wrapped_in_function_trace(cluster_model_name, run_cluster_model): + expected_scoped_metrics = { + "AffinityPropagation": [ + ("Function/MLModel/Sklearn/Named/AffinityPropagation.fit", 2), + ("Function/MLModel/Sklearn/Named/AffinityPropagation.predict", 1), + ("Function/MLModel/Sklearn/Named/AffinityPropagation.fit_predict", 1), + ], + "AgglomerativeClustering": [ + ("Function/MLModel/Sklearn/Named/AgglomerativeClustering.fit", 2), + ("Function/MLModel/Sklearn/Named/AgglomerativeClustering.fit_predict", 1), + ], + "Birch": [ + ("Function/MLModel/Sklearn/Named/Birch.fit", 2), + ( + "Function/MLModel/Sklearn/Named/Birch.predict", + 1 if SKLEARN_VERSION >= (1, 0, 0) else 3, + ), + ("Function/MLModel/Sklearn/Named/Birch.fit_predict", 1), + ("Function/MLModel/Sklearn/Named/Birch.transform", 1), + ], + "DBSCAN": [ + ("Function/MLModel/Sklearn/Named/DBSCAN.fit", 2), + ("Function/MLModel/Sklearn/Named/DBSCAN.fit_predict", 1), + ], + "FeatureAgglomeration": [ + ("Function/MLModel/Sklearn/Named/FeatureAgglomeration.fit", 1), + ("Function/MLModel/Sklearn/Named/FeatureAgglomeration.transform", 1), + ], + "KMeans": [ + ("Function/MLModel/Sklearn/Named/KMeans.fit", 2), + ("Function/MLModel/Sklearn/Named/KMeans.predict", 1), + ("Function/MLModel/Sklearn/Named/KMeans.fit_predict", 1), + ("Function/MLModel/Sklearn/Named/KMeans.transform", 1), + ], + "MeanShift": [ + ("Function/MLModel/Sklearn/Named/MeanShift.fit", 2), + ("Function/MLModel/Sklearn/Named/MeanShift.predict", 1), + ("Function/MLModel/Sklearn/Named/MeanShift.fit_predict", 1), + ], + "MiniBatchKMeans": [ + ("Function/MLModel/Sklearn/Named/MiniBatchKMeans.fit", 2), + ("Function/MLModel/Sklearn/Named/MiniBatchKMeans.predict", 1), + ("Function/MLModel/Sklearn/Named/MiniBatchKMeans.fit_predict", 1), + ], + "SpectralBiclustering": [ + ("Function/MLModel/Sklearn/Named/SpectralBiclustering.fit", 1), + ], + "SpectralCoclustering": [ + ("Function/MLModel/Sklearn/Named/SpectralCoclustering.fit", 1), + ], + "SpectralClustering": [ + ("Function/MLModel/Sklearn/Named/SpectralClustering.fit", 2), + ("Function/MLModel/Sklearn/Named/SpectralClustering.fit_predict", 1), + ], + } + expected_transaction_name = "test_cluster_models:_test" + if six.PY3: + expected_transaction_name = ( + "test_cluster_models:test_below_v1_1_model_methods_wrapped_in_function_trace.._test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[cluster_model_name], + rollup_metrics=expected_scoped_metrics[cluster_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_cluster_model(cluster_model_name) + + _test() + + +@pytest.mark.skipif(SKLEARN_VERSION < (1, 1, 0), reason="Requires sklearn > 1.1") +@pytest.mark.parametrize( + "cluster_model_name", + [ + "BisectingKMeans", + "OPTICS", + ], +) +def test_above_v1_1_model_methods_wrapped_in_function_trace(cluster_model_name, run_cluster_model): + expected_scoped_metrics = { + "BisectingKMeans": [ + ("Function/MLModel/Sklearn/Named/BisectingKMeans.fit", 2), + ("Function/MLModel/Sklearn/Named/BisectingKMeans.predict", 1), + ("Function/MLModel/Sklearn/Named/BisectingKMeans.fit_predict", 1), + ], + "OPTICS": [ + ("Function/MLModel/Sklearn/Named/OPTICS.fit", 2), + ("Function/MLModel/Sklearn/Named/OPTICS.fit_predict", 1), + ], + } + expected_transaction_name = "test_cluster_models:_test" + if six.PY3: + expected_transaction_name = ( + "test_cluster_models:test_above_v1_1_model_methods_wrapped_in_function_trace.._test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[cluster_model_name], + rollup_metrics=expected_scoped_metrics[cluster_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_cluster_model(cluster_model_name) + + _test() + + +@pytest.fixture +def run_cluster_model(): + def _run(cluster_model_name): + import sklearn.cluster + from sklearn.datasets import load_iris + from sklearn.model_selection import train_test_split + + X, y = load_iris(return_X_y=True) + x_train, x_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) + + clf = getattr(sklearn.cluster, cluster_model_name)() + + model = clf.fit(x_train, y_train) + + if hasattr(model, "predict"): + model.predict(x_test) + if hasattr(model, "score"): + model.score(x_test, y_test) + if hasattr(model, "fit_predict"): + model.fit_predict(x_test) + if hasattr(model, "predict_log_proba"): + model.predict_log_proba(x_test) + if hasattr(model, "predict_proba"): + model.predict_proba(x_test) + if hasattr(model, "transform"): + model.transform(x_test) + + return model + + return _run diff --git a/tests/mlmodel_sklearn/test_ensemble_models.py b/tests/mlmodel_sklearn/test_ensemble_models.py index 29ade4cee..9daabdb5c 100644 --- a/tests/mlmodel_sklearn/test_ensemble_models.py +++ b/tests/mlmodel_sklearn/test_ensemble_models.py @@ -279,7 +279,7 @@ def _run(ensemble_model_name): "voting": "soft", } elif ensemble_model_name == "VotingRegressor": - kwargs = {"estimators": [("rf", RandomForestRegressor()), ("lr", LinearRegression())]} + kwargs = {"estimators": [("lr", LinearRegression())]} elif ensemble_model_name == "StackingRegressor": kwargs = {"estimators": [("rf", RandomForestRegressor())]} clf = getattr(sklearn.ensemble, ensemble_model_name)(**kwargs) From e3f43f2d1eeab73248d0b5c67f75ff66a60f1a17 Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Wed, 21 Dec 2022 14:30:11 -0800 Subject: [PATCH 07/54] Add calibration model function traces (#709) * Add sklearn to tox * Add function traces around model methods * Support Python 2.7 & 3.7 sklearn * Add test for multiple calls to model method * Fixup: add comments & organize * Add ensemble models * Add ensemble model tests * Edit tests * Add ensemble library models from sklearn * Start tests with empty commit * Clean up tests * Fix tests for various versions of sklearn * Fix ensemble tests with changes from tree PR * [Mega-Linter] Apply linters fixes * Remove breakpoints * Create tests/instrumentation for calibration models * Fix calibration tests * Remove commented out code * Remove yaml file in commit * Remove duplicate ensemble module defs Co-authored-by: Hannah Stepanek Co-authored-by: lrafeei --- newrelic/config.py | 6 ++ newrelic/hooks/mlmodel_sklearn.py | 5 ++ .../test_calibration_models.py | 76 +++++++++++++++++++ 3 files changed, 87 insertions(+) create mode 100644 tests/mlmodel_sklearn/test_calibration_models.py diff --git a/newrelic/config.py b/newrelic/config.py index 6bfc2c22e..69538dcfc 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2902,6 +2902,12 @@ def _process_module_builtin_defaults(): "instrument_sklearn_ensemble_hist_models", ) + _process_module_definition( + "sklearn.calibration", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_calibration_models", + ) + _process_module_definition( "sklearn.cluster._affinity_propagation", "newrelic.hooks.mlmodel_sklearn", diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index d7d5a9b9b..f8fa231d7 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -201,6 +201,11 @@ def instrument_sklearn_ensemble_hist_models(module): _instrument_sklearn_models(module, model_classes) +def instrument_sklearn_calibration_models(module): + model_classes = ("CalibratedClassifierCV",) + _instrument_sklearn_models(module, model_classes) + + def instrument_sklearn_cluster_models(module): model_classes = ( "AffinityPropagation", diff --git a/tests/mlmodel_sklearn/test_calibration_models.py b/tests/mlmodel_sklearn/test_calibration_models.py new file mode 100644 index 000000000..39ac34cb2 --- /dev/null +++ b/tests/mlmodel_sklearn/test_calibration_models.py @@ -0,0 +1,76 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.packages import six + + +def test_model_methods_wrapped_in_function_trace(calibration_model_name, run_calibration_model): + expected_scoped_metrics = { + "CalibratedClassifierCV": [ + ("Function/MLModel/Sklearn/Named/CalibratedClassifierCV.fit", 1), + ("Function/MLModel/Sklearn/Named/CalibratedClassifierCV.predict", 1), + ("Function/MLModel/Sklearn/Named/CalibratedClassifierCV.predict_proba", 2), + ], + } + + expected_transaction_name = "test_calibration_models:_test" + if six.PY3: + expected_transaction_name = ( + "test_calibration_models:test_model_methods_wrapped_in_function_trace.._test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[calibration_model_name], + rollup_metrics=expected_scoped_metrics[calibration_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_calibration_model() + + _test() + + +@pytest.fixture(params=["CalibratedClassifierCV"]) +def calibration_model_name(request): + return request.param + + +@pytest.fixture +def run_calibration_model(calibration_model_name): + def _run(): + import sklearn.calibration + from sklearn.datasets import load_iris + from sklearn.model_selection import train_test_split + + X, y = load_iris(return_X_y=True) + x_train, x_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) + + clf = getattr(sklearn.calibration, calibration_model_name)() + + model = clf.fit(x_train, y_train) + model.predict(x_test) + + model.predict_proba(x_test) + + return model + + return _run From beedc9c95f5c21eed8f5b2d7b3cdeebf908dd94f Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Thu, 5 Jan 2023 16:14:48 -0800 Subject: [PATCH 08/54] Add svm model function traces (#733) * Add svm models * Remove extra conditionals from testing --- newrelic/config.py | 12 +++ newrelic/hooks/mlmodel_sklearn.py | 13 +++ tests/mlmodel_sklearn/test_svm_models.py | 110 +++++++++++++++++++++++ 3 files changed, 135 insertions(+) create mode 100644 tests/mlmodel_sklearn/test_svm_models.py diff --git a/newrelic/config.py b/newrelic/config.py index 69538dcfc..997e9f52e 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3022,6 +3022,18 @@ def _process_module_builtin_defaults(): "instrument_sklearn_cluster_clustering_models", ) + _process_module_definition( + "sklearn.svm._classes", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_svm_models", + ) + + _process_module_definition( + "sklearn.svm.classes", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_svm_models", + ) + _process_module_definition( "rest_framework.views", "newrelic.hooks.component_djangorestframework", diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index f8fa231d7..b77faea17 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -243,6 +243,19 @@ def instrument_sklearn_cluster_kmeans_models(module): _instrument_sklearn_models(module, model_classes) +def instrument_sklearn_svm_models(module): + model_classes = ( + "LinearSVC", + "LinearSVR", + "SVC", + "NuSVC", + "SVR", + "NuSVR", + "OneClassSVM", + ) + _instrument_sklearn_models(module, model_classes) + + def instrument_sklearn_metrics(module): for scorer in METRIC_SCORERS: if hasattr(module, scorer): diff --git a/tests/mlmodel_sklearn/test_svm_models.py b/tests/mlmodel_sklearn/test_svm_models.py new file mode 100644 index 000000000..fe95f2f46 --- /dev/null +++ b/tests/mlmodel_sklearn/test_svm_models.py @@ -0,0 +1,110 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version +from newrelic.packages import six + +SKLEARN_VERSION = tuple(map(int, get_package_version("sklearn").split("."))) + + +@pytest.mark.parametrize( + "svm_model_name", + [ + "LinearSVC", + "LinearSVR", + "SVC", + "NuSVC", + "SVR", + "NuSVR", + "OneClassSVM", + ], +) +def test_model_methods_wrapped_in_function_trace(svm_model_name, run_svm_model): + expected_scoped_metrics = { + "LinearSVC": [ + ("Function/MLModel/Sklearn/Named/LinearSVC.fit", 1), + ("Function/MLModel/Sklearn/Named/LinearSVC.predict", 1), + ], + "LinearSVR": [ + ("Function/MLModel/Sklearn/Named/LinearSVR.fit", 1), + ("Function/MLModel/Sklearn/Named/LinearSVR.predict", 1), + ], + "SVC": [ + ("Function/MLModel/Sklearn/Named/SVC.fit", 1), + ("Function/MLModel/Sklearn/Named/SVC.predict", 1), + ], + "NuSVC": [ + ("Function/MLModel/Sklearn/Named/NuSVC.fit", 1), + ("Function/MLModel/Sklearn/Named/NuSVC.predict", 1), + ], + "SVR": [ + ("Function/MLModel/Sklearn/Named/SVR.fit", 1), + ("Function/MLModel/Sklearn/Named/SVR.predict", 1), + ], + "NuSVR": [ + ("Function/MLModel/Sklearn/Named/NuSVR.fit", 1), + ("Function/MLModel/Sklearn/Named/NuSVR.predict", 1), + ], + "OneClassSVM": [ + ("Function/MLModel/Sklearn/Named/OneClassSVM.fit", 1), + ("Function/MLModel/Sklearn/Named/OneClassSVM.predict", 1), + ], + } + + expected_transaction_name = ( + "test_svm_models:test_model_methods_wrapped_in_function_trace.._test" + if six.PY3 + else "test_svm_models:_test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[svm_model_name], + rollup_metrics=expected_scoped_metrics[svm_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_svm_model(svm_model_name) + + _test() + + +@pytest.fixture +def run_svm_model(): + def _run(svm_model_name): + import sklearn.svm + from sklearn.datasets import load_iris + from sklearn.model_selection import train_test_split + + X, y = load_iris(return_X_y=True) + x_train, x_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) + + kwargs = {"random_state": 0} + if svm_model_name in ["SVR", "NuSVR", "OneClassSVM"]: + kwargs = {} + clf = getattr(sklearn.svm, svm_model_name)(**kwargs) + + model = clf.fit(x_train, y_train) + model.predict(x_test) + + return model + + return _run From 22952d397f157bb068e8e95b4bace543dd06ceea Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Thu, 5 Jan 2023 22:39:12 -0800 Subject: [PATCH 09/54] Add semi supervised models (#732) --- newrelic/config.py | 18 +++ newrelic/hooks/mlmodel_sklearn.py | 9 ++ .../test_semi_supervised_models.py | 132 ++++++++++++++++++ 3 files changed, 159 insertions(+) create mode 100644 tests/mlmodel_sklearn/test_semi_supervised_models.py diff --git a/newrelic/config.py b/newrelic/config.py index 997e9f52e..bcac6f3fb 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3022,6 +3022,24 @@ def _process_module_builtin_defaults(): "instrument_sklearn_cluster_clustering_models", ) + _process_module_definition( + "sklearn.semi_supervised._label_propagation", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_semi_supervised_models", + ) + + _process_module_definition( + "sklearn.semi_supervised._self_training", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_semi_supervised_models", + ) + + _process_module_definition( + "sklearn.semi_supervised.label_propagation", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_semi_supervised_models", + ) + _process_module_definition( "sklearn.svm._classes", "newrelic.hooks.mlmodel_sklearn", diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index b77faea17..bc0a18dbf 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -256,6 +256,15 @@ def instrument_sklearn_svm_models(module): _instrument_sklearn_models(module, model_classes) +def instrument_sklearn_semi_supervised_models(module): + model_classes = ( + "LabelPropagation", + "LabelSpreading", + "SelfTrainingClassifier", + ) + _instrument_sklearn_models(module, model_classes) + + def instrument_sklearn_metrics(module): for scorer in METRIC_SCORERS: if hasattr(module, scorer): diff --git a/tests/mlmodel_sklearn/test_semi_supervised_models.py b/tests/mlmodel_sklearn/test_semi_supervised_models.py new file mode 100644 index 000000000..f4069f75b --- /dev/null +++ b/tests/mlmodel_sklearn/test_semi_supervised_models.py @@ -0,0 +1,132 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from sklearn.ensemble import AdaBoostClassifier +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version +from newrelic.packages import six + +SKLEARN_VERSION = tuple(map(int, get_package_version("sklearn").split("."))) + + +@pytest.mark.parametrize( + "semi_supervised_model_name", + [ + "LabelPropagation", + "LabelSpreading", + ], +) +def test_model_methods_wrapped_in_function_trace(semi_supervised_model_name, run_semi_supervised_model): + expected_scoped_metrics = { + "LabelPropagation": [ + ("Function/MLModel/Sklearn/Named/LabelPropagation.fit", 1), + ("Function/MLModel/Sklearn/Named/LabelPropagation.predict", 2), + ("Function/MLModel/Sklearn/Named/LabelPropagation.predict_proba", 3), + ], + "LabelSpreading": [ + ("Function/MLModel/Sklearn/Named/LabelSpreading.fit", 1), + ("Function/MLModel/Sklearn/Named/LabelSpreading.predict", 2), + ("Function/MLModel/Sklearn/Named/LabelSpreading.predict_proba", 3), + ], + } + + expected_transaction_name = ( + "test_semi_supervised_models:test_model_methods_wrapped_in_function_trace.._test" + if six.PY3 + else "test_semi_supervised_models:_test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[semi_supervised_model_name], + rollup_metrics=expected_scoped_metrics[semi_supervised_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_semi_supervised_model(semi_supervised_model_name) + + _test() + + +@pytest.mark.skipif(SKLEARN_VERSION < (1, 0, 0), reason="Requires sklearn <= 1.0") +@pytest.mark.parametrize( + "semi_supervised_model_name", + [ + "SelfTrainingClassifier", + ], +) +def test_above_v1_0_model_methods_wrapped_in_function_trace(semi_supervised_model_name, run_semi_supervised_model): + expected_scoped_metrics = { + "SelfTrainingClassifier": [ + ("Function/MLModel/Sklearn/Named/SelfTrainingClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/SelfTrainingClassifier.predict", 1), + ("Function/MLModel/Sklearn/Named/SelfTrainingClassifier.predict_log_proba", 1), + ("Function/MLModel/Sklearn/Named/SelfTrainingClassifier.score", 1), + ("Function/MLModel/Sklearn/Named/SelfTrainingClassifier.predict_proba", 1), + ], + } + expected_transaction_name = ( + "test_semi_supervised_models:test_above_v1_0_model_methods_wrapped_in_function_trace.._test" + if six.PY3 + else "test_semi_supervised_models:_test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[semi_supervised_model_name], + rollup_metrics=expected_scoped_metrics[semi_supervised_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_semi_supervised_model(semi_supervised_model_name) + + _test() + + +@pytest.fixture +def run_semi_supervised_model(): + def _run(semi_supervised_model_name): + import sklearn.semi_supervised + from sklearn.datasets import load_iris + from sklearn.model_selection import train_test_split + + X, y = load_iris(return_X_y=True) + x_train, x_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) + + if semi_supervised_model_name == "SelfTrainingClassifier": + kwargs = {"base_estimator": AdaBoostClassifier()} + else: + kwargs = {} + clf = getattr(sklearn.semi_supervised, semi_supervised_model_name)(**kwargs) + + model = clf.fit(x_train, y_train) + if hasattr(model, "predict"): + model.predict(x_test) + if hasattr(model, "score"): + model.score(x_test, y_test) + if hasattr(model, "predict_log_proba"): + model.predict_log_proba(x_test) + if hasattr(model, "predict_proba"): + model.predict_proba(x_test) + + return model + + return _run From 90b7dc191812ab6b6314f936bbd1b5a38cb3c7c5 Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Fri, 6 Jan 2023 10:25:31 -0800 Subject: [PATCH 10/54] Add pipeline model function traces (#730) * Add pipeline models * Remove commented code --- newrelic/config.py | 6 ++ newrelic/hooks/mlmodel_sklearn.py | 8 ++ tests/mlmodel_sklearn/test_pipeline_models.py | 95 +++++++++++++++++++ 3 files changed, 109 insertions(+) create mode 100644 tests/mlmodel_sklearn/test_pipeline_models.py diff --git a/newrelic/config.py b/newrelic/config.py index bcac6f3fb..a604047da 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3022,6 +3022,12 @@ def _process_module_builtin_defaults(): "instrument_sklearn_cluster_clustering_models", ) + _process_module_definition( + "sklearn.pipeline", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_pipeline_models", + ) + _process_module_definition( "sklearn.semi_supervised._label_propagation", "newrelic.hooks.mlmodel_sklearn", diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index bc0a18dbf..de5905a8d 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -265,6 +265,14 @@ def instrument_sklearn_semi_supervised_models(module): _instrument_sklearn_models(module, model_classes) +def instrument_sklearn_pipeline_models(module): + model_classes = ( + "Pipeline", + "FeatureUnion", + ) + _instrument_sklearn_models(module, model_classes) + + def instrument_sklearn_metrics(module): for scorer in METRIC_SCORERS: if hasattr(module, scorer): diff --git a/tests/mlmodel_sklearn/test_pipeline_models.py b/tests/mlmodel_sklearn/test_pipeline_models.py new file mode 100644 index 000000000..ac9b918f4 --- /dev/null +++ b/tests/mlmodel_sklearn/test_pipeline_models.py @@ -0,0 +1,95 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from sklearn.decomposition import TruncatedSVD +from sklearn.preprocessing import StandardScaler +from sklearn.svm import SVC +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version +from newrelic.packages import six + +SKLEARN_VERSION = tuple(map(int, get_package_version("sklearn").split("."))) + + +@pytest.mark.parametrize( + "pipeline_model_name", + [ + "Pipeline", + "FeatureUnion", + ], +) +def test_model_methods_wrapped_in_function_trace(pipeline_model_name, run_pipeline_model): + expected_scoped_metrics = { + "Pipeline": [ + ("Function/MLModel/Sklearn/Named/Pipeline.fit", 1), + ("Function/MLModel/Sklearn/Named/Pipeline.predict", 1), + ("Function/MLModel/Sklearn/Named/Pipeline.score", 1), + ], + "FeatureUnion": [ + ("Function/MLModel/Sklearn/Named/FeatureUnion.fit", 1), + ("Function/MLModel/Sklearn/Named/FeatureUnion.transform", 1), + ], + } + + expected_transaction_name = ( + "test_pipeline_models:test_model_methods_wrapped_in_function_trace.._test" + if six.PY3 + else "test_pipeline_models:_test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[pipeline_model_name], + rollup_metrics=expected_scoped_metrics[pipeline_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_pipeline_model(pipeline_model_name) + + _test() + + +@pytest.fixture +def run_pipeline_model(): + def _run(pipeline_model_name): + import sklearn.pipeline + from sklearn.datasets import load_iris + from sklearn.model_selection import train_test_split + + X, y = load_iris(return_X_y=True) + x_train, x_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) + + if pipeline_model_name == "Pipeline": + kwargs = {"steps": [("scaler", StandardScaler()), ("svc", SVC())]} + else: + kwargs = {"transformer_list": [("scaler", StandardScaler()), ("svd", TruncatedSVD(n_components=2))]} + clf = getattr(sklearn.pipeline, pipeline_model_name)(**kwargs) + + model = clf.fit(x_train, y_train) + if hasattr(model, "predict"): + model.predict(x_test) + if hasattr(model, "score"): + model.score(x_test, y_test) + if hasattr(model, "transform"): + model.transform(x_test) + + return model + + return _run From d785977ee3067b010f8a336dcf4a069aee727dbd Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Fri, 6 Jan 2023 12:05:08 -0800 Subject: [PATCH 11/54] Add neural network model function traces (#729) * Add neural network models * Fixup: merge conflict Co-authored-by: Hannah Stepanek --- newrelic/config.py | 24 +++++ newrelic/hooks/mlmodel_sklearn.py | 9 ++ .../test_neural_network_models.py | 96 +++++++++++++++++++ 3 files changed, 129 insertions(+) create mode 100644 tests/mlmodel_sklearn/test_neural_network_models.py diff --git a/newrelic/config.py b/newrelic/config.py index a604047da..4f138a5de 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2902,6 +2902,30 @@ def _process_module_builtin_defaults(): "instrument_sklearn_ensemble_hist_models", ) + _process_module_definition( + "sklearn.neural_network._multilayer_perceptron", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neural_network_models", + ) + + _process_module_definition( + "sklearn.neural_network.multilayer_perceptron", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neural_network_models", + ) + + _process_module_definition( + "sklearn.neural_network._rbm", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neural_network_models", + ) + + _process_module_definition( + "sklearn.neural_network.rbm", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neural_network_models", + ) + _process_module_definition( "sklearn.calibration", "newrelic.hooks.mlmodel_sklearn", diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index de5905a8d..c023ad7a0 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -243,6 +243,15 @@ def instrument_sklearn_cluster_kmeans_models(module): _instrument_sklearn_models(module, model_classes) +def instrument_sklearn_neural_network_models(module): + model_classes = ( + "BernoulliRBM", + "MLPClassifier", + "MLPRegressor", + ) + _instrument_sklearn_models(module, model_classes) + + def instrument_sklearn_svm_models(module): model_classes = ( "LinearSVC", diff --git a/tests/mlmodel_sklearn/test_neural_network_models.py b/tests/mlmodel_sklearn/test_neural_network_models.py new file mode 100644 index 000000000..468bfb4b9 --- /dev/null +++ b/tests/mlmodel_sklearn/test_neural_network_models.py @@ -0,0 +1,96 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version +from newrelic.packages import six + +SKLEARN_VERSION = tuple(map(int, get_package_version("sklearn").split("."))) + + +@pytest.mark.parametrize( + "neural_network_model_name", + [ + "MLPClassifier", + "MLPRegressor", + "BernoulliRBM", + ], +) +def test_model_methods_wrapped_in_function_trace(neural_network_model_name, run_neural_network_model): + expected_scoped_metrics = { + "MLPClassifier": [ + ("Function/MLModel/Sklearn/Named/MLPClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/MLPClassifier.predict", 1), + ("Function/MLModel/Sklearn/Named/MLPClassifier.predict_log_proba", 1), + ("Function/MLModel/Sklearn/Named/MLPClassifier.predict_proba", 2), + ], + "MLPRegressor": [ + ("Function/MLModel/Sklearn/Named/MLPRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/MLPRegressor.predict", 1), + ], + "BernoulliRBM": [ + ("Function/MLModel/Sklearn/Named/BernoulliRBM.fit", 1), + ("Function/MLModel/Sklearn/Named/BernoulliRBM.transform", 1), + ], + } + + expected_transaction_name = ( + "test_neural_network_models:test_model_methods_wrapped_in_function_trace.._test" + if six.PY3 + else "test_neural_network_models:_test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[neural_network_model_name], + rollup_metrics=expected_scoped_metrics[neural_network_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_neural_network_model(neural_network_model_name) + + _test() + + +@pytest.fixture +def run_neural_network_model(): + def _run(neural_network_model_name): + import sklearn.neural_network + from sklearn.datasets import load_iris + from sklearn.model_selection import train_test_split + + X, y = load_iris(return_X_y=True) + x_train, x_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) + + clf = getattr(sklearn.neural_network, neural_network_model_name)() + + model = clf.fit(x_train, y_train) + if hasattr(model, "predict"): + model.predict(x_test) + if hasattr(model, "predict_log_proba"): + model.predict_log_proba(x_test) + if hasattr(model, "predict_proba"): + model.predict_proba(x_test) + if hasattr(model, "transform"): + model.transform(x_test) + + return model + + return _run From 49a22ecdd174dbe02ed09614658c49ee969932ce Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Fri, 6 Jan 2023 13:09:26 -0800 Subject: [PATCH 12/54] Add neighbors models (#728) --- newrelic/config.py | 84 +++++++++ newrelic/hooks/mlmodel_sklearn.py | 23 +++ .../mlmodel_sklearn/test_neighbors_models.py | 172 ++++++++++++++++++ 3 files changed, 279 insertions(+) create mode 100644 tests/mlmodel_sklearn/test_neighbors_models.py diff --git a/newrelic/config.py b/newrelic/config.py index 4f138a5de..2e350e2c2 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3082,6 +3082,90 @@ def _process_module_builtin_defaults(): "instrument_sklearn_svm_models", ) + _process_module_definition( + "sklearn.neighbors._classification", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neighbors_KRadius_models", + ) + + _process_module_definition( + "sklearn.neighbors.classification", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neighbors_KRadius_models", + ) + + _process_module_definition( + "sklearn.neighbors._graph", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neighbors_KRadius_models", + ) + + _process_module_definition( + "sklearn.neighbors._kde", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neighbors_models", + ) + + _process_module_definition( + "sklearn.neighbors.kde", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neighbors_models", + ) + + _process_module_definition( + "sklearn.neighbors._lof", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neighbors_models", + ) + + _process_module_definition( + "sklearn.neighbors.lof", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neighbors_models", + ) + + _process_module_definition( + "sklearn.neighbors._nca", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neighbors_models", + ) + + _process_module_definition( + "sklearn.neighbors._nearest_centroid", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neighbors_models", + ) + + _process_module_definition( + "sklearn.neighbors.nearest_centroid", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neighbors_models", + ) + + _process_module_definition( + "sklearn.neighbors._regression", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neighbors_KRadius_models", + ) + + _process_module_definition( + "sklearn.neighbors.regression", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neighbors_KRadius_models", + ) + + _process_module_definition( + "sklearn.neighbors._unsupervised", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neighbors_models", + ) + + _process_module_definition( + "sklearn.neighbors.unsupervised", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neighbors_models", + ) + _process_module_definition( "rest_framework.views", "newrelic.hooks.component_djangorestframework", diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index c023ad7a0..d5da71711 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -252,6 +252,18 @@ def instrument_sklearn_neural_network_models(module): _instrument_sklearn_models(module, model_classes) +def instrument_sklearn_neighbors_KRadius_models(module): + model_classes = ( + "KNeighborsClassifier", + "RadiusNeighborsClassifier", + "KNeighborsTransformer", + "RadiusNeighborsTransformer", + "KNeighborsRegressor", + "RadiusNeighborsRegressor", + ) + _instrument_sklearn_models(module, model_classes) + + def instrument_sklearn_svm_models(module): model_classes = ( "LinearSVC", @@ -282,6 +294,17 @@ def instrument_sklearn_pipeline_models(module): _instrument_sklearn_models(module, model_classes) +def instrument_sklearn_neighbors_models(module): + model_classes = ( + "KernelDensity", + "LocalOutlierFactor", + "NeighborhoodComponentsAnalysis", + "NearestCentroid", + "NearestNeighbors", + ) + _instrument_sklearn_models(module, model_classes) + + def instrument_sklearn_metrics(module): for scorer in METRIC_SCORERS: if hasattr(module, scorer): diff --git a/tests/mlmodel_sklearn/test_neighbors_models.py b/tests/mlmodel_sklearn/test_neighbors_models.py new file mode 100644 index 000000000..53a521157 --- /dev/null +++ b/tests/mlmodel_sklearn/test_neighbors_models.py @@ -0,0 +1,172 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from sklearn.neighbors import __init__ # noqa: Needed for get_package_version +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version +from newrelic.packages import six + +SKLEARN_VERSION = tuple(map(int, get_package_version("sklearn").split("."))) + + +@pytest.mark.parametrize( + "neighbors_model_name", + [ + "KNeighborsClassifier", + "RadiusNeighborsClassifier", + "KernelDensity", + "LocalOutlierFactor", + "NearestCentroid", + "KNeighborsRegressor", + "RadiusNeighborsRegressor", + "NearestNeighbors", + ], +) +def test_model_methods_wrapped_in_function_trace(neighbors_model_name, run_neighbors_model): + expected_scoped_metrics = { + "KNeighborsClassifier": [ + ("Function/MLModel/Sklearn/Named/KNeighborsClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/KNeighborsClassifier.predict", 2), + ("Function/MLModel/Sklearn/Named/KNeighborsClassifier.predict_proba", 1), + ], + "RadiusNeighborsClassifier": [ + ("Function/MLModel/Sklearn/Named/RadiusNeighborsClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/RadiusNeighborsClassifier.predict", 2), + ], + "KernelDensity": [ + ("Function/MLModel/Sklearn/Named/KernelDensity.fit", 1), + ("Function/MLModel/Sklearn/Named/KernelDensity.score", 1), + ], + "LocalOutlierFactor": [ + ("Function/MLModel/Sklearn/Named/LocalOutlierFactor.fit", 1), + ("Function/MLModel/Sklearn/Named/LocalOutlierFactor.predict", 1), + ], + "NearestCentroid": [ + ("Function/MLModel/Sklearn/Named/NearestCentroid.fit", 1), + ("Function/MLModel/Sklearn/Named/NearestCentroid.predict", 2), + ], + "KNeighborsRegressor": [ + ("Function/MLModel/Sklearn/Named/KNeighborsRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/KNeighborsRegressor.predict", 2), + ], + "RadiusNeighborsRegressor": [ + ("Function/MLModel/Sklearn/Named/RadiusNeighborsRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/RadiusNeighborsRegressor.predict", 2), + ], + "NearestNeighbors": [ + ("Function/MLModel/Sklearn/Named/NearestNeighbors.fit", 1), + ], + } + + expected_transaction_name = ( + "test_neighbors_models:test_model_methods_wrapped_in_function_trace.._test" + if six.PY3 + else "test_neighbors_models:_test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[neighbors_model_name], + rollup_metrics=expected_scoped_metrics[neighbors_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_neighbors_model(neighbors_model_name) + + _test() + + +@pytest.mark.skipif(SKLEARN_VERSION < (1, 0, 0), reason="Requires sklearn >= 1.0") +@pytest.mark.parametrize( + "neighbors_model_name", + [ + "KNeighborsTransformer", + "RadiusNeighborsTransformer", + "NeighborhoodComponentsAnalysis", + "RadiusNeighborsClassifier", + ], +) +def test_above_v1_0_model_methods_wrapped_in_function_trace(neighbors_model_name, run_neighbors_model): + expected_scoped_metrics = { + "KNeighborsTransformer": [ + ("Function/MLModel/Sklearn/Named/KNeighborsTransformer.fit", 1), + ("Function/MLModel/Sklearn/Named/KNeighborsTransformer.transform", 1), + ], + "RadiusNeighborsTransformer": [ + ("Function/MLModel/Sklearn/Named/RadiusNeighborsTransformer.fit", 1), + ("Function/MLModel/Sklearn/Named/RadiusNeighborsTransformer.transform", 1), + ], + "NeighborhoodComponentsAnalysis": [ + ("Function/MLModel/Sklearn/Named/NeighborhoodComponentsAnalysis.fit", 1), + ("Function/MLModel/Sklearn/Named/NeighborhoodComponentsAnalysis.transform", 1), + ], + "RadiusNeighborsClassifier": [ + ("Function/MLModel/Sklearn/Named/RadiusNeighborsClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/RadiusNeighborsClassifier.predict", 2), + ("Function/MLModel/Sklearn/Named/RadiusNeighborsClassifier.predict_proba", 3), # Added in v1.0 + ], + } + expected_transaction_name = ( + "test_neighbors_models:test_above_v1_0_model_methods_wrapped_in_function_trace.._test" + if six.PY3 + else "test_neighbors_models:_test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[neighbors_model_name], + rollup_metrics=expected_scoped_metrics[neighbors_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_neighbors_model(neighbors_model_name) + + _test() + + +@pytest.fixture +def run_neighbors_model(): + def _run(neighbors_model_name): + import sklearn.neighbors + from sklearn.datasets import load_iris + from sklearn.model_selection import train_test_split + + X, y = load_iris(return_X_y=True) + x_train, x_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) + + kwargs = {} + if neighbors_model_name == "LocalOutlierFactor": + kwargs = {"novelty": True} + clf = getattr(sklearn.neighbors, neighbors_model_name)(**kwargs) + + model = clf.fit(x_train, y_train) + if hasattr(model, "predict"): + model.predict(x_test) + if hasattr(model, "score"): + model.score(x_test, y_test) + if hasattr(model, "predict_proba"): + model.predict_proba(x_test) + if hasattr(model, "transform"): + model.transform(x_test) + + return model + + return _run From c4f1157baafc2c85694110fc40fa2693172634ad Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Fri, 6 Jan 2023 15:01:24 -0800 Subject: [PATCH 13/54] Add mixture models (#725) --- newrelic/config.py | 24 ++++++ newrelic/hooks/mlmodel_sklearn.py | 8 ++ tests/mlmodel_sklearn/test_mixture_models.py | 85 ++++++++++++++++++++ 3 files changed, 117 insertions(+) create mode 100644 tests/mlmodel_sklearn/test_mixture_models.py diff --git a/newrelic/config.py b/newrelic/config.py index 2e350e2c2..c87890bbc 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3046,6 +3046,30 @@ def _process_module_builtin_defaults(): "instrument_sklearn_cluster_clustering_models", ) + _process_module_definition( + "sklearn.mixture._bayesian_mixture", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_mixture_models", + ) + + _process_module_definition( + "sklearn.mixture.bayesian_mixture", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_mixture_models", + ) + + _process_module_definition( + "sklearn.mixture._gaussian_mixture", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_mixture_models", + ) + + _process_module_definition( + "sklearn.mixture.gaussian_mixture", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_mixture_models", + ) + _process_module_definition( "sklearn.pipeline", "newrelic.hooks.mlmodel_sklearn", diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index d5da71711..ba77bbf30 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -243,6 +243,14 @@ def instrument_sklearn_cluster_kmeans_models(module): _instrument_sklearn_models(module, model_classes) +def instrument_sklearn_mixture_models(module): + model_classes = ( + "GaussianMixture", + "BayesianGaussianMixture", + ) + _instrument_sklearn_models(module, model_classes) + + def instrument_sklearn_neural_network_models(module): model_classes = ( "BernoulliRBM", diff --git a/tests/mlmodel_sklearn/test_mixture_models.py b/tests/mlmodel_sklearn/test_mixture_models.py new file mode 100644 index 000000000..7ef838126 --- /dev/null +++ b/tests/mlmodel_sklearn/test_mixture_models.py @@ -0,0 +1,85 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.packages import six + + +@pytest.mark.parametrize( + "mixture_model_name", + [ + "GaussianMixture", + "BayesianGaussianMixture", + ], +) +def test_model_methods_wrapped_in_function_trace(mixture_model_name, run_mixture_model): + expected_scoped_metrics = { + "GaussianMixture": [ + ("Function/MLModel/Sklearn/Named/GaussianMixture.fit", 1), + ("Function/MLModel/Sklearn/Named/GaussianMixture.predict", 1), + ("Function/MLModel/Sklearn/Named/GaussianMixture.predict_proba", 1), + ("Function/MLModel/Sklearn/Named/GaussianMixture.score", 1), + ], + "BayesianGaussianMixture": [ + ("Function/MLModel/Sklearn/Named/BayesianGaussianMixture.fit", 1), + ("Function/MLModel/Sklearn/Named/BayesianGaussianMixture.predict", 1), + ("Function/MLModel/Sklearn/Named/BayesianGaussianMixture.predict_proba", 1), + ("Function/MLModel/Sklearn/Named/BayesianGaussianMixture.score", 1), + ], + } + + expected_transaction_name = ( + "test_mixture_models:test_model_methods_wrapped_in_function_trace.._test" + if six.PY3 + else "test_mixture_models:_test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[mixture_model_name], + rollup_metrics=expected_scoped_metrics[mixture_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_mixture_model(mixture_model_name) + + _test() + + +@pytest.fixture +def run_mixture_model(): + def _run(mixture_model_name): + import sklearn.mixture + from sklearn.datasets import load_iris + from sklearn.model_selection import train_test_split + + X, y = load_iris(return_X_y=True) + x_train, x_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) + + clf = getattr(sklearn.mixture, mixture_model_name)() + + model = clf.fit(x_train, y_train) + model.predict(x_test) + model.score(x_test, y_test) + model.predict_proba(x_test) + + return model + + return _run From 88391312493d75f637134fe691e042d436491b58 Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Fri, 6 Jan 2023 15:40:34 -0800 Subject: [PATCH 14/54] Add model selection model function traces (#726) * Add outline for model selection tests * Add some testing to model selection * Add hooks * Add estimator * Finish testing for model selection --- newrelic/config.py | 6 ++ newrelic/hooks/mlmodel_sklearn.py | 8 ++ .../test_model_selection_models.py | 99 +++++++++++++++++++ 3 files changed, 113 insertions(+) create mode 100644 tests/mlmodel_sklearn/test_model_selection_models.py diff --git a/newrelic/config.py b/newrelic/config.py index c87890bbc..5d3d08c2d 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3046,6 +3046,12 @@ def _process_module_builtin_defaults(): "instrument_sklearn_cluster_clustering_models", ) + _process_module_definition( + "sklearn.model_selection._search", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_model_selection_models", + ) + _process_module_definition( "sklearn.mixture._bayesian_mixture", "newrelic.hooks.mlmodel_sklearn", diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index ba77bbf30..fbdb97421 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -243,6 +243,14 @@ def instrument_sklearn_cluster_kmeans_models(module): _instrument_sklearn_models(module, model_classes) +def instrument_sklearn_model_selection_models(module): + model_classes = ( + "GridSearchCV", + "RandomizedSearchCV", + ) + _instrument_sklearn_models(module, model_classes) + + def instrument_sklearn_mixture_models(module): model_classes = ( "GaussianMixture", diff --git a/tests/mlmodel_sklearn/test_model_selection_models.py b/tests/mlmodel_sklearn/test_model_selection_models.py new file mode 100644 index 000000000..cadf7e64c --- /dev/null +++ b/tests/mlmodel_sklearn/test_model_selection_models.py @@ -0,0 +1,99 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from sklearn.ensemble import AdaBoostClassifier +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.packages import six + + +@pytest.mark.parametrize( + "model_selection_model_name", + [ + "GridSearchCV", + "RandomizedSearchCV", + ], +) +def test_model_methods_wrapped_in_function_trace(model_selection_model_name, run_model_selection_model): + expected_scoped_metrics = { + "GridSearchCV": [ + ("Function/MLModel/Sklearn/Named/GridSearchCV.fit", 1), + ("Function/MLModel/Sklearn/Named/GridSearchCV.predict", 1), + ("Function/MLModel/Sklearn/Named/GridSearchCV.predict_log_proba", 1), + ("Function/MLModel/Sklearn/Named/GridSearchCV.predict_proba", 1), + ("Function/MLModel/Sklearn/Named/GridSearchCV.score", 1), + ], + "RandomizedSearchCV": [ + ("Function/MLModel/Sklearn/Named/RandomizedSearchCV.fit", 1), + ("Function/MLModel/Sklearn/Named/RandomizedSearchCV.predict", 1), + ("Function/MLModel/Sklearn/Named/RandomizedSearchCV.predict_log_proba", 1), + ("Function/MLModel/Sklearn/Named/RandomizedSearchCV.predict_proba", 1), + ("Function/MLModel/Sklearn/Named/RandomizedSearchCV.score", 1), + ], + } + + expected_transaction_name = ( + "test_model_selection_models:test_model_methods_wrapped_in_function_trace.._test" + if six.PY3 + else "test_model_selection_models:_test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[model_selection_model_name], + rollup_metrics=expected_scoped_metrics[model_selection_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_model_selection_model(model_selection_model_name) + + _test() + + +@pytest.fixture +def run_model_selection_model(): + def _run(model_selection_model_name): + import sklearn.model_selection + from sklearn.datasets import load_iris + from sklearn.model_selection import train_test_split + + X, y = load_iris(return_X_y=True) + x_train, x_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) + + if model_selection_model_name == "GridSearchCV": + kwargs = {"estimator": AdaBoostClassifier(), "param_grid": {}} + else: + kwargs = {"estimator": AdaBoostClassifier(), "param_distributions": {}} + clf = getattr(sklearn.model_selection, model_selection_model_name)(**kwargs) + + model = clf.fit(x_train, y_train) + if hasattr(model, "predict"): + model.predict(x_test) + if hasattr(model, "score"): + model.score(x_test, y_test) + if hasattr(model, "predict_log_proba"): + model.predict_log_proba(x_test) + if hasattr(model, "predict_proba"): + model.predict_proba(x_test) + if hasattr(model, "transform"): + model.transform(x_test) + + return model + + return _run From 5a50130ba0f902fe5e693fc6896dcf4ba5bd55c2 Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Fri, 6 Jan 2023 16:16:54 -0800 Subject: [PATCH 15/54] Add naive bayes models (#724) --- newrelic/config.py | 6 + newrelic/hooks/mlmodel_sklearn.py | 11 ++ .../test_naive_bayes_models.py | 141 ++++++++++++++++++ 3 files changed, 158 insertions(+) create mode 100644 tests/mlmodel_sklearn/test_naive_bayes_models.py diff --git a/newrelic/config.py b/newrelic/config.py index 5d3d08c2d..fa52ba6c7 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3046,6 +3046,12 @@ def _process_module_builtin_defaults(): "instrument_sklearn_cluster_clustering_models", ) + _process_module_definition( + "sklearn.naive_bayes", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_naive_bayes_models", + ) + _process_module_definition( "sklearn.model_selection._search", "newrelic.hooks.mlmodel_sklearn", diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index fbdb97421..16d079df1 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -243,6 +243,17 @@ def instrument_sklearn_cluster_kmeans_models(module): _instrument_sklearn_models(module, model_classes) +def instrument_sklearn_naive_bayes_models(module): + model_classes = ( + "GaussianNB", + "MultinomialNB", + "ComplementNB", + "BernoulliNB", + "CategoricalNB", + ) + _instrument_sklearn_models(module, model_classes) + + def instrument_sklearn_model_selection_models(module): model_classes = ( "GridSearchCV", diff --git a/tests/mlmodel_sklearn/test_naive_bayes_models.py b/tests/mlmodel_sklearn/test_naive_bayes_models.py new file mode 100644 index 000000000..22dc6db1b --- /dev/null +++ b/tests/mlmodel_sklearn/test_naive_bayes_models.py @@ -0,0 +1,141 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from sklearn import __init__ # noqa: needed for get_package_version +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version +from newrelic.packages import six + +SKLEARN_VERSION = tuple(map(int, get_package_version("sklearn").split("."))) + + +@pytest.mark.skipif(SKLEARN_VERSION < (1, 0, 0), reason="Requires sklearn >= 1.0") +@pytest.mark.parametrize( + "naive_bayes_model_name", + [ + "CategoricalNB", + ], +) +def test_above_v1_0_model_methods_wrapped_in_function_trace(naive_bayes_model_name, run_naive_bayes_model): + expected_scoped_metrics = { + "CategoricalNB": [ + ("Function/MLModel/Sklearn/Named/CategoricalNB.fit", 1), + ("Function/MLModel/Sklearn/Named/CategoricalNB.predict", 1), + ("Function/MLModel/Sklearn/Named/CategoricalNB.predict_log_proba", 2), + ("Function/MLModel/Sklearn/Named/CategoricalNB.predict_proba", 1), + ], + } + expected_transaction_name = ( + "test_naive_bayes_models:test_above_v1_0_model_methods_wrapped_in_function_trace.._test" + if six.PY3 + else "test_naive_bayes_models:_test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[naive_bayes_model_name], + rollup_metrics=expected_scoped_metrics[naive_bayes_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_naive_bayes_model(naive_bayes_model_name) + + _test() + + +@pytest.mark.parametrize( + "naive_bayes_model_name", + [ + "GaussianNB", + "MultinomialNB", + "ComplementNB", + "BernoulliNB", + ], +) +def test_model_methods_wrapped_in_function_trace(naive_bayes_model_name, run_naive_bayes_model): + expected_scoped_metrics = { + "GaussianNB": [ + ("Function/MLModel/Sklearn/Named/GaussianNB.fit", 1), + ("Function/MLModel/Sklearn/Named/GaussianNB.predict", 1), + ("Function/MLModel/Sklearn/Named/GaussianNB.predict_log_proba", 2), + ("Function/MLModel/Sklearn/Named/GaussianNB.predict_proba", 1), + ], + "MultinomialNB": [ + ("Function/MLModel/Sklearn/Named/MultinomialNB.fit", 1), + ("Function/MLModel/Sklearn/Named/MultinomialNB.predict", 1), + ("Function/MLModel/Sklearn/Named/MultinomialNB.predict_log_proba", 2), + ("Function/MLModel/Sklearn/Named/MultinomialNB.predict_proba", 1), + ], + "ComplementNB": [ + ("Function/MLModel/Sklearn/Named/ComplementNB.fit", 1), + ("Function/MLModel/Sklearn/Named/ComplementNB.predict", 1), + ("Function/MLModel/Sklearn/Named/ComplementNB.predict_log_proba", 2), + ("Function/MLModel/Sklearn/Named/ComplementNB.predict_proba", 1), + ], + "BernoulliNB": [ + ("Function/MLModel/Sklearn/Named/BernoulliNB.fit", 1), + ("Function/MLModel/Sklearn/Named/BernoulliNB.predict", 1), + ("Function/MLModel/Sklearn/Named/BernoulliNB.predict_log_proba", 2), + ("Function/MLModel/Sklearn/Named/BernoulliNB.predict_proba", 1), + ], + } + + expected_transaction_name = ( + "test_naive_bayes_models:test_model_methods_wrapped_in_function_trace.._test" + if six.PY3 + else "test_naive_bayes_models:_test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[naive_bayes_model_name], + rollup_metrics=expected_scoped_metrics[naive_bayes_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_naive_bayes_model(naive_bayes_model_name) + + _test() + + +@pytest.fixture +def run_naive_bayes_model(): + def _run(naive_bayes_model_name): + import sklearn.naive_bayes + from sklearn.datasets import load_iris + from sklearn.model_selection import train_test_split + + X, y = load_iris(return_X_y=True) + x_train, x_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) + + clf = getattr(sklearn.naive_bayes, naive_bayes_model_name)() + + model = clf.fit(x_train, y_train) + if hasattr(model, "predict"): + model.predict(x_test) + if hasattr(model, "predict_log_proba"): + model.predict_log_proba(x_test) + if hasattr(model, "predict_proba"): + model.predict_proba(x_test) + + return model + + return _run From 734fa2a4bcb0fc914f3e3dd302bc9dee3bbd5acc Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Fri, 6 Jan 2023 18:24:10 -0800 Subject: [PATCH 16/54] Add multioutput models (#723) --- newrelic/config.py | 6 + newrelic/hooks/mlmodel_sklearn.py | 10 ++ .../test_multioutput_models.py | 129 ++++++++++++++++++ 3 files changed, 145 insertions(+) create mode 100644 tests/mlmodel_sklearn/test_multioutput_models.py diff --git a/newrelic/config.py b/newrelic/config.py index fa52ba6c7..fe2c89cec 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3046,6 +3046,12 @@ def _process_module_builtin_defaults(): "instrument_sklearn_cluster_clustering_models", ) + _process_module_definition( + "sklearn.multioutput", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_multioutput_models", + ) + _process_module_definition( "sklearn.naive_bayes", "newrelic.hooks.mlmodel_sklearn", diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index 16d079df1..38ccc966a 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -243,6 +243,16 @@ def instrument_sklearn_cluster_kmeans_models(module): _instrument_sklearn_models(module, model_classes) +def instrument_sklearn_multioutput_models(module): + model_classes = ( + "MultiOutputEstimator", + "MultiOutputClassifier", + "ClassifierChain", + "RegressorChain", + ) + _instrument_sklearn_models(module, model_classes) + + def instrument_sklearn_naive_bayes_models(module): model_classes = ( "GaussianNB", diff --git a/tests/mlmodel_sklearn/test_multioutput_models.py b/tests/mlmodel_sklearn/test_multioutput_models.py new file mode 100644 index 000000000..392328f28 --- /dev/null +++ b/tests/mlmodel_sklearn/test_multioutput_models.py @@ -0,0 +1,129 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from sklearn import __init__ # noqa: Needed for get_package_version +from sklearn.ensemble import AdaBoostClassifier +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version +from newrelic.packages import six + +SKLEARN_VERSION = tuple(map(int, get_package_version("sklearn").split("."))) + + +# Python 2 will not allow instantiation of abstract class +# (abstract method is __init__ here) +@pytest.mark.skipif(SKLEARN_VERSION >= (1, 0, 0) or six.PY2, reason="Requires sklearn < 1.0 and Python3") +@pytest.mark.parametrize( + "multioutput_model_name", + [ + "MultiOutputEstimator", + ], +) +def test_below_v1_0_model_methods_wrapped_in_function_trace(multioutput_model_name, run_multioutput_model): + expected_scoped_metrics = { + "MultiOutputEstimator": [ + ("Function/MLModel/Sklearn/Named/MultiOutputEstimator.fit", 1), + ("Function/MLModel/Sklearn/Named/MultiOutputEstimator.predict", 2), + ], + } + expected_transaction_name = ( + "test_multioutput_models:test_below_v1_0_model_methods_wrapped_in_function_trace.._test" + if six.PY3 + else "test_multioutput_models:_test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[multioutput_model_name], + rollup_metrics=expected_scoped_metrics[multioutput_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_multioutput_model(multioutput_model_name) + + _test() + + +@pytest.mark.parametrize( + "multioutput_model_name", + [ + "MultiOutputClassifier", + "ClassifierChain", + "RegressorChain", + ], +) +def test_above_v1_0_model_methods_wrapped_in_function_trace(multioutput_model_name, run_multioutput_model): + expected_scoped_metrics = { + "MultiOutputClassifier": [ + ("Function/MLModel/Sklearn/Named/MultiOutputClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/MultiOutputClassifier.predict_proba", 1), + ("Function/MLModel/Sklearn/Named/MultiOutputClassifier.score", 1), + ], + "ClassifierChain": [ + ("Function/MLModel/Sklearn/Named/ClassifierChain.fit", 1), + ("Function/MLModel/Sklearn/Named/ClassifierChain.predict_proba", 1), + ], + "RegressorChain": [ + ("Function/MLModel/Sklearn/Named/RegressorChain.fit", 1), + ], + } + expected_transaction_name = ( + "test_multioutput_models:test_above_v1_0_model_methods_wrapped_in_function_trace.._test" + if six.PY3 + else "test_multioutput_models:_test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[multioutput_model_name], + rollup_metrics=expected_scoped_metrics[multioutput_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_multioutput_model(multioutput_model_name) + + _test() + + +@pytest.fixture +def run_multioutput_model(): + def _run(multioutput_model_name): + import sklearn.multioutput + from sklearn.datasets import make_multilabel_classification + + X, y = make_multilabel_classification(n_classes=3, random_state=0) + + kwargs = {"estimator": AdaBoostClassifier()} + if multioutput_model_name in ["RegressorChain", "ClassifierChain"]: + kwargs = {"base_estimator": AdaBoostClassifier()} + clf = getattr(sklearn.multioutput, multioutput_model_name)(**kwargs) + + model = clf.fit(X, y) + if hasattr(model, "predict"): + model.predict(X) + if hasattr(model, "score"): + model.score(X, y) + if hasattr(model, "predict_proba"): + model.predict_proba(X) + + return model + + return _run From 58f02de08b55d2444e0bdf9a125f1b990081ab71 Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Fri, 6 Jan 2023 20:47:50 -0800 Subject: [PATCH 17/54] Add multiclass models (#722) --- newrelic/config.py | 6 ++ newrelic/hooks/mlmodel_sklearn.py | 9 ++ .../mlmodel_sklearn/test_multiclass_models.py | 91 +++++++++++++++++++ 3 files changed, 106 insertions(+) create mode 100644 tests/mlmodel_sklearn/test_multiclass_models.py diff --git a/newrelic/config.py b/newrelic/config.py index fe2c89cec..95de68aa1 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3046,6 +3046,12 @@ def _process_module_builtin_defaults(): "instrument_sklearn_cluster_clustering_models", ) + _process_module_definition( + "sklearn.multiclass", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_multiclass_models", + ) + _process_module_definition( "sklearn.multioutput", "newrelic.hooks.mlmodel_sklearn", diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index 38ccc966a..81e026192 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -243,6 +243,15 @@ def instrument_sklearn_cluster_kmeans_models(module): _instrument_sklearn_models(module, model_classes) +def instrument_sklearn_multiclass_models(module): + model_classes = ( + "OneVsRestClassifier", + "OneVsOneClassifier", + "OutputCodeClassifier", + ) + _instrument_sklearn_models(module, model_classes) + + def instrument_sklearn_multioutput_models(module): model_classes = ( "MultiOutputEstimator", diff --git a/tests/mlmodel_sklearn/test_multiclass_models.py b/tests/mlmodel_sklearn/test_multiclass_models.py new file mode 100644 index 000000000..dd10d76f1 --- /dev/null +++ b/tests/mlmodel_sklearn/test_multiclass_models.py @@ -0,0 +1,91 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from sklearn.ensemble import AdaBoostClassifier +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.packages import six + + +@pytest.mark.parametrize( + "multiclass_model_name", + [ + "OneVsRestClassifier", + "OneVsOneClassifier", + "OutputCodeClassifier", + ], +) +def test_model_methods_wrapped_in_function_trace(multiclass_model_name, run_multiclass_model): + expected_scoped_metrics = { + "OneVsRestClassifier": [ + ("Function/MLModel/Sklearn/Named/OneVsRestClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/OneVsRestClassifier.predict", 1), + ("Function/MLModel/Sklearn/Named/OneVsRestClassifier.predict_proba", 1), + ], + "OneVsOneClassifier": [ + ("Function/MLModel/Sklearn/Named/OneVsOneClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/OneVsOneClassifier.predict", 1), + ], + "OutputCodeClassifier": [ + ("Function/MLModel/Sklearn/Named/OutputCodeClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/OutputCodeClassifier.predict", 1), + ], + } + + expected_transaction_name = ( + "test_multiclass_models:test_model_methods_wrapped_in_function_trace.._test" + if six.PY3 + else "test_multiclass_models:_test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[multiclass_model_name], + rollup_metrics=expected_scoped_metrics[multiclass_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_multiclass_model(multiclass_model_name) + + _test() + + +@pytest.fixture +def run_multiclass_model(): + def _run(multiclass_model_name): + import sklearn.multiclass + from sklearn.datasets import load_iris + from sklearn.model_selection import train_test_split + + X, y = load_iris(return_X_y=True) + x_train, x_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) + + # This is an example of a model that has all the available attributes + # We could have choosen any estimator that has predict, score, + # predict_log_proba, and predict_proba + clf = getattr(sklearn.multiclass, multiclass_model_name)(estimator=AdaBoostClassifier()) + + model = clf.fit(x_train, y_train) + model.predict(x_test) + if hasattr(model, "predict_proba"): + model.predict_proba(x_test) + + return model + + return _run From f880cde32ce25862ba810d936486509a91ef7837 Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Sat, 7 Jan 2023 17:58:13 -0800 Subject: [PATCH 18/54] Add kernel ridge model function traces (#721) * Add kernel ridge models * Modify VotingRegressor test --- newrelic/config.py | 6 ++ newrelic/hooks/mlmodel_sklearn.py | 5 ++ .../test_kernel_ridge_models.py | 74 +++++++++++++++++++ 3 files changed, 85 insertions(+) create mode 100644 tests/mlmodel_sklearn/test_kernel_ridge_models.py diff --git a/newrelic/config.py b/newrelic/config.py index 95de68aa1..bc8ec395d 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2902,6 +2902,12 @@ def _process_module_builtin_defaults(): "instrument_sklearn_ensemble_hist_models", ) + _process_module_definition( + "sklearn.kernel_ridge", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_kernel_ridge_models", + ) + _process_module_definition( "sklearn.neural_network._multilayer_perceptron", "newrelic.hooks.mlmodel_sklearn", diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index 81e026192..cbb94fc67 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -201,6 +201,11 @@ def instrument_sklearn_ensemble_hist_models(module): _instrument_sklearn_models(module, model_classes) +def instrument_sklearn_kernel_ridge_models(module): + model_classes = ("KernelRidge",) + _instrument_sklearn_models(module, model_classes) + + def instrument_sklearn_calibration_models(module): model_classes = ("CalibratedClassifierCV",) _instrument_sklearn_models(module, model_classes) diff --git a/tests/mlmodel_sklearn/test_kernel_ridge_models.py b/tests/mlmodel_sklearn/test_kernel_ridge_models.py new file mode 100644 index 000000000..1cbdddc31 --- /dev/null +++ b/tests/mlmodel_sklearn/test_kernel_ridge_models.py @@ -0,0 +1,74 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.packages import six + + +@pytest.mark.parametrize( + "kernel_ridge_model_name", + [ + "KernelRidge", + ], +) +def test_model_methods_wrapped_in_function_trace(kernel_ridge_model_name, run_kernel_ridge_model): + expected_scoped_metrics = { + "KernelRidge": [ + ("Function/MLModel/Sklearn/Named/KernelRidge.fit", 1), + ("Function/MLModel/Sklearn/Named/KernelRidge.predict", 1), + ], + } + + expected_transaction_name = ( + "test_kernel_ridge_models:test_model_methods_wrapped_in_function_trace.._test" + if six.PY3 + else "test_kernel_ridge_models:_test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[kernel_ridge_model_name], + rollup_metrics=expected_scoped_metrics[kernel_ridge_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_kernel_ridge_model(kernel_ridge_model_name) + + _test() + + +@pytest.fixture +def run_kernel_ridge_model(): + def _run(kernel_ridge_model_name): + import sklearn.kernel_ridge + from sklearn.datasets import load_iris + from sklearn.model_selection import train_test_split + + X, y = load_iris(return_X_y=True) + x_train, x_test, y_train, _ = train_test_split(X, y, stratify=y, random_state=0) + + clf = getattr(sklearn.kernel_ridge, kernel_ridge_model_name)() + + model = clf.fit(x_train, y_train) + model.predict(x_test) + + return model + + return _run From 0d56315e91584ef3aaafdd0c1abef47eb6e7de1a Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Mon, 9 Jan 2023 19:43:44 +0100 Subject: [PATCH 19/54] Add custom feature events for sklearn (#727) * Add function traces around model methods * Add test for multiple calls to model method * Fixup: add comments & organize * Refactor * Instrument predict function. * Add data trains fixture. * Add testing and cleanup for custom feature events. * Update test_tree_models. * Add back training step logic to predict proxy. * Remove unused files. * Add tree model function traces (#691) * Add sklearn to tox * Add function traces around model methods * Support Python 2.7 & 3.7 sklearn * Add test for multiple calls to model method * Fixup: add comments & organize * Refactor * Follow two digit convention * Make if-else a one-liner * Abstract to re-usable instrumentation function * Use wrap_method_trace & change to Function group Co-authored-by: Timothy Pansino Co-authored-by: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Co-authored-by: Uma Annamalai * Fixup: use if-else one-liner * Use hasattr instead of model name check * Change component_sklearn to mlmodel_sklearn * Fixup: replace in model names with hasattr Co-authored-by: Timothy Pansino Co-authored-by: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Co-authored-by: Uma Annamalai * Add config setting for sklearn inference event capture. (#706) * Add sklearn to tox * Add function traces around model methods * Support Python 2.7 & 3.7 sklearn * Add test for multiple calls to model method * Fixup: add comments & organize * Refactor * Follow two digit convention * Make if-else a one-liner * Abstract to re-usable instrumentation function * Add ML inference event capture config setting. * [Mega-Linter] Apply linters fixes * Fixup: remove component_sklearn files * Add high security mode testing for ML events setting. * [Mega-Linter] Apply linters fixes Co-authored-by: Hannah Stepanek Co-authored-by: umaannamalai * Capture scorer results (#694) * Add score results attributes to metric scorers * Test un-subclassable types * [Mega-Linter] Apply linters fixes * [Mega-Linter] Apply linters fixes * Trigger tests * Remove custom subclassing code. * [Mega-Linter] Apply linters fixes * Remove unused function * Add test for iterable score results * Change name of object proxy * Fixup: rename proxy in tests too Co-authored-by: hmstepanek Co-authored-by: Tim Pansino Co-authored-by: TimPansino * Add ensemble model function traces (#697) * Add sklearn to tox * Add function traces around model methods * Support Python 2.7 & 3.7 sklearn * Add test for multiple calls to model method * Fixup: add comments & organize * Add ensemble models * Add ensemble model tests * Edit tests * Add ensemble library models from sklearn * Start tests with empty commit * Clean up tests * Fix tests for various versions of sklearn * Fix ensemble tests with changes from tree PR * [Mega-Linter] Apply linters fixes * Remove breakpoints * Remove commited config files * Group tests into more readable format * Pin startlette latest < 0.23.1 * Convert PY3 checks to one-liners * Use tuple checks for sklearn version Use tuple checks for sklearn version, string checks can result in unexpected out of order comparisons. Also use direct comparisons for easier readability. * Fix VotingRegressor test Co-authored-by: Hannah Stepanek Co-authored-by: lrafeei * Add function traces around model methods * Add test for multiple calls to model method * Fixup: add comments & organize * Refactor * Instrument predict function. * Add data trains fixture. * Add testing and cleanup for custom feature events. * Update test_tree_models. * Include training step in metric scorer name (#712) * Include training step in scorer name * Add fit_predict data proxying * Remove name comments * Fix predict being called before fit * Re-use existing fixture * Add cluster model function traces (#700) * Add sklearn to tox * Add function traces around model methods * Support Python 2.7 & 3.7 sklearn * Add test for multiple calls to model method * Fixup: add comments & organize * Add ensemble models * Add ensemble model tests * Edit tests * Add ensemble library models from sklearn * Start tests with empty commit * Clean up tests * Add cluster model instrumentaton * Fix tests for various versions of sklearn * Fix ensemble tests with changes from tree PR * [Mega-Linter] Apply linters fixes * Fix some cluster model tests * Fix tests after ensemble PR merge * Add transform to tests * Remove accidental commits * Modify cluster tests to be more readable * Break up instrumentation models * Remove duplicate ensemble module defs * Modify VotingRegressor test Co-authored-by: Hannah Stepanek Co-authored-by: lrafeei * Add calibration model function traces (#709) * Add sklearn to tox * Add function traces around model methods * Support Python 2.7 & 3.7 sklearn * Add test for multiple calls to model method * Fixup: add comments & organize * Add ensemble models * Add ensemble model tests * Edit tests * Add ensemble library models from sklearn * Start tests with empty commit * Clean up tests * Fix tests for various versions of sklearn * Fix ensemble tests with changes from tree PR * [Mega-Linter] Apply linters fixes * Remove breakpoints * Create tests/instrumentation for calibration models * Fix calibration tests * Remove commented out code * Remove yaml file in commit * Remove duplicate ensemble module defs Co-authored-by: Hannah Stepanek Co-authored-by: lrafeei * Add back training step logic to predict proxy. * Remove unused files. * Address py27 test failures and review comments. * Fix py 3.7 failures. * Remove old component_sklearn.py file * Fix lint errors * [Mega-Linter] Apply linters fixes * Merge redis fix. Co-authored-by: Hannah Stepanek Co-authored-by: Timothy Pansino Co-authored-by: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Co-authored-by: Uma Annamalai Co-authored-by: hmstepanek Co-authored-by: Tim Pansino Co-authored-by: lrafeei --- newrelic/hooks/datastore_redis.py | 5 + newrelic/hooks/mlmodel_sklearn.py | 64 +++ .../_validate_custom_events.py | 96 ++++ tests/mlmodel_sklearn/test_feature_events.py | 414 ++++++++++++++++++ tox.ini | 1 + 5 files changed, 580 insertions(+) create mode 100644 tests/mlmodel_sklearn/_validate_custom_events.py create mode 100644 tests/mlmodel_sklearn/test_feature_events.py diff --git a/newrelic/hooks/datastore_redis.py b/newrelic/hooks/datastore_redis.py index 26d1facb0..8c8b6f7a6 100644 --- a/newrelic/hooks/datastore_redis.py +++ b/newrelic/hooks/datastore_redis.py @@ -218,7 +218,12 @@ "insertnx", "keys", "lastsave", + "latency_doctor", + "latency_graph", "latency_histogram", + "latency_history", + "latency_latest", + "latency_reset", "lcs", "lindex", "linsert", diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index cbb94fc67..7db993a9f 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -13,11 +13,13 @@ # limitations under the License. import sys +import uuid from newrelic.api.function_trace import FunctionTrace from newrelic.api.time_trace import current_trace from newrelic.api.transaction import current_transaction from newrelic.common.object_wrapper import ObjectProxy, wrap_function_wrapper +from newrelic.core.config import global_settings METHODS_TO_WRAP = ("predict", "fit", "fit_predict", "predict_log_proba", "predict_proba", "transform", "score") METRIC_SCORERS = ( @@ -75,12 +77,74 @@ def _nr_wrapper_method(wrapped, instance, args, kwargs): # _nr_wrapped attrs that will attach model info to the data. if method in ("predict", "fit_predict"): training_step = getattr(instance, "_nr_wrapped_training_step", "Unknown") + wrap_predict(transaction, _class, wrapped, instance, args, kwargs) return PredictReturnTypeProxy(return_val, model_name=_class, training_step=training_step) return return_val wrap_function_wrapper(module, "%s.%s" % (_class, method), _nr_wrapper_method) +def find_type_category(value): + value_type = None + python_type = str(type(value)) + if "int" in python_type or "float" in python_type or "complex" in python_type: + value_type = "numerical" + elif "bool" in python_type: + value_type = "bool" + elif "str" in python_type or "unicode" in python_type: + value_type = "str" + return value_type + + +def bind_predict(X, *args, **kwargs): + return X + + +def wrap_predict(transaction, _class, wrapped, instance, args, kwargs): + data_set = bind_predict(*args, **kwargs) + inference_id = uuid.uuid4() + model_name = getattr(instance, "_nr_wrapped_name", _class) + model_version = getattr(instance, "_nr_wrapped_version", "0.0.0") + + settings = transaction.settings if transaction.settings is not None else global_settings() + if settings and settings.machine_learning and settings.machine_learning.inference_event_value.enabled: + # Pandas Dataframe + pd = sys.modules.get("pandas", None) + if pd and isinstance(data_set, pd.DataFrame): + for (colname, colval) in data_set.iteritems(): + for value in colval.values: + value_type = data_set[colname].dtype.name + if value_type == "category": + value_type = "categorical" + else: + value_type = find_type_category(value) + transaction.record_custom_event( + "ML Model Feature Event", + { + "inference_id": inference_id, + "model_name": model_name, + "model_version": model_version, + "feature_name": colname, + "type": value_type, + "value": str(value), + }, + ) + else: + for feature in data_set: + for col_index, value in enumerate(feature): + transaction.record_custom_event( + "ML Model Feature Event", + { + "inference_id": inference_id, + "model_name": model_name, + "model_version": model_version, + "feature_name": str(col_index), + "type": find_type_category(value), + "value": str(value), + }, + ) + + def _nr_instrument_model(module, model_class): for method_name in METHODS_TO_WRAP: if hasattr(getattr(module, model_class), method_name): diff --git a/tests/mlmodel_sklearn/_validate_custom_events.py b/tests/mlmodel_sklearn/_validate_custom_events.py new file mode 100644 index 000000000..77a63497f --- /dev/null +++ b/tests/mlmodel_sklearn/_validate_custom_events.py @@ -0,0 +1,96 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy + +from testing_support.fixtures import catch_background_exceptions + +from newrelic.common.object_wrapper import function_wrapper, transient_function_wrapper +from newrelic.packages import six + + +def validate_custom_events(events): + @function_wrapper + def _validate_wrapper(wrapped, instance, args, kwargs): + + record_called = [] + recorded_events = [] + + @transient_function_wrapper("newrelic.core.stats_engine", "StatsEngine.record_transaction") + @catch_background_exceptions + def _validate_custom_events(wrapped, instance, args, kwargs): + record_called.append(True) + try: + result = wrapped(*args, **kwargs) + except: + raise + else: + txn = args[0] + recorded_events[:] = [] + recorded_events.extend(list(txn.custom_events)) + + return result + + _new_wrapper = _validate_custom_events(wrapped) + val = _new_wrapper(*args, **kwargs) + assert record_called + custom_events = copy.copy(recorded_events) + + record_called[:] = [] + recorded_events[:] = [] + for expected in events: + matching_custom_events = 0 + mismatches = [] + for captured in custom_events: + if _check_custom_event_attributes(expected, captured, mismatches): + matching_custom_events += 1 + assert matching_custom_events == 1, _custom_event_details(matching_custom_events, custom_events, mismatches) + + return val + + def _check_custom_event_attributes(expected, captured, mismatches): + assert len(captured) == 2 # [intrinsic, user attributes] + expected_intrinsics = expected.get("intrinsics", {}) + expected_users = expected.get("users", {}) + intrinsics = captured[0] + users = captured[1] + + def _validate(expected, captured): + for key, value in six.iteritems(expected): + if key in captured: + + captured_value = captured[key] + else: + mismatches.append("key: %s, value:<%s><%s>" % (key, value, getattr(captured, key, None))) + return False + + if value is not None: + if value != captured_value: + mismatches.append("key: %s, value:<%s><%s>" % (key, value, captured_value)) + return False + + return True + + return _validate(expected_intrinsics, intrinsics) and _validate(expected_users, users) + + def _custom_event_details(matching_custom_events, captured, mismatches): + details = [ + "matching_custom_events=%d" % matching_custom_events, + "mismatches=%s" % mismatches, + "captured_events=%s" % captured, + ] + + return "\n".join(details) + + return _validate_wrapper diff --git a/tests/mlmodel_sklearn/test_feature_events.py b/tests/mlmodel_sklearn/test_feature_events.py new file mode 100644 index 000000000..044a6f293 --- /dev/null +++ b/tests/mlmodel_sklearn/test_feature_events.py @@ -0,0 +1,414 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +import numpy as np +import pandas +from _validate_custom_events import validate_custom_events +from testing_support.fixtures import ( + reset_core_stats_engine, + validate_custom_event_count, +) + +from newrelic.api.background_task import background_task + +pandas_df_category_recorded_custom_events = [ + { + "users": { + "inference_id": None, + "model_name": "DecisionTreeClassifier", + "model_version": "0.0.0", + "feature_name": "col1", + "type": "categorical", + "value": "2.0", + } + }, + { + "users": { + "inference_id": None, + "model_name": "DecisionTreeClassifier", + "model_version": "0.0.0", + "feature_name": "col1", + "type": "categorical", + "value": "3.0", + } + }, + { + "users": { + "inference_id": None, + "model_name": "DecisionTreeClassifier", + "model_version": "0.0.0", + "feature_name": "col2", + "type": "categorical", + "value": "4.0", + } + }, + { + "users": { + "inference_id": None, + "model_name": "DecisionTreeClassifier", + "model_version": "0.0.0", + "feature_name": "col2", + "type": "categorical", + "value": "1.0", + } + }, +] + + +@reset_core_stats_engine() +def test_pandas_df_categorical_feature_event(): + @validate_custom_events(pandas_df_category_recorded_custom_events) + @validate_custom_event_count(count=4) + @background_task() + def _test(): + import sklearn.tree + + clf = getattr(sklearn.tree, "DecisionTreeClassifier")(random_state=0) + model = clf.fit( + pandas.DataFrame({"col1": [0, 0], "col2": [1, 1]}, dtype="category"), pandas.DataFrame({"label": [0, 1]}) + ) + + labels = model.predict(pandas.DataFrame({"col1": [2.0, 3.0], "col2": [4.0, 1.0]}, dtype="category")) + return model + + _test() + + +pandas_df_bool_recorded_custom_events = [ + { + "users": { + "inference_id": None, + "model_name": "DecisionTreeClassifier", + "model_version": "0.0.0", + "feature_name": "col1", + "type": "bool", + "value": "True", + } + }, + { + "users": { + "inference_id": None, + "model_name": "DecisionTreeClassifier", + "model_version": "0.0.0", + "feature_name": "col1", + "type": "bool", + "value": "False", + } + }, + { + "users": { + "inference_id": None, + "model_name": "DecisionTreeClassifier", + "model_version": "0.0.0", + "feature_name": "col2", + "type": "bool", + "value": "True", + } + }, + { + "users": { + "inference_id": None, + "model_name": "DecisionTreeClassifier", + "model_version": "0.0.0", + "feature_name": "col2", + "type": "bool", + "value": "False", + } + }, +] + + +@reset_core_stats_engine() +def test_pandas_df_bool_feature_event(): + @validate_custom_events(pandas_df_bool_recorded_custom_events) + @validate_custom_event_count(count=4) + @background_task() + def _test(): + import sklearn.tree + + dtype_name = "bool" if sys.version_info < (3, 8) else "boolean" + x_train = pandas.DataFrame({"col1": [True, False], "col2": [True, False]}, dtype=dtype_name) + y_train = pandas.DataFrame({"label": [True, False]}, dtype=dtype_name) + x_test = pandas.DataFrame({"col1": [True, False], "col2": [True, False]}, dtype=dtype_name) + + clf = getattr(sklearn.tree, "DecisionTreeClassifier")(random_state=0) + model = clf.fit(x_train, y_train) + + labels = model.predict(x_test) + return model + + _test() + + +pandas_df_float_recorded_custom_events = [ + { + "users": { + "inference_id": None, + "model_name": "DecisionTreeRegressor", + "model_version": "0.0.0", + "feature_name": "col1", + "type": "numerical", + "value": "100.0", + } + }, + { + "users": { + "inference_id": None, + "model_name": "DecisionTreeRegressor", + "model_version": "0.0.0", + "feature_name": "col1", + "type": "numerical", + "value": "200.0", + } + }, + { + "users": { + "inference_id": None, + "model_name": "DecisionTreeRegressor", + "model_version": "0.0.0", + "feature_name": "col2", + "type": "numerical", + "value": "300.0", + } + }, + { + "users": { + "inference_id": None, + "model_name": "DecisionTreeRegressor", + "model_version": "0.0.0", + "feature_name": "col2", + "type": "numerical", + "value": "400.0", + } + }, +] + + +@reset_core_stats_engine() +def test_pandas_df_float_feature_event(): + @validate_custom_events(pandas_df_float_recorded_custom_events) + @validate_custom_event_count(count=4) + @background_task() + def _test(): + import sklearn.tree + + x_train = pandas.DataFrame({"col1": [120.0, 254.0], "col2": [236.9, 234.5]}, dtype="float64") + y_train = pandas.DataFrame({"label": [345.6, 456.7]}, dtype="float64") + x_test = pandas.DataFrame({"col1": [100.0, 200.0], "col2": [300.0, 400.0]}, dtype="float64") + + clf = getattr(sklearn.tree, "DecisionTreeRegressor")(random_state=0) + + model = clf.fit(x_train, y_train) + labels = model.predict(x_test) + + return model + + _test() + + +int_list_recorded_custom_events = [ + { + "users": { + "inference_id": None, + "model_name": "ExtraTreeRegressor", + "model_version": "0.0.0", + "feature_name": "0", + "type": "numerical", + "value": "1", + } + }, + { + "users": { + "inference_id": None, + "model_name": "ExtraTreeRegressor", + "model_version": "0.0.0", + "feature_name": "1", + "type": "numerical", + "value": "2", + } + }, + { + "users": { + "inference_id": None, + "model_name": "ExtraTreeRegressor", + "model_version": "0.0.0", + "feature_name": "0", + "type": "numerical", + "value": "3", + } + }, + { + "users": { + "inference_id": None, + "model_name": "ExtraTreeRegressor", + "model_version": "0.0.0", + "feature_name": "1", + "type": "numerical", + "value": "4", + } + }, +] + + +@reset_core_stats_engine() +def test_int_list(): + @validate_custom_events(int_list_recorded_custom_events) + @validate_custom_event_count(count=4) + @background_task() + def _test(): + import sklearn.tree + + x_train = [[0, 0], [1, 1]] + y_train = [0, 1] + x_test = [[1, 2], [3, 4]] + + clf = getattr(sklearn.tree, "ExtraTreeRegressor")(random_state=0) + model = clf.fit(x_train, y_train) + + labels = model.predict(x_test) + return model + + _test() + + +numpy_int_recorded_custom_events = [ + { + "users": { + "inference_id": None, + "model_name": "ExtraTreeRegressor", + "model_version": "0.0.0", + "feature_name": "0", + "type": "numerical", + "value": "12", + } + }, + { + "users": { + "inference_id": None, + "model_name": "ExtraTreeRegressor", + "model_version": "0.0.0", + "feature_name": "1", + "type": "numerical", + "value": "13", + } + }, + { + "users": { + "inference_id": None, + "model_name": "ExtraTreeRegressor", + "model_version": "0.0.0", + "feature_name": "0", + "type": "numerical", + "value": "14", + } + }, + { + "users": { + "inference_id": None, + "model_name": "ExtraTreeRegressor", + "model_version": "0.0.0", + "feature_name": "1", + "type": "numerical", + "value": "15", + } + }, +] + + +@reset_core_stats_engine() +def test_numpy_int_array(): + @validate_custom_events(numpy_int_recorded_custom_events) + @validate_custom_event_count(count=4) + @background_task() + def _test(): + import sklearn.tree + + x_train = np.array([[10, 10], [11, 11]], dtype="int") + y_train = np.array([10, 11], dtype="int") + x_test = np.array([[12, 13], [14, 15]], dtype="int") + + clf = getattr(sklearn.tree, "ExtraTreeRegressor")(random_state=0) + model = clf.fit(x_train, y_train) + + labels = model.predict(x_test) + return model + + _test() + + +numpy_str_recorded_custom_events = [ + { + "users": { + "inference_id": None, + "model_name": "DecisionTreeClassifier", + "model_version": "0.0.0", + "feature_name": "0", + "type": "str", + "value": "20", + } + }, + { + "users": { + "inference_id": None, + "model_name": "DecisionTreeClassifier", + "model_version": "0.0.0", + "feature_name": "1", + "type": "str", + "value": "21", + } + }, + { + "users": { + "inference_id": None, + "model_name": "DecisionTreeClassifier", + "model_version": "0.0.0", + "feature_name": "0", + "type": "str", + "value": "22", + } + }, + { + "users": { + "inference_id": None, + "model_name": "DecisionTreeClassifier", + "model_version": "0.0.0", + "feature_name": "1", + "type": "str", + "value": "23", + } + }, +] + + +@reset_core_stats_engine() +def test_numpy_str_array(): + @validate_custom_events(numpy_str_recorded_custom_events) + @validate_custom_event_count(count=4) + @background_task() + def _test(): + import sklearn.tree + + x_train = np.array([[20, 20], [21, 21]], dtype=" Date: Mon, 9 Jan 2023 12:44:50 -0800 Subject: [PATCH 20/54] Add feature selection model function traces (#719) * Add feature selection models * Modify VotingRegressor test --- newrelic/config.py | 42 ++++++ newrelic/hooks/mlmodel_sklearn.py | 17 +++ .../test_feature_selection_models.py | 138 ++++++++++++++++++ 3 files changed, 197 insertions(+) create mode 100644 tests/mlmodel_sklearn/test_feature_selection_models.py diff --git a/newrelic/config.py b/newrelic/config.py index bc8ec395d..aaf4a18aa 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2902,6 +2902,48 @@ def _process_module_builtin_defaults(): "instrument_sklearn_ensemble_hist_models", ) + _process_module_definition( + "sklearn.feature_selection._rfe", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_feature_selection_rfe_models", + ) + + _process_module_definition( + "sklearn.feature_selection.rfe", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_feature_selection_rfe_models", + ) + + _process_module_definition( + "sklearn.feature_selection._variance_threshold", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_feature_selection_models", + ) + + _process_module_definition( + "sklearn.feature_selection.variance_threshold", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_feature_selection_models", + ) + + _process_module_definition( + "sklearn.feature_selection._from_model", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_feature_selection_models", + ) + + _process_module_definition( + "sklearn.feature_selection.from_model", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_feature_selection_models", + ) + + _process_module_definition( + "sklearn.feature_selection._sequential", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_feature_selection_models", + ) + _process_module_definition( "sklearn.kernel_ridge", "newrelic.hooks.mlmodel_sklearn", diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index 7db993a9f..b745657a0 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -265,6 +265,14 @@ def instrument_sklearn_ensemble_hist_models(module): _instrument_sklearn_models(module, model_classes) +def instrument_sklearn_feature_selection_rfe_models(module): + model_classes = ( + "RFE", + "RFECV", + ) + _instrument_sklearn_models(module, model_classes) + + def instrument_sklearn_kernel_ridge_models(module): model_classes = ("KernelRidge",) _instrument_sklearn_models(module, model_classes) @@ -286,6 +294,15 @@ def instrument_sklearn_cluster_models(module): _instrument_sklearn_models(module, model_classes) +def instrument_sklearn_feature_selection_models(module): + model_classes = ( + "VarianceThreshold", + "SelectFromModel", + "SequentialFeatureSelector", + ) + _instrument_sklearn_models(module, model_classes) + + def instrument_sklearn_cluster_agglomerative_models(module): model_classes = ( "AgglomerativeClustering", diff --git a/tests/mlmodel_sklearn/test_feature_selection_models.py b/tests/mlmodel_sklearn/test_feature_selection_models.py new file mode 100644 index 000000000..f4d601d32 --- /dev/null +++ b/tests/mlmodel_sklearn/test_feature_selection_models.py @@ -0,0 +1,138 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from sklearn.ensemble import AdaBoostClassifier +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version +from newrelic.packages import six + +SKLEARN_VERSION = tuple(map(int, get_package_version("sklearn").split("."))) + + +@pytest.mark.parametrize( + "feature_selection_model_name", + [ + "VarianceThreshold", + "RFE", + "RFECV", + "SelectFromModel", + ], +) +def test_below_v1_0_model_methods_wrapped_in_function_trace(feature_selection_model_name, run_feature_selection_model): + expected_scoped_metrics = { + "VarianceThreshold": [ + ("Function/MLModel/Sklearn/Named/VarianceThreshold.fit", 1), + ], + "RFE": [ + ("Function/MLModel/Sklearn/Named/RFE.fit", 1), + ("Function/MLModel/Sklearn/Named/RFE.predict", 1), + ("Function/MLModel/Sklearn/Named/RFE.score", 1), + ("Function/MLModel/Sklearn/Named/RFE.predict_log_proba", 1), + ("Function/MLModel/Sklearn/Named/RFE.predict_proba", 1), + ], + "RFECV": [ + ("Function/MLModel/Sklearn/Named/RFECV.fit", 1), + ], + "SelectFromModel": [ + ("Function/MLModel/Sklearn/Named/SelectFromModel.fit", 1), + ], + } + + expected_transaction_name = ( + "test_feature_selection_models:test_below_v1_0_model_methods_wrapped_in_function_trace.._test" + if six.PY3 + else "test_feature_selection_models:_test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[feature_selection_model_name], + rollup_metrics=expected_scoped_metrics[feature_selection_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_feature_selection_model(feature_selection_model_name) + + _test() + + +@pytest.mark.skipif(SKLEARN_VERSION < (1, 0, 0), reason="Requires sklearn >= 1.0") +@pytest.mark.parametrize( + "feature_selection_model_name", + [ + "SequentialFeatureSelector", + ], +) +def test_above_v1_0_model_methods_wrapped_in_function_trace(feature_selection_model_name, run_feature_selection_model): + expected_scoped_metrics = { + "SequentialFeatureSelector": [ + ("Function/MLModel/Sklearn/Named/SequentialFeatureSelector.fit", 1), + ], + } + expected_transaction_name = ( + "test_feature_selection_models:test_above_v1_0_model_methods_wrapped_in_function_trace.._test" + if six.PY3 + else "test_feature_selection_models:_test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[feature_selection_model_name], + rollup_metrics=expected_scoped_metrics[feature_selection_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_feature_selection_model(feature_selection_model_name) + + _test() + + +@pytest.fixture +def run_feature_selection_model(): + def _run(feature_selection_model_name): + import sklearn.feature_selection + from sklearn.datasets import load_iris + from sklearn.model_selection import train_test_split + + X, y = load_iris(return_X_y=True) + x_train, x_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) + + kwargs = {} + if feature_selection_model_name in ["RFE", "SequentialFeatureSelector", "SelectFromModel", "RFECV"]: + # This is an example of a model that has all the available attributes + # We could have choosen any estimator that has predict, score, + # predict_log_proba, and predict_proba + kwargs = {"estimator": AdaBoostClassifier()} + clf = getattr(sklearn.feature_selection, feature_selection_model_name)(**kwargs) + + model = clf.fit(x_train, y_train) + if hasattr(model, "predict"): + model.predict(x_test) + if hasattr(model, "score"): + model.score(x_test, y_test) + if hasattr(model, "predict_log_proba"): + model.predict_log_proba(x_test) + if hasattr(model, "predict_proba"): + model.predict_proba(x_test) + + return model + + return _run From bdea25d0ad1f22edcc771f95b8ec0042b329d73b Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Mon, 9 Jan 2023 13:22:51 -0800 Subject: [PATCH 21/54] Add dummy model function traces (#718) --- newrelic/config.py | 6 ++ newrelic/hooks/mlmodel_sklearn.py | 8 ++ tests/mlmodel_sklearn/test_dummy_models.py | 94 ++++++++++++++++++++++ 3 files changed, 108 insertions(+) create mode 100644 tests/mlmodel_sklearn/test_dummy_models.py diff --git a/newrelic/config.py b/newrelic/config.py index aaf4a18aa..42b90514a 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2902,6 +2902,12 @@ def _process_module_builtin_defaults(): "instrument_sklearn_ensemble_hist_models", ) + _process_module_definition( + "sklearn.dummy", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_dummy_models", + ) + _process_module_definition( "sklearn.feature_selection._rfe", "newrelic.hooks.mlmodel_sklearn", diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index b745657a0..53216954a 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -265,6 +265,14 @@ def instrument_sklearn_ensemble_hist_models(module): _instrument_sklearn_models(module, model_classes) +def instrument_sklearn_dummy_models(module): + model_classes = ( + "DummyClassifier", + "DummyRegressor", + ) + _instrument_sklearn_models(module, model_classes) + + def instrument_sklearn_feature_selection_rfe_models(module): model_classes = ( "RFE", diff --git a/tests/mlmodel_sklearn/test_dummy_models.py b/tests/mlmodel_sklearn/test_dummy_models.py new file mode 100644 index 000000000..d1059add1 --- /dev/null +++ b/tests/mlmodel_sklearn/test_dummy_models.py @@ -0,0 +1,94 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from sklearn import __init__ # noqa: needed for get_package_version +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version +from newrelic.packages import six + +SKLEARN_VERSION = tuple(map(int, get_package_version("sklearn").split("."))) + + +@pytest.mark.parametrize( + "dummy_model_name", + [ + "DummyClassifier", + "DummyRegressor", + ], +) +def test_model_methods_wrapped_in_function_trace(dummy_model_name, run_dummy_model): + expected_scoped_metrics = { + "DummyClassifier": [ + ("Function/MLModel/Sklearn/Named/DummyClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/DummyClassifier.predict", 2), + ("Function/MLModel/Sklearn/Named/DummyClassifier.predict_log_proba", 1), + ("Function/MLModel/Sklearn/Named/DummyClassifier.predict_proba", 2 if SKLEARN_VERSION > (1, 0, 0) else 4), + ("Function/MLModel/Sklearn/Named/DummyClassifier.score", 1), + ], + "DummyRegressor": [ + ("Function/MLModel/Sklearn/Named/DummyRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/DummyRegressor.predict", 2), + ("Function/MLModel/Sklearn/Named/DummyRegressor.score", 1), + ], + } + + expected_transaction_name = ( + "test_dummy_models:test_model_methods_wrapped_in_function_trace.._test" + if six.PY3 + else "test_dummy_models:_test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[dummy_model_name], + rollup_metrics=expected_scoped_metrics[dummy_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_dummy_model(dummy_model_name) + + _test() + + +@pytest.fixture +def run_dummy_model(): + def _run(dummy_model_name): + import sklearn.dummy + from sklearn.datasets import load_iris + from sklearn.model_selection import train_test_split + + X, y = load_iris(return_X_y=True) + x_train, x_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) + + clf = getattr(sklearn.dummy, dummy_model_name)() + + model = clf.fit(x_train, y_train) + if hasattr(model, "predict"): + model.predict(x_test) + if hasattr(model, "score"): + model.score(x_test, y_test) + if hasattr(model, "predict_log_proba"): + model.predict_log_proba(x_test) + if hasattr(model, "predict_proba"): + model.predict_proba(x_test) + + return model + + return _run From 3ad3096bb8a61ba039a3fadb2d7d16f96ed7e702 Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Mon, 9 Jan 2023 13:58:56 -0800 Subject: [PATCH 22/54] Add gaussian process model function traces (#720) * Add gaussian process models * Modify VotingRegressor test * Modify Gaussian Process models tests --- newrelic/config.py | 24 ++++++ newrelic/hooks/mlmodel_sklearn.py | 8 ++ .../test_gaussian_process_models.py | 83 +++++++++++++++++++ 3 files changed, 115 insertions(+) create mode 100644 tests/mlmodel_sklearn/test_gaussian_process_models.py diff --git a/newrelic/config.py b/newrelic/config.py index 42b90514a..f991359fb 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2902,6 +2902,30 @@ def _process_module_builtin_defaults(): "instrument_sklearn_ensemble_hist_models", ) + _process_module_definition( + "sklearn.gaussian_process._gpc", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_gaussian_process_models", + ) + + _process_module_definition( + "sklearn.gaussian_process.gpc", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_gaussian_process_models", + ) + + _process_module_definition( + "sklearn.gaussian_process._gpr", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_gaussian_process_models", + ) + + _process_module_definition( + "sklearn.gaussian_process.gpr", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_gaussian_process_models", + ) + _process_module_definition( "sklearn.dummy", "newrelic.hooks.mlmodel_sklearn", diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index 53216954a..72db34af5 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -265,6 +265,14 @@ def instrument_sklearn_ensemble_hist_models(module): _instrument_sklearn_models(module, model_classes) +def instrument_sklearn_gaussian_process_models(module): + model_classes = ( + "GaussianProcessClassifier", + "GaussianProcessRegressor", + ) + _instrument_sklearn_models(module, model_classes) + + def instrument_sklearn_dummy_models(module): model_classes = ( "DummyClassifier", diff --git a/tests/mlmodel_sklearn/test_gaussian_process_models.py b/tests/mlmodel_sklearn/test_gaussian_process_models.py new file mode 100644 index 000000000..7a78fc703 --- /dev/null +++ b/tests/mlmodel_sklearn/test_gaussian_process_models.py @@ -0,0 +1,83 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.packages import six + + +@pytest.mark.parametrize( + "gaussian_process_model_name", + [ + "GaussianProcessClassifier", + "GaussianProcessRegressor", + ], +) +def test_model_methods_wrapped_in_function_trace(gaussian_process_model_name, run_gaussian_process_model): + expected_scoped_metrics = { + "GaussianProcessClassifier": [ + ("Function/MLModel/Sklearn/Named/GaussianProcessClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/GaussianProcessClassifier.predict", 1), + ("Function/MLModel/Sklearn/Named/GaussianProcessClassifier.predict_proba", 1), + ], + "GaussianProcessRegressor": [ + ("Function/MLModel/Sklearn/Named/GaussianProcessRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/GaussianProcessRegressor.predict", 1), + ], + } + + expected_transaction_name = ( + "test_gaussian_process_models:test_model_methods_wrapped_in_function_trace.._test" + if six.PY3 + else "test_gaussian_process_models:_test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[gaussian_process_model_name], + rollup_metrics=expected_scoped_metrics[gaussian_process_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_gaussian_process_model(gaussian_process_model_name) + + _test() + + +@pytest.fixture +def run_gaussian_process_model(): + def _run(gaussian_process_model_name): + import sklearn.gaussian_process + from sklearn.datasets import load_iris + from sklearn.model_selection import train_test_split + + X, y = load_iris(return_X_y=True) + x_train, x_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) + + clf = getattr(sklearn.gaussian_process, gaussian_process_model_name)(random_state=0) + + model = clf.fit(x_train, y_train) + if hasattr(model, "predict"): + model.predict(x_test) + if hasattr(model, "predict_proba"): + model.predict_proba(x_test) + + return model + + return _run From 22b9588449488e3c03d938d850798f160c471dfe Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Mon, 9 Jan 2023 15:12:32 -0800 Subject: [PATCH 23/54] Add discriminant analysis model (#717) --- newrelic/config.py | 6 ++ newrelic/hooks/mlmodel_sklearn.py | 8 ++ .../test_discriminant_analysis_models.py | 91 +++++++++++++++++++ 3 files changed, 105 insertions(+) create mode 100644 tests/mlmodel_sklearn/test_discriminant_analysis_models.py diff --git a/newrelic/config.py b/newrelic/config.py index f991359fb..20d73f5d3 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2902,6 +2902,12 @@ def _process_module_builtin_defaults(): "instrument_sklearn_ensemble_hist_models", ) + _process_module_definition( + "sklearn.discriminant_analysis", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_discriminant_analysis_models", + ) + _process_module_definition( "sklearn.gaussian_process._gpc", "newrelic.hooks.mlmodel_sklearn", diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index 72db34af5..894304dd0 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -265,6 +265,14 @@ def instrument_sklearn_ensemble_hist_models(module): _instrument_sklearn_models(module, model_classes) +def instrument_sklearn_discriminant_analysis_models(module): + model_classes = ( + "LinearDiscriminantAnalysis", + "QuadraticDiscriminantAnalysis", + ) + _instrument_sklearn_models(module, model_classes) + + def instrument_sklearn_gaussian_process_models(module): model_classes = ( "GaussianProcessClassifier", diff --git a/tests/mlmodel_sklearn/test_discriminant_analysis_models.py b/tests/mlmodel_sklearn/test_discriminant_analysis_models.py new file mode 100644 index 000000000..de1182696 --- /dev/null +++ b/tests/mlmodel_sklearn/test_discriminant_analysis_models.py @@ -0,0 +1,91 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.packages import six + + +@pytest.mark.parametrize( + "discriminant_analysis_model_name", + [ + "LinearDiscriminantAnalysis", + "QuadraticDiscriminantAnalysis", + ], +) +def test_model_methods_wrapped_in_function_trace(discriminant_analysis_model_name, run_discriminant_analysis_model): + expected_scoped_metrics = { + "LinearDiscriminantAnalysis": [ + ("Function/MLModel/Sklearn/Named/LinearDiscriminantAnalysis.fit", 1), + ("Function/MLModel/Sklearn/Named/LinearDiscriminantAnalysis.predict_log_proba", 1), + ("Function/MLModel/Sklearn/Named/LinearDiscriminantAnalysis.predict_proba", 2), + ("Function/MLModel/Sklearn/Named/LinearDiscriminantAnalysis.transform", 1), + ], + "QuadraticDiscriminantAnalysis": [ + ("Function/MLModel/Sklearn/Named/QuadraticDiscriminantAnalysis.fit", 1), + ("Function/MLModel/Sklearn/Named/QuadraticDiscriminantAnalysis.predict", 1), + ("Function/MLModel/Sklearn/Named/QuadraticDiscriminantAnalysis.predict_proba", 2), + ("Function/MLModel/Sklearn/Named/QuadraticDiscriminantAnalysis.predict_log_proba", 1), + ], + } + + expected_transaction_name = ( + "test_discriminant_analysis_models:test_model_methods_wrapped_in_function_trace.._test" + if six.PY3 + else "test_discriminant_analysis_models:_test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[discriminant_analysis_model_name], + rollup_metrics=expected_scoped_metrics[discriminant_analysis_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_discriminant_analysis_model(discriminant_analysis_model_name) + + _test() + + +@pytest.fixture +def run_discriminant_analysis_model(): + def _run(discriminant_analysis_model_name): + import sklearn.discriminant_analysis + from sklearn.datasets import load_iris + from sklearn.model_selection import train_test_split + + X, y = load_iris(return_X_y=True) + x_train, x_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) + + kwargs = {} + clf = getattr(sklearn.discriminant_analysis, discriminant_analysis_model_name)(**kwargs) + + model = clf.fit(x_train, y_train) + if hasattr(model, "predict"): + model.predict(x_test) + if hasattr(model, "predict_log_proba"): + model.predict_log_proba(x_test) + if hasattr(model, "predict_proba"): + model.predict_proba(x_test) + if hasattr(model, "transform"): + model.transform(x_test) + + return model + + return _run From 600a0f14fab267fcd0cdd028cc59f2b1c9a87671 Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Mon, 9 Jan 2023 16:26:07 -0800 Subject: [PATCH 24/54] Add cross decomposition models (#716) --- newrelic/config.py | 12 +++ newrelic/hooks/mlmodel_sklearn.py | 8 ++ .../test_cross_decomposition_models.py | 81 +++++++++++++++++++ 3 files changed, 101 insertions(+) create mode 100644 tests/mlmodel_sklearn/test_cross_decomposition_models.py diff --git a/newrelic/config.py b/newrelic/config.py index 20d73f5d3..5f103dfbd 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2902,6 +2902,18 @@ def _process_module_builtin_defaults(): "instrument_sklearn_ensemble_hist_models", ) + _process_module_definition( + "sklearn.cross_decomposition._pls", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cross_decomposition_models", + ) + + _process_module_definition( + "sklearn.cross_decomposition.pls_", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cross_decomposition_models", + ) + _process_module_definition( "sklearn.discriminant_analysis", "newrelic.hooks.mlmodel_sklearn", diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index 894304dd0..56dc42117 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -265,6 +265,14 @@ def instrument_sklearn_ensemble_hist_models(module): _instrument_sklearn_models(module, model_classes) +def instrument_sklearn_cross_decomposition_models(module): + model_classes = ( + "PLSRegression", + "PLSSVD", + ) + _instrument_sklearn_models(module, model_classes) + + def instrument_sklearn_discriminant_analysis_models(module): model_classes = ( "LinearDiscriminantAnalysis", diff --git a/tests/mlmodel_sklearn/test_cross_decomposition_models.py b/tests/mlmodel_sklearn/test_cross_decomposition_models.py new file mode 100644 index 000000000..6a053350f --- /dev/null +++ b/tests/mlmodel_sklearn/test_cross_decomposition_models.py @@ -0,0 +1,81 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.packages import six + + +@pytest.mark.parametrize( + "cross_decomposition_model_name", + [ + "PLSRegression", + "PLSSVD", + ], +) +def test_model_methods_wrapped_in_function_trace(cross_decomposition_model_name, run_cross_decomposition_model): + expected_scoped_metrics = { + "PLSRegression": [ + ("Function/MLModel/Sklearn/Named/PLSRegression.fit", 1), + ], + "PLSSVD": [ + ("Function/MLModel/Sklearn/Named/PLSSVD.fit", 1), + ("Function/MLModel/Sklearn/Named/PLSSVD.transform", 1), + ], + } + expected_transaction_name = ( + "test_cross_decomposition_models:test_model_methods_wrapped_in_function_trace.._test" + if six.PY3 + else "test_cross_decomposition_models:_test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[cross_decomposition_model_name], + rollup_metrics=expected_scoped_metrics[cross_decomposition_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_cross_decomposition_model(cross_decomposition_model_name) + + _test() + + +@pytest.fixture +def run_cross_decomposition_model(): + def _run(cross_decomposition_model_name): + import sklearn.cross_decomposition + from sklearn.datasets import load_iris + from sklearn.model_selection import train_test_split + + X, y = load_iris(return_X_y=True) + x_train, x_test, y_train, _ = train_test_split(X, y, stratify=y, random_state=0) + + kwargs = {} + if cross_decomposition_model_name == "PLSSVD": + kwargs = {"n_components": 1} + clf = getattr(sklearn.cross_decomposition, cross_decomposition_model_name)(**kwargs) + + model = clf.fit(x_train, y_train) + if hasattr(model, "transform"): + model.transform(x_test) + + return model + + return _run From 178a00017fdfd49877bddb11bf2af21b90d62ebe Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Mon, 9 Jan 2023 17:16:28 -0800 Subject: [PATCH 25/54] Add covariance model (#714) --- newrelic/config.py | 60 ++++++++++ newrelic/hooks/mlmodel_sklearn.py | 26 +++++ .../mlmodel_sklearn/test_covariance_models.py | 110 ++++++++++++++++++ 3 files changed, 196 insertions(+) create mode 100644 tests/mlmodel_sklearn/test_covariance_models.py diff --git a/newrelic/config.py b/newrelic/config.py index 5f103dfbd..540441283 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2818,6 +2818,66 @@ def _process_module_builtin_defaults(): "instrument_sklearn_metrics", ) + _process_module_definition( + "sklearn.covariance._empirical_covariance", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_covariance_models", + ) + + _process_module_definition( + "sklearn.covariance.empirical_covariance_", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_covariance_models", + ) + + _process_module_definition( + "sklearn.covariance.shrunk_covariance_", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_covariance_shrunk_models", + ) + + _process_module_definition( + "sklearn.covariance._shrunk_covariance", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_covariance_shrunk_models", + ) + + _process_module_definition( + "sklearn.covariance.robust_covariance_", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_covariance_models", + ) + + _process_module_definition( + "sklearn.covariance._robust_covariance", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_covariance_models", + ) + + _process_module_definition( + "sklearn.covariance.graph_lasso_", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_covariance_graph_models", + ) + + _process_module_definition( + "sklearn.covariance._graph_lasso", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_covariance_graph_models", + ) + + _process_module_definition( + "sklearn.covariance.elliptic_envelope", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_covariance_models", + ) + + _process_module_definition( + "sklearn.covariance._elliptic_envelope", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_covariance_models", + ) + _process_module_definition( "sklearn.ensemble._bagging", "newrelic.hooks.mlmodel_sklearn", diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index 56dc42117..bf01c63be 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -265,6 +265,15 @@ def instrument_sklearn_ensemble_hist_models(module): _instrument_sklearn_models(module, model_classes) +def instrument_sklearn_covariance_shrunk_models(module): + model_classes = ( + "ShrunkCovariance", + "LedoitWolf", + "OAS", + ) + _instrument_sklearn_models(module, model_classes) + + def instrument_sklearn_cross_decomposition_models(module): model_classes = ( "PLSRegression", @@ -273,6 +282,14 @@ def instrument_sklearn_cross_decomposition_models(module): _instrument_sklearn_models(module, model_classes) +def instrument_sklearn_covariance_graph_models(module): + model_classes = ( + "GraphicalLasso", + "GraphicalLassoCV", + ) + _instrument_sklearn_models(module, model_classes) + + def instrument_sklearn_discriminant_analysis_models(module): model_classes = ( "LinearDiscriminantAnalysis", @@ -281,6 +298,15 @@ def instrument_sklearn_discriminant_analysis_models(module): _instrument_sklearn_models(module, model_classes) +def instrument_sklearn_covariance_models(module): + model_classes = ( + "EmpiricalCovariance", + "MinCovDet", + "EllipticEnvelope", + ) + _instrument_sklearn_models(module, model_classes) + + def instrument_sklearn_gaussian_process_models(module): model_classes = ( "GaussianProcessClassifier", diff --git a/tests/mlmodel_sklearn/test_covariance_models.py b/tests/mlmodel_sklearn/test_covariance_models.py new file mode 100644 index 000000000..afa5c31c2 --- /dev/null +++ b/tests/mlmodel_sklearn/test_covariance_models.py @@ -0,0 +1,110 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.packages import six + + +@pytest.mark.parametrize( + "covariance_model_name", + [ + "EllipticEnvelope", + "EmpiricalCovariance", + "GraphicalLasso", + "GraphicalLassoCV", + "MinCovDet", + "ShrunkCovariance", + "LedoitWolf", + "OAS", + ], +) +def test_model_methods_wrapped_in_function_trace(covariance_model_name, run_covariance_model): + expected_scoped_metrics = { + "EllipticEnvelope": [ + ("Function/MLModel/Sklearn/Named/EllipticEnvelope.fit", 1), + ("Function/MLModel/Sklearn/Named/EllipticEnvelope.predict", 2), + ("Function/MLModel/Sklearn/Named/EllipticEnvelope.score", 1), + ], + "EmpiricalCovariance": [ + ("Function/MLModel/Sklearn/Named/EmpiricalCovariance.fit", 1), + ("Function/MLModel/Sklearn/Named/EmpiricalCovariance.score", 1), + ], + "GraphicalLasso": [ + ("Function/MLModel/Sklearn/Named/GraphicalLasso.fit", 1), + ], + "GraphicalLassoCV": [ + ("Function/MLModel/Sklearn/Named/GraphicalLassoCV.fit", 1), + ], + "MinCovDet": [ + ("Function/MLModel/Sklearn/Named/MinCovDet.fit", 1), + ], + "ShrunkCovariance": [ + ("Function/MLModel/Sklearn/Named/ShrunkCovariance.fit", 1), + ], + "LedoitWolf": [ + ("Function/MLModel/Sklearn/Named/LedoitWolf.fit", 1), + ], + "OAS": [ + ("Function/MLModel/Sklearn/Named/OAS.fit", 1), + ], + } + expected_transaction_name = ( + "test_covariance_models:test_model_methods_wrapped_in_function_trace.._test" + if six.PY3 + else "test_covariance_models:_test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[covariance_model_name], + rollup_metrics=expected_scoped_metrics[covariance_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_covariance_model(covariance_model_name) + + _test() + + +@pytest.fixture +def run_covariance_model(): + def _run(covariance_model_name): + import sklearn.covariance + from sklearn.datasets import load_iris + from sklearn.model_selection import train_test_split + + X, y = load_iris(return_X_y=True) + x_train, x_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) + + kwargs = {} + if covariance_model_name in ["EllipticEnvelope", "MinCovDet"]: + kwargs = {"random_state": 0} + + clf = getattr(sklearn.covariance, covariance_model_name)(**kwargs) + + model = clf.fit(x_train, y_train) + if hasattr(model, "predict"): + model.predict(x_test) + if hasattr(model, "score"): + model.score(x_test, y_test) + + return model + + return _run From e9ce7f7aa4bf73e0240cbb82e460f733b3699da9 Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Wed, 11 Jan 2023 15:33:20 -0800 Subject: [PATCH 26/54] Add compose models (#713) --- newrelic/config.py | 12 +++ newrelic/hooks/mlmodel_sklearn.py | 8 ++ tests/mlmodel_sklearn/test_compose_models.py | 94 ++++++++++++++++++++ 3 files changed, 114 insertions(+) create mode 100644 tests/mlmodel_sklearn/test_compose_models.py diff --git a/newrelic/config.py b/newrelic/config.py index 540441283..5a30f9700 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2818,6 +2818,18 @@ def _process_module_builtin_defaults(): "instrument_sklearn_metrics", ) + _process_module_definition( + "sklearn.compose._column_transformer", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_compose_models", + ) + + _process_module_definition( + "sklearn.compose._target", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_compose_models", + ) + _process_module_definition( "sklearn.covariance._empirical_covariance", "newrelic.hooks.mlmodel_sklearn", diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index bf01c63be..07f1d78c2 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -265,6 +265,14 @@ def instrument_sklearn_ensemble_hist_models(module): _instrument_sklearn_models(module, model_classes) +def instrument_sklearn_compose_models(module): + model_classes = ( + "ColumnTransformer", + "TransformedTargetRegressor", + ) + _instrument_sklearn_models(module, model_classes) + + def instrument_sklearn_covariance_shrunk_models(module): model_classes = ( "ShrunkCovariance", diff --git a/tests/mlmodel_sklearn/test_compose_models.py b/tests/mlmodel_sklearn/test_compose_models.py new file mode 100644 index 000000000..eab076fc3 --- /dev/null +++ b/tests/mlmodel_sklearn/test_compose_models.py @@ -0,0 +1,94 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from sklearn.linear_model import LinearRegression +from sklearn.preprocessing import Normalizer +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.packages import six + + +@pytest.mark.parametrize( + "compose_model_name", + [ + "ColumnTransformer", + "TransformedTargetRegressor", + ], +) +def test_model_methods_wrapped_in_function_trace(compose_model_name, run_compose_model): + expected_scoped_metrics = { + "ColumnTransformer": [ + ("Function/MLModel/Sklearn/Named/ColumnTransformer.fit", 1), + ("Function/MLModel/Sklearn/Named/ColumnTransformer.transform", 1), + ], + "TransformedTargetRegressor": [ + ("Function/MLModel/Sklearn/Named/TransformedTargetRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/TransformedTargetRegressor.predict", 1), + ], + } + + expected_transaction_name = ( + "test_compose_models:test_model_methods_wrapped_in_function_trace.._test" + if six.PY3 + else "test_compose_models:_test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[compose_model_name], + rollup_metrics=expected_scoped_metrics[compose_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_compose_model(compose_model_name) + + _test() + + +@pytest.fixture +def run_compose_model(): + def _run(compose_model_name): + import numpy as np + import sklearn.compose + + if compose_model_name == "TransformedTargetRegressor": + kwargs = {"regressor": LinearRegression()} + X = np.arange(4).reshape(-1, 1) + y = np.exp(2 * X).ravel() + else: + X = [[0.0, 1.0, 2.0, 2.0], [1.0, 1.0, 0.0, 1.0]] + y = None + kwargs = { + "transformers": [ + ("norm1", Normalizer(norm="l1"), [0, 1]), + ("norm2", Normalizer(norm="l1"), slice(2, 4)), + ] + } + + clf = getattr(sklearn.compose, compose_model_name)(**kwargs) + + model = clf.fit(X, y) + if hasattr(model, "predict"): + model.predict(X) + if hasattr(model, "transform"): + model.transform(X) + + return model + + return _run From db76aca9459fce9e5678f7d61b900b4ce2b7eb44 Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Wed, 11 Jan 2023 16:07:45 -0800 Subject: [PATCH 27/54] Add linear model function traces (#703) * Add sklearn to tox * Add function traces around model methods * Support Python 2.7 & 3.7 sklearn * Add test for multiple calls to model method * Fixup: add comments & organize * Add ensemble models * Add ensemble model tests * Edit tests * Add ensemble library models from sklearn * Start tests with empty commit * Clean up tests * Initial linear model commit * Clean up tests for linear models * Fix tests for various versions of sklearn * Fix ensemble tests with changes from tree PR * [Mega-Linter] Apply linters fixes * Remove breakpoints * Merge changes from ensemble PR * Fix tests for v0.20.0 * Rewrite linear tests to be more readable * Break up instrumentation in config * Remove commented code * Remove yaml file in commit * Remove duplicate ensemble module defs * Remove old no longer used file * Remove commented out code. * Change test name and modify VotingRegressor test * Push empty commit * Modify VotingRegression test * Add estimator to VotingRegressor * Revert VotingRegressor test * Fix ensemble tests * Add different models for VotingRegressor test Co-authored-by: Hannah Stepanek Co-authored-by: lrafeei --- newrelic/config.py | 168 +++++++++ newrelic/hooks/mlmodel_sklearn.py | 97 +++++ tests/mlmodel_sklearn/test_ensemble_models.py | 5 +- tests/mlmodel_sklearn/test_linear_models.py | 335 ++++++++++++++++++ 4 files changed, 603 insertions(+), 2 deletions(-) create mode 100644 tests/mlmodel_sklearn/test_linear_models.py diff --git a/newrelic/config.py b/newrelic/config.py index 5a30f9700..a8ab01d3e 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2974,6 +2974,174 @@ def _process_module_builtin_defaults(): "instrument_sklearn_ensemble_hist_models", ) + _process_module_definition( + "sklearn.linear_model._base", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_models", + ) + + _process_module_definition( + "sklearn.linear_model.base", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_models", + ) + + _process_module_definition( + "sklearn.linear_model._bayes", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_bayes_models", + ) + + _process_module_definition( + "sklearn.linear_model.bayes", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_bayes_models", + ) + + _process_module_definition( + "sklearn.linear_model._least_angle", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_least_angle_models", + ) + + _process_module_definition( + "sklearn.linear_model.least_angle", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_least_angle_models", + ) + + _process_module_definition( + "sklearn.linear_model.coordinate_descent", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_coordinate_descent_models", + ) + + _process_module_definition( + "sklearn.linear_model._coordinate_descent", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_coordinate_descent_models", + ) + + _process_module_definition( + "sklearn.linear_model._glm", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_GLM_models", + ) + + _process_module_definition( + "sklearn.linear_model._huber", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_models", + ) + + _process_module_definition( + "sklearn.linear_model.huber", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_models", + ) + + _process_module_definition( + "sklearn.linear_model._stochastic_gradient", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_stochastic_gradient_models", + ) + + _process_module_definition( + "sklearn.linear_model.stochastic_gradient", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_stochastic_gradient_models", + ) + + _process_module_definition( + "sklearn.linear_model._ridge", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_ridge_models", + ) + + _process_module_definition( + "sklearn.linear_model.ridge", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_ridge_models", + ) + + _process_module_definition( + "sklearn.linear_model._logistic", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_logistic_models", + ) + + _process_module_definition( + "sklearn.linear_model.logistic", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_logistic_models", + ) + + _process_module_definition( + "sklearn.linear_model._omp", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_OMP_models", + ) + + _process_module_definition( + "sklearn.linear_model.omp", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_OMP_models", + ) + + _process_module_definition( + "sklearn.linear_model._passive_aggressive", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_passive_aggressive_models", + ) + + _process_module_definition( + "sklearn.linear_model.passive_aggressive", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_passive_aggressive_models", + ) + + _process_module_definition( + "sklearn.linear_model._perceptron", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_models", + ) + + _process_module_definition( + "sklearn.linear_model.perceptron", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_models", + ) + + _process_module_definition( + "sklearn.linear_model._quantile", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_models", + ) + + _process_module_definition( + "sklearn.linear_model._ransac", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_models", + ) + + _process_module_definition( + "sklearn.linear_model.ransac", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_models", + ) + + _process_module_definition( + "sklearn.linear_model._theil_sen", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_models", + ) + + _process_module_definition( + "sklearn.linear_model.theil_sen", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_models", + ) + _process_module_definition( "sklearn.cross_decomposition._pls", "newrelic.hooks.mlmodel_sklearn", diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index 07f1d78c2..7c5f78aaa 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -265,6 +265,20 @@ def instrument_sklearn_ensemble_hist_models(module): _instrument_sklearn_models(module, model_classes) +def instrument_sklearn_linear_coordinate_descent_models(module): + model_classes = ( + "Lasso", + "LassoCV", + "ElasticNet", + "ElasticNetCV", + "MultiTaskLasso", + "MultiTaskLassoCV", + "MultiTaskElasticNet", + "MultiTaskElasticNetCV", + ) + _instrument_sklearn_models(module, model_classes) + + def instrument_sklearn_compose_models(module): model_classes = ( "ColumnTransformer", @@ -360,6 +374,17 @@ def instrument_sklearn_cluster_models(module): _instrument_sklearn_models(module, model_classes) +def instrument_sklearn_linear_least_angle_models(module): + model_classes = ( + "Lars", + "LarsCV", + "LassoLars", + "LassoLarsCV", + "LassoLarsIC", + ) + _instrument_sklearn_models(module, model_classes) + + def instrument_sklearn_feature_selection_models(module): model_classes = ( "VarianceThreshold", @@ -377,6 +402,15 @@ def instrument_sklearn_cluster_agglomerative_models(module): _instrument_sklearn_models(module, model_classes) +def instrument_sklearn_linear_GLM_models(module): + model_classes = ( + "PoissonRegressor", + "GammaRegressor", + "TweedieRegressor", + ) + _instrument_sklearn_models(module, model_classes) + + def instrument_sklearn_cluster_clustering_models(module): model_classes = ( "SpectralBiclustering", @@ -386,6 +420,69 @@ def instrument_sklearn_cluster_clustering_models(module): _instrument_sklearn_models(module, model_classes) +def instrument_sklearn_linear_stochastic_gradient_models(module): + model_classes = ( + "SGDClassifier", + "SGDRegressor", + "SGDOneClassSVM", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_linear_ridge_models(module): + model_classes = ( + "Ridge", + "RidgeCV", + "RidgeClassifier", + "RidgeClassifierCV", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_linear_logistic_models(module): + model_classes = ( + "LogisticRegression", + "LogisticRegressionCV", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_linear_OMP_models(module): + model_classes = ( + "OrthogonalMatchingPursuit", + "OrthogonalMatchingPursuitCV", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_linear_passive_aggressive_models(module): + model_classes = ( + "PassiveAggressiveClassifier", + "PassiveAggressiveRegressor", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_linear_bayes_models(module): + model_classes = ( + "ARDRegression", + "BayesianRidge", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_linear_models(module): + model_classes = ( + "HuberRegressor", + "LinearRegression", + "Perceptron", + "QuantileRegressor", + "TheilSenRegressor", + "RANSACRegressor", + ) + _instrument_sklearn_models(module, model_classes) + + def instrument_sklearn_cluster_kmeans_models(module): model_classes = ( "BisectingKMeans", diff --git a/tests/mlmodel_sklearn/test_ensemble_models.py b/tests/mlmodel_sklearn/test_ensemble_models.py index 9daabdb5c..bcb36cce1 100644 --- a/tests/mlmodel_sklearn/test_ensemble_models.py +++ b/tests/mlmodel_sklearn/test_ensemble_models.py @@ -14,7 +14,6 @@ import pytest from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor -from sklearn.linear_model import LinearRegression from testing_support.validators.validate_transaction_metrics import ( validate_transaction_metrics, ) @@ -279,7 +278,9 @@ def _run(ensemble_model_name): "voting": "soft", } elif ensemble_model_name == "VotingRegressor": - kwargs = {"estimators": [("lr", LinearRegression())]} + x_train = x_test = [[1, 1]] + y_train = y_test = [0] + kwargs = {"estimators": [("rf", RandomForestRegressor())]} elif ensemble_model_name == "StackingRegressor": kwargs = {"estimators": [("rf", RandomForestRegressor())]} clf = getattr(sklearn.ensemble, ensemble_model_name)(**kwargs) diff --git a/tests/mlmodel_sklearn/test_linear_models.py b/tests/mlmodel_sklearn/test_linear_models.py new file mode 100644 index 000000000..582a4750e --- /dev/null +++ b/tests/mlmodel_sklearn/test_linear_models.py @@ -0,0 +1,335 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version +from newrelic.packages import six + +SKLEARN_VERSION = tuple(map(int, get_package_version("sklearn").split("."))) + + +@pytest.mark.parametrize( + "linear_model_name", + [ + "ARDRegression", + "BayesianRidge", + "ElasticNet", + "ElasticNetCV", + "HuberRegressor", + "Lars", + "LarsCV", + "Lasso", + "LassoCV", + "LassoLars", + "LassoLarsCV", + "LassoLarsIC", + "LinearRegression", + "LogisticRegression", + "LogisticRegressionCV", + "MultiTaskElasticNet", + "MultiTaskElasticNetCV", + "MultiTaskLasso", + "MultiTaskLassoCV", + "OrthogonalMatchingPursuit", + "OrthogonalMatchingPursuitCV", + "PassiveAggressiveClassifier", + "PassiveAggressiveRegressor", + "Perceptron", + "Ridge", + "RidgeCV", + "RidgeClassifier", + "RidgeClassifierCV", + "TheilSenRegressor", + "RANSACRegressor", + ], +) +def test_model_methods_wrapped_in_function_trace(linear_model_name, run_linear_model): + expected_scoped_metrics = { + "ARDRegression": [ + ("Function/MLModel/Sklearn/Named/ARDRegression.fit", 1), + ("Function/MLModel/Sklearn/Named/ARDRegression.predict", 2), + ("Function/MLModel/Sklearn/Named/ARDRegression.score", 1), + ], + "BayesianRidge": [ + ("Function/MLModel/Sklearn/Named/BayesianRidge.fit", 1), + ("Function/MLModel/Sklearn/Named/BayesianRidge.predict", 2), + ("Function/MLModel/Sklearn/Named/BayesianRidge.score", 1), + ], + "ElasticNet": [ + ("Function/MLModel/Sklearn/Named/ElasticNet.fit", 1), + ("Function/MLModel/Sklearn/Named/ElasticNet.predict", 2), + ("Function/MLModel/Sklearn/Named/ElasticNet.score", 1), + ], + "ElasticNetCV": [ + ("Function/MLModel/Sklearn/Named/ElasticNetCV.fit", 1), + ("Function/MLModel/Sklearn/Named/ElasticNetCV.predict", 2), + ("Function/MLModel/Sklearn/Named/ElasticNetCV.score", 1), + ], + "HuberRegressor": [ + ("Function/MLModel/Sklearn/Named/HuberRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/HuberRegressor.predict", 2), + ("Function/MLModel/Sklearn/Named/HuberRegressor.score", 1), + ], + "Lars": [ + ("Function/MLModel/Sklearn/Named/Lars.fit", 1), + ("Function/MLModel/Sklearn/Named/Lars.predict", 2), + ("Function/MLModel/Sklearn/Named/Lars.score", 1), + ], + "LarsCV": [ + ("Function/MLModel/Sklearn/Named/LarsCV.fit", 1), + ("Function/MLModel/Sklearn/Named/LarsCV.predict", 2), + ("Function/MLModel/Sklearn/Named/LarsCV.score", 1), + ], + "Lasso": [ + ("Function/MLModel/Sklearn/Named/Lasso.fit", 1), + ("Function/MLModel/Sklearn/Named/Lasso.predict", 2), + ("Function/MLModel/Sklearn/Named/Lasso.score", 1), + ], + "LassoCV": [ + ("Function/MLModel/Sklearn/Named/LassoCV.fit", 1), + ("Function/MLModel/Sklearn/Named/LassoCV.predict", 2), + ("Function/MLModel/Sklearn/Named/LassoCV.score", 1), + ], + "LassoLars": [ + ("Function/MLModel/Sklearn/Named/LassoLars.fit", 1), + ("Function/MLModel/Sklearn/Named/LassoLars.predict", 2), + ("Function/MLModel/Sklearn/Named/LassoLars.score", 1), + ], + "LassoLarsCV": [ + ("Function/MLModel/Sklearn/Named/LassoLarsCV.fit", 1), + ("Function/MLModel/Sklearn/Named/LassoLarsCV.predict", 2), + ("Function/MLModel/Sklearn/Named/LassoLarsCV.score", 1), + ], + "LassoLarsIC": [ + ("Function/MLModel/Sklearn/Named/LassoLarsIC.fit", 1), + ("Function/MLModel/Sklearn/Named/LassoLarsIC.predict", 2), + ("Function/MLModel/Sklearn/Named/LassoLarsIC.score", 1), + ], + "LinearRegression": [ + ("Function/MLModel/Sklearn/Named/LinearRegression.fit", 1), + ("Function/MLModel/Sklearn/Named/LinearRegression.predict", 2), + ("Function/MLModel/Sklearn/Named/LinearRegression.score", 1), + ], + "LogisticRegression": [ + ("Function/MLModel/Sklearn/Named/LogisticRegression.fit", 1), + ("Function/MLModel/Sklearn/Named/LogisticRegression.predict", 2), + ("Function/MLModel/Sklearn/Named/LogisticRegression.score", 1), + ], + "LogisticRegressionCV": [ + ("Function/MLModel/Sklearn/Named/LogisticRegressionCV.fit", 1), + ("Function/MLModel/Sklearn/Named/LogisticRegressionCV.predict", 2), + ("Function/MLModel/Sklearn/Named/LogisticRegressionCV.score", 1), + ], + "MultiTaskElasticNet": [ + ("Function/MLModel/Sklearn/Named/MultiTaskElasticNet.fit", 1), + ("Function/MLModel/Sklearn/Named/MultiTaskElasticNet.predict", 2), + ("Function/MLModel/Sklearn/Named/MultiTaskElasticNet.score", 1), + ], + "MultiTaskElasticNetCV": [ + ("Function/MLModel/Sklearn/Named/MultiTaskElasticNetCV.fit", 1), + ("Function/MLModel/Sklearn/Named/MultiTaskElasticNetCV.predict", 2), + ("Function/MLModel/Sklearn/Named/MultiTaskElasticNetCV.score", 1), + ], + "MultiTaskLasso": [ + ("Function/MLModel/Sklearn/Named/MultiTaskLasso.fit", 1), + ("Function/MLModel/Sklearn/Named/MultiTaskLasso.predict", 2), + ("Function/MLModel/Sklearn/Named/MultiTaskLasso.score", 1), + ], + "MultiTaskLassoCV": [ + ("Function/MLModel/Sklearn/Named/MultiTaskLassoCV.fit", 1), + ("Function/MLModel/Sklearn/Named/MultiTaskLassoCV.predict", 2), + ("Function/MLModel/Sklearn/Named/MultiTaskLassoCV.score", 1), + ], + "OrthogonalMatchingPursuit": [ + ("Function/MLModel/Sklearn/Named/OrthogonalMatchingPursuit.fit", 1), + ("Function/MLModel/Sklearn/Named/OrthogonalMatchingPursuit.predict", 2), + ("Function/MLModel/Sklearn/Named/OrthogonalMatchingPursuit.score", 1), + ], + "OrthogonalMatchingPursuitCV": [ + ("Function/MLModel/Sklearn/Named/OrthogonalMatchingPursuitCV.fit", 1), + ("Function/MLModel/Sklearn/Named/OrthogonalMatchingPursuitCV.predict", 2), + ("Function/MLModel/Sklearn/Named/OrthogonalMatchingPursuitCV.score", 1), + ], + "PassiveAggressiveClassifier": [ + ("Function/MLModel/Sklearn/Named/PassiveAggressiveClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/PassiveAggressiveClassifier.predict", 2), + ("Function/MLModel/Sklearn/Named/PassiveAggressiveClassifier.score", 1), + ], + "PassiveAggressiveRegressor": [ + ("Function/MLModel/Sklearn/Named/PassiveAggressiveRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/PassiveAggressiveRegressor.predict", 2), + ("Function/MLModel/Sklearn/Named/PassiveAggressiveRegressor.score", 1), + ], + "Perceptron": [ + ("Function/MLModel/Sklearn/Named/Perceptron.fit", 1), + ("Function/MLModel/Sklearn/Named/Perceptron.predict", 2), + ("Function/MLModel/Sklearn/Named/Perceptron.score", 1), + ], + "Ridge": [ + ("Function/MLModel/Sklearn/Named/Ridge.fit", 1), + ("Function/MLModel/Sklearn/Named/Ridge.predict", 2), + ("Function/MLModel/Sklearn/Named/Ridge.score", 1), + ], + "RidgeCV": [ + ("Function/MLModel/Sklearn/Named/RidgeCV.fit", 1), + ("Function/MLModel/Sklearn/Named/RidgeCV.predict", 2), + ("Function/MLModel/Sklearn/Named/RidgeCV.score", 1), + ], + "RidgeClassifier": [ + ("Function/MLModel/Sklearn/Named/RidgeClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/RidgeClassifier.predict", 2), + ("Function/MLModel/Sklearn/Named/RidgeClassifier.score", 1), + ], + "RidgeClassifierCV": [ + ("Function/MLModel/Sklearn/Named/RidgeClassifierCV.fit", 1), + ("Function/MLModel/Sklearn/Named/RidgeClassifierCV.predict", 2), + ("Function/MLModel/Sklearn/Named/RidgeClassifierCV.score", 1), + ], + "TheilSenRegressor": [ + ("Function/MLModel/Sklearn/Named/TheilSenRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/TheilSenRegressor.predict", 2), + ("Function/MLModel/Sklearn/Named/TheilSenRegressor.score", 1), + ], + "RANSACRegressor": [ + ("Function/MLModel/Sklearn/Named/RANSACRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/RANSACRegressor.predict", 1), + ("Function/MLModel/Sklearn/Named/RANSACRegressor.score", 1), + ], + } + expected_transaction_name = "test_linear_models:_test" + if six.PY3: + expected_transaction_name = "test_linear_models:test_model_methods_wrapped_in_function_trace.._test" + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[linear_model_name], + rollup_metrics=expected_scoped_metrics[linear_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_linear_model(linear_model_name) + + _test() + + +@pytest.mark.skipif(SKLEARN_VERSION < (1, 1, 0), reason="Requires sklearn >= v1.1") +@pytest.mark.parametrize( + "linear_model_name", + [ + "PoissonRegressor", + "GammaRegressor", + "TweedieRegressor", + "QuantileRegressor", + "SGDClassifier", + "SGDRegressor", + "SGDOneClassSVM", + ], +) +def test_above_v1_1_model_methods_wrapped_in_function_trace(linear_model_name, run_linear_model): + expected_scoped_metrics = { + "PoissonRegressor": [ + ("Function/MLModel/Sklearn/Named/PoissonRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/PoissonRegressor.predict", 1), + ("Function/MLModel/Sklearn/Named/PoissonRegressor.score", 1), + ], + "GammaRegressor": [ + ("Function/MLModel/Sklearn/Named/GammaRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/GammaRegressor.predict", 1), + ("Function/MLModel/Sklearn/Named/GammaRegressor.score", 1), + ], + "TweedieRegressor": [ + ("Function/MLModel/Sklearn/Named/TweedieRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/TweedieRegressor.predict", 1), + ("Function/MLModel/Sklearn/Named/TweedieRegressor.score", 1), + ], + "QuantileRegressor": [ + ("Function/MLModel/Sklearn/Named/QuantileRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/QuantileRegressor.predict", 2), + ("Function/MLModel/Sklearn/Named/QuantileRegressor.score", 1), + ], + "SGDClassifier": [ + ("Function/MLModel/Sklearn/Named/SGDClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/SGDClassifier.predict", 2), + ("Function/MLModel/Sklearn/Named/SGDClassifier.score", 1), + ], + "SGDRegressor": [ + ("Function/MLModel/Sklearn/Named/SGDRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/SGDRegressor.predict", 2), + ("Function/MLModel/Sklearn/Named/SGDRegressor.score", 1), + ], + "SGDOneClassSVM": [ + ("Function/MLModel/Sklearn/Named/SGDOneClassSVM.fit", 1), + ("Function/MLModel/Sklearn/Named/SGDOneClassSVM.predict", 1), + ], + } + expected_transaction_name = "test_linear_models:_test" + if six.PY3: + expected_transaction_name = ( + "test_linear_models:test_above_v1_1_model_methods_wrapped_in_function_trace.._test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[linear_model_name], + rollup_metrics=expected_scoped_metrics[linear_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_linear_model(linear_model_name) + + _test() + + +@pytest.fixture +def run_linear_model(): + def _run(linear_model_name): + import sklearn.linear_model + from sklearn.datasets import load_iris + from sklearn.model_selection import train_test_split + + X, y = load_iris(return_X_y=True) + x_train, x_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) + + if linear_model_name == "GammaRegressor": + x_train = [[1, 2], [2, 3], [3, 4], [4, 3]] + y_train = [19, 26, 33, 30] + x_test = [[1, 2], [2, 3], [3, 4], [4, 3]] + y_test = [19, 26, 33, 30] + elif linear_model_name in [ + "MultiTaskElasticNet", + "MultiTaskElasticNetCV", + "MultiTaskLasso", + "MultiTaskLassoCV", + ]: + y_train = x_train + y_test = x_test + + clf = getattr(sklearn.linear_model, linear_model_name)() + + model = clf.fit(x_train, y_train) + model.predict(x_test) + + if hasattr(model, "score"): + model.score(x_test, y_test) + + return model + + return _run From 6273e4d09de97f66ad1ae61c26f792b275b5072b Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Wed, 25 Jan 2023 09:27:57 -0800 Subject: [PATCH 28/54] Add ml_model function wrapper API (#739) * Add function traces around model methods * Add test for multiple calls to model method * Fixup: add comments & organize * Refactor * Instrument predict function. * Add data trains fixture. * Add testing and cleanup for custom feature events. * Update test_tree_models. * Add back training step logic to predict proxy. * Remove unused files. * Add ml_model wrapper and tests. * Add column name logic. * Update branch. * Update column name mapping logic. * Fix py2 failures. * Fix pypy37 failure. * Revise feature_names logic. * Add more ml_model wrapper testing. * Fix linter unused import * [Mega-Linter] Apply linters fixes * Bump tests. * Bump tests. Co-authored-by: Hannah Stepanek Co-authored-by: umaannamalai --- newrelic/api/ml_model.py | 33 ++ newrelic/hooks/mlmodel_sklearn.py | 116 ++++--- tests/mlmodel_sklearn/test_ml_model.py | 445 +++++++++++++++++++++++++ 3 files changed, 549 insertions(+), 45 deletions(-) create mode 100644 newrelic/api/ml_model.py create mode 100644 tests/mlmodel_sklearn/test_ml_model.py diff --git a/newrelic/api/ml_model.py b/newrelic/api/ml_model.py new file mode 100644 index 000000000..dafceec98 --- /dev/null +++ b/newrelic/api/ml_model.py @@ -0,0 +1,33 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +from newrelic.common.object_names import callable_name +from newrelic.hooks.mlmodel_sklearn import _nr_instrument_model + + +def wrap_mlmodel(model, name=None, version=None, feature_names=None, label_names=None): + model_callable_name = callable_name(model) + _class = model.__class__.__name__ + module = sys.modules[model_callable_name.split(":")[0]] + _nr_instrument_model(module, _class) + if name: + model._nr_wrapped_name = name + if version: + model._nr_wrapped_version = version + if feature_names: + model._nr_wrapped_feature_names = feature_names + if label_names: + model._nr_wrapped_label_names = label_names diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index 7c5f78aaa..3faabc661 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import sys import uuid @@ -32,6 +33,7 @@ "r2_score", ) PY2 = sys.version_info[0] == 2 +_logger = logging.getLogger(__name__) class PredictReturnTypeProxy(ObjectProxy): @@ -84,16 +86,54 @@ def _nr_wrapper_method(wrapped, instance, args, kwargs): wrap_function_wrapper(module, "%s.%s" % (_class, method), _nr_wrapper_method) -def find_type_category(value): - value_type = None - python_type = str(type(value)) +def find_type_category(data_set, row_index, column_index): + # If pandas DataFrame, return type of column. + pd = sys.modules.get("pandas", None) + if pd and isinstance(data_set, pd.DataFrame): + value_type = data_set.iloc[:, column_index].dtype.name + if value_type == "category": + return "categorical" + categorized_value_type = categorize_data_type(value_type) + return categorized_value_type + # If it's not a pandas DataFrame then it is a list or numpy array. + python_type = str(type(data_set[column_index][row_index])) + return categorize_data_type(python_type) + + +def categorize_data_type(python_type): if "int" in python_type or "float" in python_type or "complex" in python_type: - value_type = "numerical" - elif "bool" in python_type: - value_type = "bool" - elif "str" in python_type or "unicode" in python_type: - value_type = "str" - return value_type + return "numerical" + if "bool" in python_type: + return "bool" + if "str" in python_type or "unicode" in python_type: + return "str" + else: + return python_type + + +def _get_feature_column_names(user_provided_feature_names, features): + import numpy as np + + num_feature_columns = np.array(features).shape[1] + + # If the user provided feature names are the correct size, return the user provided feature + # names. + if user_provided_feature_names and len(user_provided_feature_names) == num_feature_columns: + return user_provided_feature_names + + # If the user provided feature names aren't the correct size, log a warning and do not use the user provided feature names. + if user_provided_feature_names: + _logger.warning( + "The number of feature names passed to the ml_model wrapper function is not equal to the number of columns in the data set. Please supply the correct number of feature names." + ) + + # If the user doesn't provide the feature names or they were provided but the size was incorrect and the features are a pandas data frame, return the column names from the pandas data frame. + pd = sys.modules.get("pandas", None) + if pd and isinstance(features, pd.DataFrame): + return features.columns + + # If the user doesn't provide the feature names or they were provided but the size was incorrect and the features are not a pandas data frame, return the column indexes as the feature names. + return np.array(range(num_feature_columns)) def bind_predict(X, *args, **kwargs): @@ -101,48 +141,34 @@ def bind_predict(X, *args, **kwargs): def wrap_predict(transaction, _class, wrapped, instance, args, kwargs): + import numpy as np + data_set = bind_predict(*args, **kwargs) inference_id = uuid.uuid4() model_name = getattr(instance, "_nr_wrapped_name", _class) model_version = getattr(instance, "_nr_wrapped_version", "0.0.0") - + user_provided_feature_names = getattr(instance, "_nr_wrapped_feature_names", None) settings = transaction.settings if transaction.settings is not None else global_settings() + if settings and settings.machine_learning and settings.machine_learning.inference_event_value.enabled: - # Pandas Dataframe - pd = sys.modules.get("pandas", None) - if pd and isinstance(data_set, pd.DataFrame): - for (colname, colval) in data_set.iteritems(): - for value in colval.values: - value_type = data_set[colname].dtype.name - if value_type == "category": - value_type = "categorical" - else: - value_type = find_type_category(value) - transaction.record_custom_event( - "ML Model Feature Event", - { - "inference_id": inference_id, - "model_name": model_name, - "model_version": model_version, - "feature_name": colname, - "type": value_type, - "value": str(value), - }, - ) - else: - for feature in data_set: - for col_index, value in enumerate(feature): - transaction.record_custom_event( - "ML Model Feature Event", - { - "inference_id": inference_id, - "model_name": model_name, - "model_version": model_version, - "feature_name": str(col_index), - "type": find_type_category(value), - "value": str(value), - }, - ) + final_feature_names = _get_feature_column_names(user_provided_feature_names, data_set) + np_casted_data_set = np.array(data_set) + + for col_index, feature in enumerate(np_casted_data_set): + for row_index, value in enumerate(feature): + value_type = find_type_category(data_set, row_index, col_index) + + transaction.record_custom_event( + "ML Model Feature Event", + { + "inference_id": inference_id, + "model_name": model_name, + "model_version": model_version, + "feature_name": str(final_feature_names[row_index]), + "type": value_type, + "value": str(value), + }, + ) def _nr_instrument_model(module, model_class): diff --git a/tests/mlmodel_sklearn/test_ml_model.py b/tests/mlmodel_sklearn/test_ml_model.py new file mode 100644 index 000000000..7745cf357 --- /dev/null +++ b/tests/mlmodel_sklearn/test_ml_model.py @@ -0,0 +1,445 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import pandas +import six +from _validate_custom_events import validate_custom_events +from testing_support.fixtures import ( + reset_core_stats_engine, + validate_custom_event_count, +) + +from newrelic.api.background_task import background_task +from newrelic.api.ml_model import wrap_mlmodel + +try: + from sklearn.tree._classes import BaseDecisionTree +except ImportError: + from sklearn.tree.tree import BaseDecisionTree + +_logger = logging.getLogger(__name__) + + +# Create custom model that isn't auto-instrumented to validate ml_model wrapper functionality +class CustomTestModel(BaseDecisionTree): + if six.PY2: + + def __init__( + self, + criterion="mse", + splitter="random", + max_depth=None, + min_samples_split=2, + min_samples_leaf=1, + min_weight_fraction_leaf=0.0, + max_features=None, + random_state=0, + max_leaf_nodes=None, + min_impurity_decrease=0.0, + min_impurity_split=None, + class_weight=None, + presort=False, + ): + + super(CustomTestModel, self).__init__( + criterion=criterion, + splitter=splitter, + max_depth=max_depth, + min_samples_split=min_samples_split, + min_samples_leaf=min_samples_leaf, + min_weight_fraction_leaf=min_weight_fraction_leaf, + max_features=max_features, + max_leaf_nodes=max_leaf_nodes, + class_weight=class_weight, + random_state=random_state, + min_impurity_decrease=min_impurity_decrease, + min_impurity_split=min_impurity_split, + presort=presort, + ) + + else: + + def __init__( + self, + criterion="poisson", + splitter="random", + max_depth=None, + min_samples_split=2, + min_samples_leaf=1, + min_weight_fraction_leaf=0.0, + max_features=None, + random_state=0, + max_leaf_nodes=None, + min_impurity_decrease=0.0, + class_weight=None, + ccp_alpha=0.0, + ): + + super().__init__( + criterion=criterion, + splitter=splitter, + max_depth=max_depth, + min_samples_split=min_samples_split, + min_samples_leaf=min_samples_leaf, + min_weight_fraction_leaf=min_weight_fraction_leaf, + max_features=max_features, + max_leaf_nodes=max_leaf_nodes, + class_weight=class_weight, + random_state=random_state, + min_impurity_decrease=min_impurity_decrease, + ccp_alpha=ccp_alpha, + ) + + def fit(self, X, y, sample_weight=None, check_input=True): + return super(CustomTestModel, self).fit( + X, + y, + sample_weight=sample_weight, + check_input=check_input, + ) + + def predict(self, X, check_input=True): + return super(CustomTestModel, self).predict(X, check_input=check_input) + + +int_list_recorded_custom_events = [ + { + "users": { + "inference_id": None, + "model_name": "MyCustomModel", + "model_version": "1.2.3", + "feature_name": None, + "type": "numerical", + "value": "1.0", + } + }, + { + "users": { + "inference_id": None, + "model_name": "MyCustomModel", + "model_version": "1.2.3", + "feature_name": None, + "type": "numerical", + "value": "2.0", + } + }, +] + + +@reset_core_stats_engine() +def test_wrapper_attrs_custom_model_int_list(): + @validate_custom_event_count(count=2) + @validate_custom_events(int_list_recorded_custom_events) + @background_task() + def _test(): + x_train = [[0, 0], [1, 1]] + y_train = [0, 1] + x_test = [[1.0, 2.0]] + + model = CustomTestModel().fit(x_train, y_train) + wrap_mlmodel(model, name="MyCustomModel", version="1.2.3") + + labels = model.predict(x_test) + + return model + + _test() + + +pandas_df_recorded_custom_events = [ + { + "users": { + "inference_id": None, + "model_name": "PandasTestModel", + "model_version": "1.5.0b1", + "feature_name": "feature1", + "type": "categorical", + "value": "5.0", + } + }, + { + "users": { + "inference_id": None, + "model_name": "PandasTestModel", + "model_version": "1.5.0b1", + "feature_name": "feature1", + "type": "categorical", + "value": "6.0", + } + }, + { + "users": { + "inference_id": None, + "model_name": "PandasTestModel", + "model_version": "1.5.0b1", + "feature_name": "feature2", + "type": "categorical", + "value": "7.0", + } + }, + { + "users": { + "inference_id": None, + "model_name": "PandasTestModel", + "model_version": "1.5.0b1", + "feature_name": "feature2", + "type": "categorical", + "value": "8.0", + } + }, + { + "users": { + "inference_id": None, + "model_name": "PandasTestModel", + "model_version": "1.5.0b1", + "feature_name": "feature3", + "type": "categorical", + "value": "9.0", + } + }, + { + "users": { + "inference_id": None, + "model_name": "PandasTestModel", + "model_version": "1.5.0b1", + "feature_name": "feature3", + "type": "categorical", + "value": "10.0", + } + }, +] + + +@reset_core_stats_engine() +def test_wrapper_attrs_custom_model_pandas_df(): + @validate_custom_event_count(count=6) + @validate_custom_events(pandas_df_recorded_custom_events) + @background_task() + def _test(): + x_train = pandas.DataFrame({"col1": [0, 0], "col2": [1, 1], "col3": [2, 2]}, dtype="category") + y_train = pandas.DataFrame({"label": [0, 1]}, dtype="category") + x_test = pandas.DataFrame({"col1": [5.0, 6.0], "col2": [7.0, 8.0], "col3": [9.0, 10.0]}, dtype="category") + + model = CustomTestModel().fit(x_train, y_train) + wrap_mlmodel( + model, name="PandasTestModel", version="1.5.0b1", feature_names=["feature1", "feature2", "feature3"] + ) + labels = model.predict(x_test) + + return model + + _test() + + +pandas_df_recorded_builtin_events = [ + { + "users": { + "inference_id": None, + "model_name": "MyDecisionTreeClassifier", + "model_version": "1.5.0b1", + "feature_name": "feature1", + "type": "numerical", + "value": "12", + } + }, + { + "users": { + "inference_id": None, + "model_name": "MyDecisionTreeClassifier", + "model_version": "1.5.0b1", + "feature_name": "feature1", + "type": "numerical", + "value": "13", + } + }, + { + "users": { + "inference_id": None, + "model_name": "MyDecisionTreeClassifier", + "model_version": "1.5.0b1", + "feature_name": "feature2", + "type": "numerical", + "value": "14", + } + }, + { + "users": { + "inference_id": None, + "model_name": "MyDecisionTreeClassifier", + "model_version": "1.5.0b1", + "feature_name": "feature2", + "type": "numerical", + "value": "15", + } + }, +] + + +@reset_core_stats_engine() +def test_wrapper_attrs_builtin_model(): + @validate_custom_event_count(count=4) + @validate_custom_events(pandas_df_recorded_builtin_events) + @background_task() + def _test(): + import sklearn.tree + + x_train = pandas.DataFrame({"col1": [0, 0], "col2": [1, 1]}, dtype="int") + y_train = pandas.DataFrame({"label": [0, 1]}, dtype="int") + x_test = pandas.DataFrame({"col1": [12, 13], "col2": [14, 15]}, dtype="int") + + clf = getattr(sklearn.tree, "DecisionTreeClassifier")(random_state=0) + + model = clf.fit(x_train, y_train) + wrap_mlmodel(model, name="MyDecisionTreeClassifier", version="1.5.0b1", feature_names=["feature1", "feature2"]) + labels = model.predict(x_test) + + return model + + _test() + + +pandas_df_mismatched_custom_events = [ + { + "users": { + "inference_id": None, + "model_name": "MyDecisionTreeClassifier", + "model_version": "1.5.0b1", + "feature_name": "col1", + "type": "numerical", + "value": "12", + } + }, + { + "users": { + "inference_id": None, + "model_name": "MyDecisionTreeClassifier", + "model_version": "1.5.0b1", + "feature_name": "col1", + "type": "numerical", + "value": "13", + } + }, + { + "users": { + "inference_id": None, + "model_name": "MyDecisionTreeClassifier", + "model_version": "1.5.0b1", + "feature_name": "col2", + "type": "numerical", + "value": "14", + } + }, + { + "users": { + "inference_id": None, + "model_name": "MyDecisionTreeClassifier", + "model_version": "1.5.0b1", + "feature_name": "col2", + "type": "numerical", + "value": "15", + } + }, + { + "users": { + "inference_id": None, + "model_name": "MyDecisionTreeClassifier", + "model_version": "1.5.0b1", + "feature_name": "col3", + "type": "numerical", + "value": "16", + } + }, + { + "users": { + "inference_id": None, + "model_name": "MyDecisionTreeClassifier", + "model_version": "1.5.0b1", + "feature_name": "col3", + "type": "numerical", + "value": "17", + } + }, +] + + +@reset_core_stats_engine() +def test_wrapper_mismatched_feature_names_and_cols_df(): + @validate_custom_event_count(count=6) + @validate_custom_events(pandas_df_mismatched_custom_events) + @background_task() + def _test(): + import sklearn.tree + + x_train = pandas.DataFrame({"col1": [0, 0], "col2": [1, 1], "col3": [2, 2]}, dtype="int") + y_train = pandas.DataFrame({"label": [0, 1]}, dtype="int") + x_test = pandas.DataFrame({"col1": [12, 13], "col2": [14, 15], "col3": [16, 17]}, dtype="int") + + clf = getattr(sklearn.tree, "DecisionTreeClassifier")(random_state=0) + + model = clf.fit(x_train, y_train) + wrap_mlmodel(model, name="MyDecisionTreeClassifier", version="1.5.0b1", feature_names=["feature1", "feature2"]) + labels = model.predict(x_test) + + return model + + _test() + + +numpy_str_mismatched_custom_events = [ + { + "users": { + "inference_id": None, + "model_name": "MyDecisionTreeClassifier", + "model_version": "0.0.1", + "feature_name": "0", + "type": "str", + "value": "20", + } + }, + { + "users": { + "inference_id": None, + "model_name": "MyDecisionTreeClassifier", + "model_version": "0.0.1", + "feature_name": "1", + "type": "str", + "value": "21", + } + }, +] + + +@reset_core_stats_engine() +def test_wrapper_mismatched_feature_names_and_cols_np_array(): + @validate_custom_events(numpy_str_mismatched_custom_events) + @validate_custom_event_count(count=2) + @background_task() + def _test(): + import numpy as np + import sklearn.tree + + x_train = np.array([[20, 20], [21, 21]], dtype=" Date: Fri, 27 Jan 2023 15:28:08 -0800 Subject: [PATCH 29/54] Report feature event w/o value (#754) Report the feature event w/o the raw value if `machine_learning.inference_event_value.enabled` is False. --- newrelic/hooks/mlmodel_sklearn.py | 36 ++++++++-------- tests/mlmodel_sklearn/test_feature_events.py | 45 ++++++++++++++++++++ 2 files changed, 62 insertions(+), 19 deletions(-) diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index 3faabc661..f4f8b115d 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -150,25 +150,23 @@ def wrap_predict(transaction, _class, wrapped, instance, args, kwargs): user_provided_feature_names = getattr(instance, "_nr_wrapped_feature_names", None) settings = transaction.settings if transaction.settings is not None else global_settings() - if settings and settings.machine_learning and settings.machine_learning.inference_event_value.enabled: - final_feature_names = _get_feature_column_names(user_provided_feature_names, data_set) - np_casted_data_set = np.array(data_set) - - for col_index, feature in enumerate(np_casted_data_set): - for row_index, value in enumerate(feature): - value_type = find_type_category(data_set, row_index, col_index) - - transaction.record_custom_event( - "ML Model Feature Event", - { - "inference_id": inference_id, - "model_name": model_name, - "model_version": model_version, - "feature_name": str(final_feature_names[row_index]), - "type": value_type, - "value": str(value), - }, - ) + final_feature_names = _get_feature_column_names(user_provided_feature_names, data_set) + np_casted_data_set = np.array(data_set) + + for col_index, feature in enumerate(np_casted_data_set): + for row_index, value in enumerate(feature): + value_type = find_type_category(data_set, row_index, col_index) + event = { + "inference_id": inference_id, + "model_name": model_name, + "model_version": model_version, + "feature_name": str(final_feature_names[row_index]), + "type": value_type, + } + # Don't include the raw value when inference_event_value is disabled. + if settings and settings.machine_learning and settings.machine_learning.inference_event_value.enabled: + event["value"] = str(value) + transaction.record_custom_event("ML Model Feature Event", event) def _nr_instrument_model(module, model_class): diff --git a/tests/mlmodel_sklearn/test_feature_events.py b/tests/mlmodel_sklearn/test_feature_events.py index 044a6f293..40d764fd3 100644 --- a/tests/mlmodel_sklearn/test_feature_events.py +++ b/tests/mlmodel_sklearn/test_feature_events.py @@ -18,6 +18,7 @@ import pandas from _validate_custom_events import validate_custom_events from testing_support.fixtures import ( + override_application_settings, reset_core_stats_engine, validate_custom_event_count, ) @@ -412,3 +413,47 @@ def _test(): return model _test() + + +numpy_str_recorded_custom_events_no_value = [ + { + "users": { + "inference_id": None, + "model_name": "DecisionTreeClassifier", + "model_version": "0.0.0", + "feature_name": "0", + "type": "str", + } + }, + { + "users": { + "inference_id": None, + "model_name": "DecisionTreeClassifier", + "model_version": "0.0.0", + "feature_name": "1", + "type": "str", + } + }, +] + + +@reset_core_stats_engine() +@override_application_settings({"machine_learning.inference_event_value.enabled": False}) +def test_does_not_include_value_when_inference_event_value_enabled_is_false(): + @validate_custom_events(numpy_str_recorded_custom_events_no_value) + @validate_custom_event_count(count=2) + @background_task() + def _test(): + import sklearn.tree + + x_train = np.array([[20, 20], [21, 21]], dtype=" Date: Wed, 15 Feb 2023 11:02:45 -0700 Subject: [PATCH 30/54] Prediction metric stats (#715) * Add function traces around model methods * Add test for multiple calls to model method * Fixup: add comments & organize * Refactor * Instrument predict function. * Add data trains fixture. * Add testing and cleanup for custom feature events. * Update test_tree_models. * Add back training step logic to predict proxy. * Remove unused files. * Update branch. * Add impl and testing for label events. * [Mega-Linter] Apply linters fixes * Add feature_name to expected events for int test. * Fix py37 test. * Add function traces around model methods * Add test for multiple calls to model method * Fixup: add comments & organize * Refactor * Instrument predict function. * Add data trains fixture. * Add testing and cleanup for custom feature events. * Update test_tree_models. * Add back training step logic to predict proxy. * Remove unused files. * Update branch. * Add impl and testing for label events. * Add feature_name to expected events for int test. * [Mega-Linter] Apply linters fixes * Fix py37 test. * Update logic to not report value when setting is disabled. * Fixup flake8 unused import * Touch up label event logic and rename test_feature_events to test_inference_events. * Add test case for multilabel output. * Fix 2D np array test. * Add metrics about prediction data * Fixup * Fix user provided label names * Fixup label related column name tests * [Mega-Linter] Apply linters fixes * Assert label events in multilabel output test * Remove return of stats * Remove spaces from version specification * Fix py2.7 output difference in pandas test --------- Co-authored-by: Uma Annamalai Co-authored-by: umaannamalai Co-authored-by: hmstepanek Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- newrelic/hooks/mlmodel_sklearn.py | 135 ++++++++- ...ure_events.py => test_inference_events.py} | 167 ++++++----- tests/mlmodel_sklearn/test_ml_model.py | 151 +++++----- .../mlmodel_sklearn/test_prediction_stats.py | 282 ++++++++++++++++++ tox.ini | 4 +- 5 files changed, 573 insertions(+), 166 deletions(-) rename tests/mlmodel_sklearn/{test_feature_events.py => test_inference_events.py} (80%) create mode 100644 tests/mlmodel_sklearn/test_prediction_stats.py diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index f4f8b115d..8bf8a7d64 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -36,6 +36,17 @@ _logger = logging.getLogger(__name__) +def isnumeric(column): + import numpy as np + + try: + column.astype(np.float64) + return [True] * len(column) + except: + pass + return [False] * len(column) + + class PredictReturnTypeProxy(ObjectProxy): def __init__(self, wrapped, model_name, training_step): super(ObjectProxy, self).__init__(wrapped) @@ -79,13 +90,129 @@ def _nr_wrapper_method(wrapped, instance, args, kwargs): # _nr_wrapped attrs that will attach model info to the data. if method in ("predict", "fit_predict"): training_step = getattr(instance, "_nr_wrapped_training_step", "Unknown") - wrap_predict(transaction, _class, wrapped, instance, args, kwargs) + inference_id = uuid.uuid4() + create_feature_event(transaction, _class, inference_id, instance, args, kwargs) + create_label_event(transaction, _class, inference_id, instance, return_val) return PredictReturnTypeProxy(return_val, model_name=_class, training_step=training_step) return return_val wrap_function_wrapper(module, "%s.%s" % (_class, method), _nr_wrapper_method) +def _calc_prediction_feature_stats(prediction_input, _class, feature_column_names): + import numpy as np + + # Drop any feature columns that are not numeric since we can't compute stats + # on non-numeric columns. + x = np.array(prediction_input) + isnumeric_features = np.apply_along_axis(isnumeric, 0, x) + numeric_features = x[isnumeric_features] + + # Drop any feature column names that are not numeric since we can't compute stats + # on non-numeric columns. + feature_column_names = feature_column_names[isnumeric_features[0]] + + # Only compute stats for features if we have any feature columns left after dropping + # non-numeric columns. + num_cols = len(feature_column_names) + if num_cols > 0: + # Boolean selection of numpy array values reshapes the array to a single + # dimension so we have to reshape it back into a 2D array. + features = np.reshape(numeric_features, (len(numeric_features) // num_cols, num_cols)) + features = features.astype(dtype=np.float64) + + _record_stats(features, feature_column_names, _class, "Feature") + + +def _record_stats(data, column_names, _class, column_type): + import numpy as np + + mean = np.mean(data, axis=0) + percentile25 = np.percentile(data, q=0.25, axis=0) + percentile50 = np.percentile(data, q=0.50, axis=0) + percentile75 = np.percentile(data, q=0.75, axis=0) + standard_deviation = np.std(data, axis=0) + _min = np.min(data, axis=0) + _max = np.max(data, axis=0) + _count = data.shape[0] + + transaction = current_transaction() + + # Currently record_metric only supports a subset of these stats so we have + # to upload them one at a time instead of as a dictionary of stats per + # feature column. + for index, col_name in enumerate(column_names): + metric_name = "MLModel/Sklearn/Named/%s/Predict/%s/%s" % (_class, column_type, col_name) + transaction.record_custom_metrics( + [ + ("%s/%s" % (metric_name, "Mean"), float(mean[index])), + ("%s/%s" % (metric_name, "Percentile25"), float(percentile25[index])), + ("%s/%s" % (metric_name, "Percentile50"), float(percentile50[index])), + ("%s/%s" % (metric_name, "Percentile75"), float(percentile75[index])), + ("%s/%s" % (metric_name, "StandardDeviation"), float(standard_deviation[index])), + ("%s/%s" % (metric_name, "Min"), float(_min[index])), + ("%s/%s" % (metric_name, "Max"), float(_max[index])), + ("%s/%s" % (metric_name, "Count"), _count), + ] + ) + + +def _calc_prediction_label_stats(labels, _class, label_column_names): + import numpy as np + + labels = np.array(labels, dtype=np.float64) + _record_stats(labels, label_column_names, _class, "Label") + + +def create_label_event(transaction, _class, inference_id, instance, return_val): + model_name = getattr(instance, "_nr_wrapped_name", _class) + model_version = getattr(instance, "_nr_wrapped_version", "0.0.0") + label_names = getattr(instance, "_nr_wrapped_label_names", None) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if return_val is not None: + import numpy as np + + if not hasattr(return_val, "__iter__"): + labels = np.array([return_val]) + else: + labels = np.array(return_val) + if len(labels.shape) == 1: + labels = np.reshape(labels, (len(labels) // 1, 1)) + + label_names_list = _get_label_names(label_names, labels) + _calc_prediction_label_stats(labels, _class, label_names_list) + for prediction in labels: + for index, value in enumerate(prediction): + python_value_type = str(type(value)) + value_type = str(categorize_data_type(python_value_type)) + + event = { + "inference_id": inference_id, + "model_name": model_name, + "model_version": model_version, + "label_name": str(label_names_list[index]), + "type": value_type, + "value": str(value), + } + # Don't include the raw value when inference_event_value is disabled. + if settings and settings.machine_learning and settings.machine_learning.inference_event_value.enabled: + event["value"] = str(value) + transaction.record_custom_event("ML Model Label Event", event) + + +def _get_label_names(user_defined_label_names, prediction_array): + import numpy as np + + if user_defined_label_names is None or len(user_defined_label_names) != prediction_array.shape[1]: + _logger.warning( + "The number of label names passed to the ml_model wrapper function is not equal to the number of predictions in the data set. Please supply the correct number of label names." + ) + return np.array(range(prediction_array.shape[1])) + else: + return user_defined_label_names + + def find_type_category(data_set, row_index, column_index): # If pandas DataFrame, return type of column. pd = sys.modules.get("pandas", None) @@ -119,7 +246,7 @@ def _get_feature_column_names(user_provided_feature_names, features): # If the user provided feature names are the correct size, return the user provided feature # names. if user_provided_feature_names and len(user_provided_feature_names) == num_feature_columns: - return user_provided_feature_names + return np.array(user_provided_feature_names) # If the user provided feature names aren't the correct size, log a warning and do not use the user provided feature names. if user_provided_feature_names: @@ -140,11 +267,10 @@ def bind_predict(X, *args, **kwargs): return X -def wrap_predict(transaction, _class, wrapped, instance, args, kwargs): +def create_feature_event(transaction, _class, inference_id, instance, args, kwargs): import numpy as np data_set = bind_predict(*args, **kwargs) - inference_id = uuid.uuid4() model_name = getattr(instance, "_nr_wrapped_name", _class) model_version = getattr(instance, "_nr_wrapped_version", "0.0.0") user_provided_feature_names = getattr(instance, "_nr_wrapped_feature_names", None) @@ -152,6 +278,7 @@ def wrap_predict(transaction, _class, wrapped, instance, args, kwargs): final_feature_names = _get_feature_column_names(user_provided_feature_names, data_set) np_casted_data_set = np.array(data_set) + _calc_prediction_feature_stats(data_set, _class, final_feature_names) for col_index, feature in enumerate(np_casted_data_set): for row_index, value in enumerate(feature): diff --git a/tests/mlmodel_sklearn/test_feature_events.py b/tests/mlmodel_sklearn/test_inference_events.py similarity index 80% rename from tests/mlmodel_sklearn/test_feature_events.py rename to tests/mlmodel_sklearn/test_inference_events.py index 40d764fd3..be38f36ce 100644 --- a/tests/mlmodel_sklearn/test_feature_events.py +++ b/tests/mlmodel_sklearn/test_inference_events.py @@ -36,16 +36,6 @@ "value": "2.0", } }, - { - "users": { - "inference_id": None, - "model_name": "DecisionTreeClassifier", - "model_version": "0.0.0", - "feature_name": "col1", - "type": "categorical", - "value": "3.0", - } - }, { "users": { "inference_id": None, @@ -61,9 +51,9 @@ "inference_id": None, "model_name": "DecisionTreeClassifier", "model_version": "0.0.0", - "feature_name": "col2", - "type": "categorical", - "value": "1.0", + "label_name": "0", + "type": "numerical", + "value": "27.0", } }, ] @@ -72,22 +62,26 @@ @reset_core_stats_engine() def test_pandas_df_categorical_feature_event(): @validate_custom_events(pandas_df_category_recorded_custom_events) - @validate_custom_event_count(count=4) + @validate_custom_event_count(count=3) @background_task() def _test(): import sklearn.tree clf = getattr(sklearn.tree, "DecisionTreeClassifier")(random_state=0) model = clf.fit( - pandas.DataFrame({"col1": [0, 0], "col2": [1, 1]}, dtype="category"), pandas.DataFrame({"label": [0, 1]}) + pandas.DataFrame({"col1": [27.0, 24.0], "col2": [23.0, 25.0]}, dtype="category"), + pandas.DataFrame({"label": [27.0, 28.0]}), ) - labels = model.predict(pandas.DataFrame({"col1": [2.0, 3.0], "col2": [4.0, 1.0]}, dtype="category")) + labels = model.predict(pandas.DataFrame({"col1": [2.0], "col2": [4.0]}, dtype="category")) return model _test() +label_type = "bool" if sys.version_info < (3, 8) else "numerical" +true_label_value = "True" if sys.version_info < (3, 8) else "1.0" +false_label_value = "False" if sys.version_info < (3, 8) else "0.0" pandas_df_bool_recorded_custom_events = [ { "users": { @@ -99,16 +93,6 @@ def _test(): "value": "True", } }, - { - "users": { - "inference_id": None, - "model_name": "DecisionTreeClassifier", - "model_version": "0.0.0", - "feature_name": "col1", - "type": "bool", - "value": "False", - } - }, { "users": { "inference_id": None, @@ -124,9 +108,9 @@ def _test(): "inference_id": None, "model_name": "DecisionTreeClassifier", "model_version": "0.0.0", - "feature_name": "col2", - "type": "bool", - "value": "False", + "label_name": "0", + "type": label_type, + "value": true_label_value, } }, ] @@ -135,7 +119,7 @@ def _test(): @reset_core_stats_engine() def test_pandas_df_bool_feature_event(): @validate_custom_events(pandas_df_bool_recorded_custom_events) - @validate_custom_event_count(count=4) + @validate_custom_event_count(count=3) @background_task() def _test(): import sklearn.tree @@ -143,7 +127,7 @@ def _test(): dtype_name = "bool" if sys.version_info < (3, 8) else "boolean" x_train = pandas.DataFrame({"col1": [True, False], "col2": [True, False]}, dtype=dtype_name) y_train = pandas.DataFrame({"label": [True, False]}, dtype=dtype_name) - x_test = pandas.DataFrame({"col1": [True, False], "col2": [True, False]}, dtype=dtype_name) + x_test = pandas.DataFrame({"col1": [True], "col2": [True]}, dtype=dtype_name) clf = getattr(sklearn.tree, "DecisionTreeClassifier")(random_state=0) model = clf.fit(x_train, y_train) @@ -165,16 +149,6 @@ def _test(): "value": "100.0", } }, - { - "users": { - "inference_id": None, - "model_name": "DecisionTreeRegressor", - "model_version": "0.0.0", - "feature_name": "col1", - "type": "numerical", - "value": "200.0", - } - }, { "users": { "inference_id": None, @@ -190,9 +164,9 @@ def _test(): "inference_id": None, "model_name": "DecisionTreeRegressor", "model_version": "0.0.0", - "feature_name": "col2", + "label_name": "0", "type": "numerical", - "value": "400.0", + "value": "345.6", } }, ] @@ -201,14 +175,14 @@ def _test(): @reset_core_stats_engine() def test_pandas_df_float_feature_event(): @validate_custom_events(pandas_df_float_recorded_custom_events) - @validate_custom_event_count(count=4) + @validate_custom_event_count(count=3) @background_task() def _test(): import sklearn.tree x_train = pandas.DataFrame({"col1": [120.0, 254.0], "col2": [236.9, 234.5]}, dtype="float64") y_train = pandas.DataFrame({"label": [345.6, 456.7]}, dtype="float64") - x_test = pandas.DataFrame({"col1": [100.0, 200.0], "col2": [300.0, 400.0]}, dtype="float64") + x_test = pandas.DataFrame({"col1": [100.0], "col2": [300.0]}, dtype="float64") clf = getattr(sklearn.tree, "DecisionTreeRegressor")(random_state=0) @@ -246,19 +220,9 @@ def _test(): "inference_id": None, "model_name": "ExtraTreeRegressor", "model_version": "0.0.0", - "feature_name": "0", - "type": "numerical", - "value": "3", - } - }, - { - "users": { - "inference_id": None, - "model_name": "ExtraTreeRegressor", - "model_version": "0.0.0", - "feature_name": "1", + "label_name": "0", "type": "numerical", - "value": "4", + "value": "1.0", } }, ] @@ -267,14 +231,14 @@ def _test(): @reset_core_stats_engine() def test_int_list(): @validate_custom_events(int_list_recorded_custom_events) - @validate_custom_event_count(count=4) + @validate_custom_event_count(count=3) @background_task() def _test(): import sklearn.tree x_train = [[0, 0], [1, 1]] y_train = [0, 1] - x_test = [[1, 2], [3, 4]] + x_test = [[1, 2]] clf = getattr(sklearn.tree, "ExtraTreeRegressor")(random_state=0) model = clf.fit(x_train, y_train) @@ -311,19 +275,9 @@ def _test(): "inference_id": None, "model_name": "ExtraTreeRegressor", "model_version": "0.0.0", - "feature_name": "0", + "label_name": "0", "type": "numerical", - "value": "14", - } - }, - { - "users": { - "inference_id": None, - "model_name": "ExtraTreeRegressor", - "model_version": "0.0.0", - "feature_name": "1", - "type": "numerical", - "value": "15", + "value": "11.0", } }, ] @@ -332,14 +286,14 @@ def _test(): @reset_core_stats_engine() def test_numpy_int_array(): @validate_custom_events(numpy_int_recorded_custom_events) - @validate_custom_event_count(count=4) + @validate_custom_event_count(count=3) @background_task() def _test(): import sklearn.tree x_train = np.array([[10, 10], [11, 11]], dtype="int") y_train = np.array([10, 11], dtype="int") - x_test = np.array([[12, 13], [14, 15]], dtype="int") + x_test = np.array([[12, 13]], dtype="int") clf = getattr(sklearn.tree, "ExtraTreeRegressor")(random_state=0) model = clf.fit(x_train, y_train) @@ -395,9 +349,9 @@ def _test(): @reset_core_stats_engine() -def test_numpy_str_array(): +def test_numpy_str_array_multiple_features(): @validate_custom_events(numpy_str_recorded_custom_events) - @validate_custom_event_count(count=4) + @validate_custom_event_count(count=6) @background_task() def _test(): import sklearn.tree @@ -434,6 +388,15 @@ def _test(): "type": "str", } }, + { + "users": { + "inference_id": None, + "model_name": "DecisionTreeClassifier", + "model_version": "0.0.0", + "label_name": "0", + "type": "str", + } + }, ] @@ -441,7 +404,7 @@ def _test(): @override_application_settings({"machine_learning.inference_event_value.enabled": False}) def test_does_not_include_value_when_inference_event_value_enabled_is_false(): @validate_custom_events(numpy_str_recorded_custom_events_no_value) - @validate_custom_event_count(count=2) + @validate_custom_event_count(count=3) @background_task() def _test(): import sklearn.tree @@ -457,3 +420,55 @@ def _test(): return model _test() + + +multilabel_output_label_events = [ + { + "users": { + "inference_id": None, + "model_name": "MultiOutputClassifier", + "model_version": "0.0.0", + "label_name": "0", + "type": "numerical", + "value": "1", + } + }, + { + "users": { + "inference_id": None, + "model_name": "MultiOutputClassifier", + "model_version": "0.0.0", + "label_name": "1", + "type": "numerical", + "value": "0", + } + }, + { + "users": { + "inference_id": None, + "model_name": "MultiOutputClassifier", + "model_version": "0.0.0", + "label_name": "2", + "type": "numerical", + "value": "1", + } + }, +] + + +@reset_core_stats_engine() +def test_custom_event_count_multilabel_output(): + @validate_custom_events(multilabel_output_label_events) + # The expected count of 23 comes from 20 feature events + 3 label events to be generated + @validate_custom_event_count(count=23) + @background_task() + def _test(): + from sklearn.datasets import make_multilabel_classification + from sklearn.linear_model import LogisticRegression + from sklearn.multioutput import MultiOutputClassifier + + x_train, y_train = make_multilabel_classification(n_classes=3, random_state=0) + clf = MultiOutputClassifier(LogisticRegression()).fit(x_train, y_train) + clf.predict([x_train[-1]]) + + _test() diff --git a/tests/mlmodel_sklearn/test_ml_model.py b/tests/mlmodel_sklearn/test_ml_model.py index 7745cf357..305188b70 100644 --- a/tests/mlmodel_sklearn/test_ml_model.py +++ b/tests/mlmodel_sklearn/test_ml_model.py @@ -53,7 +53,6 @@ def __init__( class_weight=None, presort=False, ): - super(CustomTestModel, self).__init__( criterion=criterion, splitter=splitter, @@ -87,7 +86,6 @@ def __init__( class_weight=None, ccp_alpha=0.0, ): - super().__init__( criterion=criterion, splitter=splitter, @@ -115,13 +113,14 @@ def predict(self, X, check_input=True): return super(CustomTestModel, self).predict(X, check_input=check_input) +label_value = "1.0" if six.PY2 else "0.5" int_list_recorded_custom_events = [ { "users": { "inference_id": None, "model_name": "MyCustomModel", "model_version": "1.2.3", - "feature_name": None, + "feature_name": "0", "type": "numerical", "value": "1.0", } @@ -131,17 +130,27 @@ def predict(self, X, check_input=True): "inference_id": None, "model_name": "MyCustomModel", "model_version": "1.2.3", - "feature_name": None, + "feature_name": "1", "type": "numerical", "value": "2.0", } }, + { + "users": { + "inference_id": None, + "model_name": "MyCustomModel", + "model_version": "1.2.3", + "label_name": "0", + "type": "numerical", + "value": label_value, + } + }, ] @reset_core_stats_engine() -def test_wrapper_attrs_custom_model_int_list(): - @validate_custom_event_count(count=2) +def test_custom_model_int_list_no_features_and_labels(): + @validate_custom_event_count(count=3) @validate_custom_events(int_list_recorded_custom_events) @background_task() def _test(): @@ -167,17 +176,7 @@ def _test(): "model_version": "1.5.0b1", "feature_name": "feature1", "type": "categorical", - "value": "5.0", - } - }, - { - "users": { - "inference_id": None, - "model_name": "PandasTestModel", - "model_version": "1.5.0b1", - "feature_name": "feature1", - "type": "categorical", - "value": "6.0", + "value": "0", } }, { @@ -187,17 +186,7 @@ def _test(): "model_version": "1.5.0b1", "feature_name": "feature2", "type": "categorical", - "value": "7.0", - } - }, - { - "users": { - "inference_id": None, - "model_name": "PandasTestModel", - "model_version": "1.5.0b1", - "feature_name": "feature2", - "type": "categorical", - "value": "8.0", + "value": "0", } }, { @@ -207,7 +196,7 @@ def _test(): "model_version": "1.5.0b1", "feature_name": "feature3", "type": "categorical", - "value": "9.0", + "value": "1", } }, { @@ -215,9 +204,9 @@ def _test(): "inference_id": None, "model_name": "PandasTestModel", "model_version": "1.5.0b1", - "feature_name": "feature3", - "type": "categorical", - "value": "10.0", + "label_name": "label1", + "type": "numerical", + "value": "0.5" if six.PY3 else "0.0", } }, ] @@ -225,20 +214,23 @@ def _test(): @reset_core_stats_engine() def test_wrapper_attrs_custom_model_pandas_df(): - @validate_custom_event_count(count=6) + @validate_custom_event_count(count=4) @validate_custom_events(pandas_df_recorded_custom_events) @background_task() def _test(): - x_train = pandas.DataFrame({"col1": [0, 0], "col2": [1, 1], "col3": [2, 2]}, dtype="category") - y_train = pandas.DataFrame({"label": [0, 1]}, dtype="category") - x_test = pandas.DataFrame({"col1": [5.0, 6.0], "col2": [7.0, 8.0], "col3": [9.0, 10.0]}, dtype="category") + x_train = pandas.DataFrame({"col1": [0, 1], "col2": [0, 1], "col3": [1, 2]}, dtype="category") + y_train = [0, 1] + x_test = pandas.DataFrame({"col1": [0], "col2": [0], "col3": [1]}, dtype="category") - model = CustomTestModel().fit(x_train, y_train) + model = CustomTestModel(random_state=0).fit(x_train, y_train) wrap_mlmodel( - model, name="PandasTestModel", version="1.5.0b1", feature_names=["feature1", "feature2", "feature3"] + model, + name="PandasTestModel", + version="1.5.0b1", + feature_names=["feature1", "feature2", "feature3"], + label_names=["label1"], ) labels = model.predict(x_test) - return model _test() @@ -255,16 +247,6 @@ def _test(): "value": "12", } }, - { - "users": { - "inference_id": None, - "model_name": "MyDecisionTreeClassifier", - "model_version": "1.5.0b1", - "feature_name": "feature1", - "type": "numerical", - "value": "13", - } - }, { "users": { "inference_id": None, @@ -280,9 +262,9 @@ def _test(): "inference_id": None, "model_name": "MyDecisionTreeClassifier", "model_version": "1.5.0b1", - "feature_name": "feature2", + "label_name": "label1", "type": "numerical", - "value": "15", + "value": "0", } }, ] @@ -290,7 +272,7 @@ def _test(): @reset_core_stats_engine() def test_wrapper_attrs_builtin_model(): - @validate_custom_event_count(count=4) + @validate_custom_event_count(count=3) @validate_custom_events(pandas_df_recorded_builtin_events) @background_task() def _test(): @@ -298,12 +280,18 @@ def _test(): x_train = pandas.DataFrame({"col1": [0, 0], "col2": [1, 1]}, dtype="int") y_train = pandas.DataFrame({"label": [0, 1]}, dtype="int") - x_test = pandas.DataFrame({"col1": [12, 13], "col2": [14, 15]}, dtype="int") + x_test = pandas.DataFrame({"col1": [12], "col2": [14]}, dtype="int") clf = getattr(sklearn.tree, "DecisionTreeClassifier")(random_state=0) model = clf.fit(x_train, y_train) - wrap_mlmodel(model, name="MyDecisionTreeClassifier", version="1.5.0b1", feature_names=["feature1", "feature2"]) + wrap_mlmodel( + model, + name="MyDecisionTreeClassifier", + version="1.5.0b1", + feature_names=["feature1", "feature2"], + label_names=["label1"], + ) labels = model.predict(x_test) return model @@ -322,16 +310,6 @@ def _test(): "value": "12", } }, - { - "users": { - "inference_id": None, - "model_name": "MyDecisionTreeClassifier", - "model_version": "1.5.0b1", - "feature_name": "col1", - "type": "numerical", - "value": "13", - } - }, { "users": { "inference_id": None, @@ -342,16 +320,6 @@ def _test(): "value": "14", } }, - { - "users": { - "inference_id": None, - "model_name": "MyDecisionTreeClassifier", - "model_version": "1.5.0b1", - "feature_name": "col2", - "type": "numerical", - "value": "15", - } - }, { "users": { "inference_id": None, @@ -367,32 +335,37 @@ def _test(): "inference_id": None, "model_name": "MyDecisionTreeClassifier", "model_version": "1.5.0b1", - "feature_name": "col3", + "label_name": "0", "type": "numerical", - "value": "17", + "value": "1", } }, ] @reset_core_stats_engine() -def test_wrapper_mismatched_feature_names_and_cols_df(): - @validate_custom_event_count(count=6) +def test_wrapper_mismatched_features_and_labels_df(): + @validate_custom_event_count(count=4) @validate_custom_events(pandas_df_mismatched_custom_events) @background_task() def _test(): import sklearn.tree - x_train = pandas.DataFrame({"col1": [0, 0], "col2": [1, 1], "col3": [2, 2]}, dtype="int") + x_train = pandas.DataFrame({"col1": [7, 8], "col2": [9, 10], "col3": [24, 25]}, dtype="int") y_train = pandas.DataFrame({"label": [0, 1]}, dtype="int") - x_test = pandas.DataFrame({"col1": [12, 13], "col2": [14, 15], "col3": [16, 17]}, dtype="int") + x_test = pandas.DataFrame({"col1": [12], "col2": [14], "col3": [16]}, dtype="int") clf = getattr(sklearn.tree, "DecisionTreeClassifier")(random_state=0) model = clf.fit(x_train, y_train) - wrap_mlmodel(model, name="MyDecisionTreeClassifier", version="1.5.0b1", feature_names=["feature1", "feature2"]) + wrap_mlmodel( + model, + name="MyDecisionTreeClassifier", + version="1.5.0b1", + feature_names=["feature1", "feature2"], + label_names=["label1", "label2"], + ) labels = model.predict(x_test) - return model _test() @@ -419,13 +392,23 @@ def _test(): "value": "21", } }, + { + "users": { + "inference_id": None, + "model_name": "MyDecisionTreeClassifier", + "model_version": "0.0.1", + "label_name": "0", + "type": "str", + "value": "21", + } + }, ] @reset_core_stats_engine() -def test_wrapper_mismatched_feature_names_and_cols_np_array(): +def test_wrapper_mismatched_features_and_labels_np_array(): @validate_custom_events(numpy_str_mismatched_custom_events) - @validate_custom_event_count(count=2) + @validate_custom_event_count(count=3) @background_task() def _test(): import numpy as np diff --git a/tests/mlmodel_sklearn/test_prediction_stats.py b/tests/mlmodel_sklearn/test_prediction_stats.py new file mode 100644 index 000000000..e38595039 --- /dev/null +++ b/tests/mlmodel_sklearn/test_prediction_stats.py @@ -0,0 +1,282 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import pandas as pd +import pytest +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.packages import six + + +@pytest.mark.parametrize( + "x_train,y_train,x_test,metrics", + [ + ( + [[0, 0], [1, 1]], + [0, 1], + [[2.0, 2.0], [0, 0.5]], + [ + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Mean", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Percentile25", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Percentile50", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Percentile75", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/StandardDeviation", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Min", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Max", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Count", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Mean", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Percentile25", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Percentile50", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Percentile75", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/StandardDeviation", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Min", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Max", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Count", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Mean", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Percentile25", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Percentile50", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Percentile75", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/StandardDeviation", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Min", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Max", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Count", 1), + ], + ), + ( + np.array([[0, 0], [1, 1]]), + [0, 1], + np.array([[2.0, 2.0], [0, 0.5]]), + [ + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Mean", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Percentile25", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Percentile50", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Percentile75", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/StandardDeviation", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Min", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Max", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Count", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Mean", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Percentile25", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Percentile50", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Percentile75", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/StandardDeviation", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Min", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Max", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Count", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Mean", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Percentile25", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Percentile50", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Percentile75", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/StandardDeviation", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Min", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Max", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Count", 1), + ], + ), + ( + np.array([["a", 0, 4], ["b", 1, 3]], dtype="._test" if six.PY3 else "test_prediction_stats:_test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + custom_metrics=metrics, + background_task=True, + ) + @background_task() + def _test(): + run_model(x_train, y_train, x_test) + + _test() + + +def test_prediction_stats_multilabel_output(): + expected_transaction_name = ( + "test_prediction_stats:test_prediction_stats_multilabel_output.._test" + if six.PY3 + else "test_prediction_stats:_test" + ) + stats = ["Mean", "Percentile25", "Percentile50", "Percentile75", "StandardDeviation", "Min", "Max", "Count"] + metrics = [ + ("MLModel/Sklearn/Named/MultiOutputClassifier/Predict/Feature/%s/%s" % (feature_col, stat_name), 1) + for feature_col in range(20) + for stat_name in stats + ] + metrics.extend( + [ + ("MLModel/Sklearn/Named/MultiOutputClassifier/Predict/Label/%s/%s" % (label_col, stat_name), 1) + for label_col in range(3) + for stat_name in stats + ] + ) + + @validate_transaction_metrics( + expected_transaction_name, + custom_metrics=metrics, + background_task=True, + ) + @background_task() + def _test(): + from sklearn.datasets import make_multilabel_classification + from sklearn.linear_model import LogisticRegression + from sklearn.multioutput import MultiOutputClassifier + + x_train, y_train = make_multilabel_classification(n_classes=3, random_state=0) + clf = MultiOutputClassifier(LogisticRegression()).fit(x_train, y_train) + clf.predict([x_train[-1]]) + + _test() + + +@pytest.fixture +def run_model(): + def _run(x_train, y_train, x_test): + from sklearn import dummy + + clf = dummy.DummyClassifier(random_state=0) + model = clf.fit(x_train, y_train) + + labels = model.predict(x_test) + + return _run diff --git a/tox.ini b/tox.ini index c896eae99..f2cb4518b 100644 --- a/tox.ini +++ b/tox.ini @@ -213,8 +213,8 @@ deps = application_gearman: gearman<3.0.0 mlmodel_sklearn: pandas mlmodel_sklearn-scikitlearnlatest: scikit-learn - mlmodel_sklearn-scikitlearn0101: scikit-learn < 1.1 - mlmodel_sklearn-scikitlearn0020: scikit-learn < 0.21 + mlmodel_sklearn-scikitlearn0101: scikit-learn<1.1 + mlmodel_sklearn-scikitlearn0020: scikit-learn<0.21 component_djangorestframework-djangorestframework0300: Django<1.9 component_djangorestframework-djangorestframework0300: djangorestframework<3.1 component_djangorestframework-djangorestframeworklatest: Django From 1b706a38c4b0fc422c9e13e212c5590f2160e76d Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Tue, 2 May 2023 13:43:02 -0700 Subject: [PATCH 31/54] Add new ML event type (#802) * Add new machine learning event data type Co-authored-by: Tim Pansino Co-authored-by: Lalleh Rafeei Co-authored-by: Uma Annamalai * Validate new machine learning event data type Co-authored-by: Tim Pansino Co-authored-by: Lalleh Rafeei Co-authored-by: Uma Annamalai * Fixup: minor inconsistencies * Fixup * Remove code coverage fixture * Fix lint errors * Increase timeout for python tests --------- Co-authored-by: Tim Pansino Co-authored-by: Lalleh Rafeei Co-authored-by: Uma Annamalai --- .github/workflows/tests.yml | 2 +- newrelic/agent.py | 2 + newrelic/api/application.py | 4 + newrelic/api/transaction.py | 52 ++++++- newrelic/core/agent.py | 7 + newrelic/core/application.py | 38 +++++ newrelic/core/config.py | 1 + newrelic/core/data_collector.py | 7 + newrelic/core/stats_engine.py | 84 ++++++++--- newrelic/core/transaction_node.py | 1 + tests/agent_features/test_ml_events.py | 137 ++++++++++++++++++ tests/agent_unittests/test_harvest_loop.py | 6 + tests/mlmodel_sklearn/conftest.py | 7 - ...validate_log_events_outside_transaction.py | 15 +- .../validators/validate_ml_event_count.py | 54 +++++++ ...date_ml_event_count_outside_transaction.py | 55 +++++++ .../validators/validate_ml_events.py | 110 ++++++++++++++ .../validate_ml_events_outside_transaction.py | 64 ++++++++ 18 files changed, 610 insertions(+), 36 deletions(-) create mode 100644 tests/agent_features/test_ml_events.py create mode 100644 tests/testing_support/validators/validate_ml_event_count.py create mode 100644 tests/testing_support/validators/validate_ml_event_count_outside_transaction.py create mode 100644 tests/testing_support/validators/validate_ml_events.py create mode 100644 tests/testing_support/validators/validate_ml_events_outside_transaction.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 15d105b83..f8b8bdeca 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -117,7 +117,7 @@ jobs: ] runs-on: ubuntu-20.04 - timeout-minutes: 30 + timeout-minutes: 45 steps: - uses: actions/checkout@v3 diff --git a/newrelic/agent.py b/newrelic/agent.py index 95a540780..70dd3a14d 100644 --- a/newrelic/agent.py +++ b/newrelic/agent.py @@ -59,6 +59,7 @@ from newrelic.api.transaction import record_custom_metric as __record_custom_metric from newrelic.api.transaction import record_custom_metrics as __record_custom_metrics from newrelic.api.transaction import record_log_event as __record_log_event +from newrelic.api.transaction import record_ml_event as __record_ml_event from newrelic.api.transaction import set_background_task as __set_background_task from newrelic.api.transaction import set_transaction_name as __set_transaction_name from newrelic.api.transaction import suppress_apdex_metric as __suppress_apdex_metric @@ -248,6 +249,7 @@ def __asgi_application(*args, **kwargs): record_custom_metrics = __wrap_api_call(__record_custom_metrics, "record_custom_metrics") record_custom_event = __wrap_api_call(__record_custom_event, "record_custom_event") record_log_event = __wrap_api_call(__record_log_event, "record_log_event") +record_ml_event = __wrap_api_call(__record_ml_event, "record_ml_event") accept_distributed_trace_payload = __wrap_api_call( __accept_distributed_trace_payload, "accept_distributed_trace_payload" ) diff --git a/newrelic/api/application.py b/newrelic/api/application.py index ea57829f2..1ff425a7b 100644 --- a/newrelic/api/application.py +++ b/newrelic/api/application.py @@ -146,6 +146,10 @@ def record_custom_event(self, event_type, params): if self.active: self._agent.record_custom_event(self._name, event_type, params) + def record_ml_event(self, event_type, params): + if self.active: + self._agent.record_ml_event(self._name, event_type, params) + def record_transaction(self, data): if self.active: self._agent.record_transaction(self._name, data) diff --git a/newrelic/api/transaction.py b/newrelic/api/transaction.py index f04bcba84..db29c9748 100644 --- a/newrelic/api/transaction.py +++ b/newrelic/api/transaction.py @@ -60,7 +60,11 @@ DST_NONE, DST_TRANSACTION_TRACER, ) -from newrelic.core.config import CUSTOM_EVENT_RESERVOIR_SIZE, LOG_EVENT_RESERVOIR_SIZE +from newrelic.core.config import ( + CUSTOM_EVENT_RESERVOIR_SIZE, + LOG_EVENT_RESERVOIR_SIZE, + ML_EVENT_RESERVOIR_SIZE, +) from newrelic.core.custom_event import create_custom_event from newrelic.core.log_event_node import LogEventNode from newrelic.core.stack_trace import exception_stack @@ -330,12 +334,15 @@ def __init__(self, application, enabled=None, source=None): self._custom_events = SampledDataSet( capacity=self._settings.event_harvest_config.harvest_limits.custom_event_data ) + # TODO Fix this with actual setting + self._ml_events = SampledDataSet(capacity=ML_EVENT_RESERVOIR_SIZE) self._log_events = SampledDataSet( capacity=self._settings.event_harvest_config.harvest_limits.log_event_data ) else: self._custom_events = SampledDataSet(capacity=CUSTOM_EVENT_RESERVOIR_SIZE) self._log_events = SampledDataSet(capacity=LOG_EVENT_RESERVOIR_SIZE) + self._ml_events = SampledDataSet(capacity=ML_EVENT_RESERVOIR_SIZE) def __del__(self): self._dead = True @@ -584,6 +591,7 @@ def __exit__(self, exc, value, tb): errors=tuple(self._errors), slow_sql=tuple(self._slow_sql), custom_events=self._custom_events, + ml_events=self._ml_events, log_events=self._log_events, apdex_t=self.apdex, suppress_apdex=self.suppress_apdex, @@ -1613,6 +1621,20 @@ def record_custom_event(self, event_type, params): if event: self._custom_events.add(event, priority=self.priority) + def record_ml_event(self, event_type, params): + settings = self._settings + + if not settings: + return + + # TODO Fix settings + if not settings.custom_insights_events.enabled: + return + + event = create_custom_event(event_type, params) + if event: + self._ml_events.add(event, priority=self.priority) + def _intern_string(self, value): return self._string_cache.setdefault(value, value) @@ -1926,6 +1948,34 @@ def record_custom_event(event_type, params, application=None): application.record_custom_event(event_type, params) +def record_ml_event(event_type, params, application=None): + """Record a machine learning custom event. + + Args: + event_type (str): The type (name) of the ml event. + params (dict): Attributes to add to the event. + application (newrelic.api.Application): Application instance. + + """ + + if application is None: + transaction = current_transaction() + if transaction: + transaction.record_ml_event(event_type, params) + else: + _logger.debug( + "record_ml_event has been called but no " + "transaction was running. As a result, the following event " + "has not been recorded. event_type: %r params: %r. To correct " + "this problem, supply an application object as a parameter to " + "this record_ml_event call.", + event_type, + params, + ) + elif application.enabled: + application.record_ml_event(event_type, params) + + def record_log_event(message, level=None, timestamp=None, application=None, priority=None): """Record a log event. diff --git a/newrelic/core/agent.py b/newrelic/core/agent.py index 6ab9571a4..8aab80d7e 100644 --- a/newrelic/core/agent.py +++ b/newrelic/core/agent.py @@ -531,6 +531,13 @@ def record_custom_event(self, app_name, event_type, params): application.record_custom_event(event_type, params) + def record_ml_event(self, app_name, event_type, params): + application = self._applications.get(app_name, None) + if application is None or not application.active: + return + + application.record_ml_event(event_type, params) + def record_log_event(self, app_name, message, level=None, timestamp=None, priority=None): application = self._applications.get(app_name, None) if application is None or not application.active: diff --git a/newrelic/core/application.py b/newrelic/core/application.py index 7be217428..3d905d84f 100644 --- a/newrelic/core/application.py +++ b/newrelic/core/application.py @@ -876,6 +876,23 @@ def record_custom_event(self, event_type, params): self._global_events_account += 1 self._stats_engine.record_custom_event(event) + def record_ml_event(self, event_type, params): + if not self._active_session: + return + + settings = self._stats_engine.settings + + # TODO Fix this with actual settings + if settings is None or not settings.custom_insights_events.enabled: + return + + event = create_custom_event(event_type, params) + + if event: + with self._stats_custom_lock: + self._global_events_account += 1 + self._stats_engine.record_ml_event(event) + def record_log_event(self, message, level=None, timestamp=None, priority=None): if not self._active_session: return @@ -1335,6 +1352,27 @@ def harvest(self, shutdown=False, flexible=False): stats.reset_custom_events() + # Send machine learning events + + # TODO Fix this with actual settings names + if configuration.collect_custom_events and configuration.custom_insights_events.enabled: + ml_events = stats.ml_events + + if ml_events: + if ml_events.num_samples > 0: + ml_event_samples = list(ml_events) + + _logger.debug("Sending machine learning event data for harvest of %r.", self._app_name) + + self._active_session.send_ml_events(ml_events.sampling_info, ml_event_samples) + ml_event_samples = None + + # As per spec + internal_count_metric("Supportability/Events/Customer/Seen", ml_events.num_seen) + internal_count_metric("Supportability/Events/Customer/Sent", ml_events.num_samples) + + stats.reset_ml_events() + # Send log events if ( diff --git a/newrelic/core/config.py b/newrelic/core/config.py index d8c719a5a..6f31ad3d3 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -56,6 +56,7 @@ ERROR_EVENT_RESERVOIR_SIZE = 100 SPAN_EVENT_RESERVOIR_SIZE = 2000 LOG_EVENT_RESERVOIR_SIZE = 10000 +ML_EVENT_RESERVOIR_SIZE = 100000 # settings that should be completely ignored if set server side IGNORED_SERVER_SIDE_SETTINGS = [ diff --git a/newrelic/core/data_collector.py b/newrelic/core/data_collector.py index 985e37240..44539020e 100644 --- a/newrelic/core/data_collector.py +++ b/newrelic/core/data_collector.py @@ -112,6 +112,13 @@ def send_custom_events(self, sampling_info, custom_event_data): payload = (self.agent_run_id, sampling_info, custom_event_data) return self._protocol.send("custom_event_data", payload) + def send_ml_events(self, sampling_info, custom_event_data): + """Called to submit sample set for machine learning events.""" + + # TODO Make this send to MELT/OTLP endpoint instead of agent listener + payload = (self.agent_run_id, sampling_info, custom_event_data) # TODO this payload will be different + return self._protocol.send("custom_event_data", payload) + def send_span_events(self, sampling_info, span_event_data): """Called to submit sample set for span events.""" diff --git a/newrelic/core/stats_engine.py b/newrelic/core/stats_engine.py index 203e3e796..04c870ae9 100644 --- a/newrelic/core/stats_engine.py +++ b/newrelic/core/stats_engine.py @@ -61,7 +61,7 @@ "reset_synthetics_events", ), "span_event_data": ("reset_span_events",), - "custom_event_data": ("reset_custom_events",), + "custom_event_data": ("reset_custom_events", "reset_ml_events"), "error_event_data": ("reset_error_events",), "log_event_data": ("reset_log_events",), } @@ -436,6 +436,7 @@ def __init__(self): self._transaction_events = SampledDataSet() self._error_events = SampledDataSet() self._custom_events = SampledDataSet() + self._ml_events = SampledDataSet() self._span_events = SampledDataSet() self._log_events = SampledDataSet() self._span_stream = None @@ -464,6 +465,10 @@ def transaction_events(self): def custom_events(self): return self._custom_events + @property + def ml_events(self): + return self._ml_events + @property def span_events(self): return self._span_events @@ -716,7 +721,6 @@ def notice_error(self, error=None, attributes=None, expected=None, ignore=None, user_attributes = create_user_attributes(custom_attributes, settings.attribute_filter) - # Extract additional details about the exception as agent attributes agent_attributes = {} @@ -728,28 +732,37 @@ def notice_error(self, error=None, attributes=None, expected=None, ignore=None, error_group_name = None try: # Call callback to obtain error group name - error_group_name_raw = settings.error_collector.error_group_callback(value, { - "traceback": tb, - "error.class": exc, - "error.message": message_raw, - "error.expected": is_expected, - "custom_params": attributes, - # Transaction specific items should be set to None - "transactionName": None, - "response.status": None, - "request.method": None, - "request.uri": None, - }) + error_group_name_raw = settings.error_collector.error_group_callback( + value, + { + "traceback": tb, + "error.class": exc, + "error.message": message_raw, + "error.expected": is_expected, + "custom_params": attributes, + # Transaction specific items should be set to None + "transactionName": None, + "response.status": None, + "request.method": None, + "request.uri": None, + }, + ) if error_group_name_raw: _, error_group_name = process_user_attribute("error.group.name", error_group_name_raw) if error_group_name is None or not isinstance(error_group_name, six.string_types): - raise ValueError("Invalid attribute value for error.group.name. Expected string, got: %s" % repr(error_group_name_raw)) + raise ValueError( + "Invalid attribute value for error.group.name. Expected string, got: %s" + % repr(error_group_name_raw) + ) else: agent_attributes["error.group.name"] = error_group_name except Exception: - _logger.error("Encountered error when calling error group callback:\n%s", "".join(traceback.format_exception(*sys.exc_info()))) - + _logger.error( + "Encountered error when calling error group callback:\n%s", + "".join(traceback.format_exception(*sys.exc_info())), + ) + agent_attributes = create_agent_attributes(agent_attributes, settings.attribute_filter) # Record the exception details. @@ -774,7 +787,7 @@ def notice_error(self, error=None, attributes=None, expected=None, ignore=None, for attr in agent_attributes: if attr.destinations & DST_ERROR_COLLECTOR: attributes["agentAttributes"][attr.name] = attr.value - + error_details = TracedError( start_time=time.time(), path="Exception", message=message, type=fullname, parameters=attributes ) @@ -829,6 +842,16 @@ def record_custom_event(self, event): if settings.collect_custom_events and settings.custom_insights_events.enabled: self._custom_events.add(event) + def record_ml_event(self, event): + settings = self.__settings + + if not settings: + return + + # TODO Fix this with actual settings + if settings.collect_custom_events and settings.custom_insights_events.enabled: + self._ml_events.add(event) + def record_custom_metric(self, name, value): """Record a single value metric, merging the data with any data from prior value metrics with the same name. @@ -1042,6 +1065,12 @@ def record_transaction(self, transaction): if settings.collect_custom_events and settings.custom_insights_events.enabled: self.custom_events.merge(transaction.custom_events) + # Merge in machine learning events + + # TODO Fix this with actual settings + if settings.collect_custom_events and settings.custom_insights_events.enabled: + self.ml_events.merge(transaction.ml_events) + # Merge in span events if settings.distributed_tracing.enabled and settings.span_events.enabled and settings.collect_span_events: @@ -1447,6 +1476,7 @@ def reset_stats(self, settings, reset_stream=False): self.reset_transaction_events() self.reset_error_events() self.reset_custom_events() + self.reset_ml_events() self.reset_span_events() self.reset_log_events() self.reset_synthetics_events() @@ -1489,6 +1519,13 @@ def reset_custom_events(self): else: self._custom_events = SampledDataSet() + def reset_ml_events(self): + if self.__settings is not None: + # TODO fix this with the actual setting + self._ml_events = SampledDataSet(8333) + else: + self._ml_events = SampledDataSet() + def reset_span_events(self): if self.__settings is not None: self._span_events = SampledDataSet(self.__settings.event_harvest_config.harvest_limits.span_event_data) @@ -1622,6 +1659,7 @@ def merge(self, snapshot): self._merge_error_events(snapshot) self._merge_error_traces(snapshot) self._merge_custom_events(snapshot) + self._merge_ml_events(snapshot) self._merge_span_events(snapshot) self._merge_log_events(snapshot) self._merge_sql(snapshot) @@ -1647,6 +1685,7 @@ def rollback(self, snapshot): self._merge_synthetics_events(snapshot, rollback=True) self._merge_error_events(snapshot) self._merge_custom_events(snapshot, rollback=True) + self._merge_ml_events(snapshot, rollback=True) self._merge_span_events(snapshot, rollback=True) self._merge_log_events(snapshot, rollback=True) @@ -1716,6 +1755,12 @@ def _merge_custom_events(self, snapshot, rollback=False): return self._custom_events.merge(events) + def _merge_ml_events(self, snapshot, rollback=False): + events = snapshot.ml_events + if not events: + return + self._ml_events.merge(events) + def _merge_span_events(self, snapshot, rollback=False): events = snapshot.span_events if not events: @@ -1798,6 +1843,9 @@ def reset_transaction_events(self): def reset_custom_events(self): self._custom_events = None + def reset_ml_events(self): + self._ml_events = None + def reset_span_events(self): self._span_events = None diff --git a/newrelic/core/transaction_node.py b/newrelic/core/transaction_node.py index 0faae3790..056d45a48 100644 --- a/newrelic/core/transaction_node.py +++ b/newrelic/core/transaction_node.py @@ -60,6 +60,7 @@ "errors", "slow_sql", "custom_events", + "ml_events", "log_events", "apdex_t", "suppress_apdex", diff --git a/tests/agent_features/test_ml_events.py b/tests/agent_features/test_ml_events.py new file mode 100644 index 000000000..9e616887d --- /dev/null +++ b/tests/agent_features/test_ml_events.py @@ -0,0 +1,137 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +import pytest +from testing_support.fixtures import ( # function_not_called,; override_application_settings, + reset_core_stats_engine, +) +from testing_support.validators.validate_ml_event_count import validate_ml_event_count +from testing_support.validators.validate_ml_events import validate_ml_events +from testing_support.validators.validate_ml_events_outside_transaction import ( + validate_ml_events_outside_transaction, +) + +from newrelic.api.application import application_instance as application +from newrelic.api.background_task import background_task +from newrelic.api.transaction import record_ml_event + +_now = time.time() + +_intrinsics = { + "type": "LabelEvent", + "timestamp": _now, +} + + +@pytest.mark.parametrize( + "params,expected", + [ + ({"foo": "bar"}, [(_intrinsics, {"foo": "bar"})]), + ({"foo": "bar", 123: "bad key"}, [(_intrinsics, {"foo": "bar"})]), + ({"foo": "bar", "*" * 256: "too long"}, [(_intrinsics, {"foo": "bar"})]), + ], + ids=["Valid key/value", "Bad key", "Value too long"], +) +def test_record_ml_event_inside_transaction(params, expected): + @validate_ml_events(expected) + @background_task() + def _test(): + record_ml_event("LabelEvent", params) + + _test() + + +@pytest.mark.parametrize( + "params,expected", + [ + ({"foo": "bar"}, [(_intrinsics, {"foo": "bar"})]), + ({"foo": "bar", 123: "bad key"}, [(_intrinsics, {"foo": "bar"})]), + ({"foo": "bar", "*" * 256: "too long"}, [(_intrinsics, {"foo": "bar"})]), + ], + ids=["Valid key/value", "Bad key", "Value too long"], +) +@reset_core_stats_engine() +def test_record_ml_event_outside_transaction(params, expected): + @validate_ml_events_outside_transaction(expected) + def _test(): + app = application() + record_ml_event("LabelEvent", params, application=app) + + _test() + + +@validate_ml_event_count(count=0) +@background_task() +def test_record_ml_event_inside_transaction_bad_event_type(): + record_ml_event("!@#$%^&*()", {"foo": "bar"}) + + +@reset_core_stats_engine() +@validate_ml_event_count(count=0) +def test_record_ml_event_outside_transaction_bad_event_type(): + app = application() + record_ml_event("!@#$%^&*()", {"foo": "bar"}, application=app) + + +@validate_ml_event_count(count=0) +@background_task() +def test_record_ml_event_inside_transaction_params_not_a_dict(): + record_ml_event("ParamsListEvent", ["not", "a", "dict"]) + + +@reset_core_stats_engine() +@validate_ml_event_count(count=0) +def test_record_ml_event_outside_transaction_params_not_a_dict(): + app = application() + record_ml_event("ParamsListEvent", ["not", "a", "dict"], application=app) + + +# Tests for ML Events configuration settings + +# TODO Test config settings here + +# @override_application_settings({'collect_ml_events': False}) +# @reset_core_stats_engine() +# @validate_ml_event_count(count=0) +# @background_task() +# def test_ml_event_settings_check_collector_flag(): +# record_ml_event('FooEvent', _user_params) + +# @override_application_settings({'ml_insights_events.enabled': False}) +# @reset_core_stats_engine() +# @validate_ml_event_count(count=0) +# @background_task() +# def test_ml_event_settings_check_ml_insights_enabled(): +# record_ml_event('FooEvent', _user_params) + +# Test that record_ml_event() methods will short-circuit. +# +# If the ml_insights_events setting is False, verify that the +# `create_ml_event()` function is not called, in order to avoid the +# event_type and attribute processing. + +# @override_application_settings({'ml_insights_events.enabled': False}) +# @function_not_called('newrelic.api.transaction', 'create_ml_event') +# @background_task() +# def test_transaction_create_ml_event_not_called(): +# record_ml_event('FooEvent', _user_params) + +# @override_application_settings({'ml_insights_events.enabled': False}) +# @function_not_called('newrelic.core.application', 'create_ml_event') +# @background_task() +# def test_application_create_ml_event_not_called(): +# app = application() +# record_ml_event('FooEvent', _user_params, application=app) diff --git a/tests/agent_unittests/test_harvest_loop.py b/tests/agent_unittests/test_harvest_loop.py index 305622107..5f14b270c 100644 --- a/tests/agent_unittests/test_harvest_loop.py +++ b/tests/agent_unittests/test_harvest_loop.py @@ -49,6 +49,11 @@ def transaction_node(request): event = create_custom_event("Custom", {}) custom_events.add(event) + ml_events = SampledDataSet(capacity=num_events) + for _ in range(num_events): + event = create_custom_event("Custom", {}) + ml_events.add(event) + log_events = SampledDataSet(capacity=num_events) for _ in range(num_events): event = LogEventNode(1653609717, "WARNING", "A", {}) @@ -122,6 +127,7 @@ def transaction_node(request): errors=errors, slow_sql=(), custom_events=custom_events, + ml_events=ml_events, log_events=log_events, apdex_t=0.5, suppress_apdex=False, diff --git a/tests/mlmodel_sklearn/conftest.py b/tests/mlmodel_sklearn/conftest.py index e48bc52a9..86884320f 100644 --- a/tests/mlmodel_sklearn/conftest.py +++ b/tests/mlmodel_sklearn/conftest.py @@ -13,17 +13,10 @@ # limitations under the License. from testing_support.fixtures import ( # noqa: F401, pylint: disable=W0611 - code_coverage_fixture, collector_agent_registration_fixture, collector_available_fixture, ) -_coverage_source = [ - "newrelic.hooks.mlmodel_sklearn", -] - -code_coverage = code_coverage_fixture(source=_coverage_source) - _default_settings = { "transaction_tracer.explain_threshold": 0.0, "transaction_tracer.transaction_threshold": 0.0, diff --git a/tests/testing_support/validators/validate_log_events_outside_transaction.py b/tests/testing_support/validators/validate_log_events_outside_transaction.py index f46b6e843..4bc941965 100644 --- a/tests/testing_support/validators/validate_log_events_outside_transaction.py +++ b/tests/testing_support/validators/validate_log_events_outside_transaction.py @@ -14,11 +14,11 @@ import copy +from testing_support.fixtures import catch_background_exceptions + +from newrelic.common.object_wrapper import function_wrapper, transient_function_wrapper from newrelic.packages import six -from newrelic.common.object_wrapper import (transient_function_wrapper, - function_wrapper) -from testing_support.fixtures import catch_background_exceptions def validate_log_events_outside_transaction(events): @function_wrapper @@ -35,18 +35,16 @@ def _validate_log_events_outside_transaction(wrapped, instance, args, kwargs): result = wrapped(*args, **kwargs) except: raise - else: - recorded_logs[:] = [] - recorded_logs.extend(list(instance._log_events)) + recorded_logs[:] = [] + recorded_logs.extend(list(instance._log_events)) return result - _new_wrapper = _validate_log_events_outside_transaction(wrapped) val = _new_wrapper(*args, **kwargs) assert record_called logs = copy.copy(recorded_logs) - + record_called[:] = [] recorded_logs[:] = [] @@ -60,7 +58,6 @@ def _validate_log_events_outside_transaction(wrapped, instance, args, kwargs): return val - def _check_log_attributes(expected, captured, mismatches): for key, value in six.iteritems(expected): if hasattr(captured, key): diff --git a/tests/testing_support/validators/validate_ml_event_count.py b/tests/testing_support/validators/validate_ml_event_count.py new file mode 100644 index 000000000..ec5de8dcf --- /dev/null +++ b/tests/testing_support/validators/validate_ml_event_count.py @@ -0,0 +1,54 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy + +from testing_support.fixtures import catch_background_exceptions + +from newrelic.common.object_wrapper import function_wrapper, transient_function_wrapper + + +def validate_ml_event_count(count=1): + @function_wrapper + def _validate_wrapper(wrapped, instance, args, kwargs): + + record_called = [] + recorded_events = [] + + @transient_function_wrapper("newrelic.core.stats_engine", "StatsEngine.record_transaction") + @catch_background_exceptions + def _validate_ml_event_count(wrapped, instance, args, kwargs): + record_called.append(True) + try: + result = wrapped(*args, **kwargs) + except: + raise + recorded_events.extend(list(instance._ml_events)) + + return result + + _new_wrapper = _validate_ml_event_count(wrapped) + val = _new_wrapper(*args, **kwargs) + if count: + assert record_called + events = copy.copy(recorded_events) + + record_called[:] = [] + recorded_events[:] = [] + + assert count == len(events), len(events) + + return val + + return _validate_wrapper diff --git a/tests/testing_support/validators/validate_ml_event_count_outside_transaction.py b/tests/testing_support/validators/validate_ml_event_count_outside_transaction.py new file mode 100644 index 000000000..6ac764d1a --- /dev/null +++ b/tests/testing_support/validators/validate_ml_event_count_outside_transaction.py @@ -0,0 +1,55 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy + +from testing_support.fixtures import catch_background_exceptions + +from newrelic.common.object_wrapper import function_wrapper, transient_function_wrapper + + +def validate_ml_event_count_outside_transaction(count=1): + @function_wrapper + def _validate_wrapper(wrapped, instance, args, kwargs): + + record_called = [] + recorded_events = [] + + @transient_function_wrapper("newrelic.core.stats_engine", "StatsEngine.record_ml_event") + @catch_background_exceptions + def _validate_ml_event_count_outside_transaction(wrapped, instance, args, kwargs): + record_called.append(True) + try: + result = wrapped(*args, **kwargs) + except: + raise + recorded_events[:] = [] + recorded_events.extend(list(instance._ml_events)) + + return result + + _new_wrapper = _validate_ml_event_count_outside_transaction(wrapped) + val = _new_wrapper(*args, **kwargs) + if count: + assert record_called + events = copy.copy(recorded_events) + + record_called[:] = [] + recorded_events[:] = [] + + assert count == len(events), len(events) + + return val + + return _validate_wrapper diff --git a/tests/testing_support/validators/validate_ml_events.py b/tests/testing_support/validators/validate_ml_events.py new file mode 100644 index 000000000..8f3150225 --- /dev/null +++ b/tests/testing_support/validators/validate_ml_events.py @@ -0,0 +1,110 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import time + +from testing_support.fixtures import catch_background_exceptions + +from newrelic.common.object_wrapper import function_wrapper, transient_function_wrapper +from newrelic.packages import six + + +def validate_ml_events(events): + @function_wrapper + def _validate_wrapper(wrapped, instance, args, kwargs): + + record_called = [] + recorded_events = [] + + @transient_function_wrapper("newrelic.core.stats_engine", "StatsEngine.record_transaction") + @catch_background_exceptions + def _validate_ml_events(wrapped, instance, args, kwargs): + record_called.append(True) + try: + result = wrapped(*args, **kwargs) + except: + raise + recorded_events[:] = [] + recorded_events.extend(list(instance._ml_events)) + + return result + + _new_wrapper = _validate_ml_events(wrapped) + val = _new_wrapper(*args, **kwargs) + assert record_called + events = copy.copy(recorded_events) + + record_called[:] = [] + recorded_events[:] = [] + + for expected in events: + matching_ml_events = 0 + mismatches = [] + for captured in events: + if _check_event_attributes(expected, captured, mismatches): + matching_ml_events += 1 + assert matching_ml_events == 1, _event_details(matching_ml_events, events, mismatches) + + return val + + return _validate_wrapper + + +def _check_event_attributes(expected, captured, mismatches): + assert len(captured) == 2 # [intrinsic, user attributes] + + intrinsics = captured[0] + + if intrinsics["type"] != expected[0]["type"]: + mismatches.append("key: type, value:<%s><%s>" % (expected[0]["type"], captured[0].get("type", None))) + return False + + now = time.time() + + if not (isinstance(intrinsics["timestamp"], int) and intrinsics["timestamp"] <= 1000.0 * now): + mismatches.append("key: timestamp, value:<%s>" % intrinsics["timestamp"]) + return False + + captured_keys = set(six.iterkeys(captured[1])) + expected_keys = set(six.iterkeys(expected[1])) + extra_keys = captured_keys - expected_keys + + if extra_keys: + mismatches.append("extra_keys: %s" % tuple(extra_keys)) + return False + + for key, value in six.iteritems(expected[1]): + if key in captured[1]: + captured_value = captured[1].get(key, None) + else: + mismatches.append("key: %s, value:<%s><%s>" % (key, value, captured[1].get(key, None))) + return False + + if value is not None: + if value != captured_value: + mismatches.append("key: %s, value:<%s><%s>" % (key, value, captured_value)) + return False + + return True + + +def _event_details(matching_ml_events, captured, mismatches): + details = [ + "matching_ml_events=%d" % matching_ml_events, + "mismatches=%s" % mismatches, + "captured_events=%s" % captured, + ] + + return "\n".join(details) diff --git a/tests/testing_support/validators/validate_ml_events_outside_transaction.py b/tests/testing_support/validators/validate_ml_events_outside_transaction.py new file mode 100644 index 000000000..107771442 --- /dev/null +++ b/tests/testing_support/validators/validate_ml_events_outside_transaction.py @@ -0,0 +1,64 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy + +from testing_support.fixtures import catch_background_exceptions +from testing_support.validators.validate_ml_events import ( + _check_event_attributes, + _event_details, +) + +from newrelic.common.object_wrapper import function_wrapper, transient_function_wrapper + + +def validate_ml_events_outside_transaction(events): + @function_wrapper + def _validate_wrapper(wrapped, instance, args, kwargs): + + record_called = [] + recorded_events = [] + + @transient_function_wrapper("newrelic.core.stats_engine", "StatsEngine.record_ml_event") + @catch_background_exceptions + def _validate_ml_events_outside_transaction(wrapped, instance, args, kwargs): + record_called.append(True) + try: + result = wrapped(*args, **kwargs) + except: + raise + recorded_events[:] = [] + recorded_events.extend(list(instance._ml_events)) + + return result + + _new_wrapper = _validate_ml_events_outside_transaction(wrapped) + val = _new_wrapper(*args, **kwargs) + assert record_called + events = copy.copy(recorded_events) + + record_called[:] = [] + recorded_events[:] = [] + + for expected in events: + matching_ml_events = 0 + mismatches = [] + for captured in events: + if _check_event_attributes(expected, captured, mismatches): + matching_ml_events += 1 + assert matching_ml_events == 1, _event_details(matching_ml_events, events, mismatches) + + return val + + return _validate_wrapper From d10c9f7438ae1b3df3c7176a9daa52cc7d96ce01 Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Fri, 5 May 2023 16:20:01 -0700 Subject: [PATCH 32/54] Add machine learning and ml_event config options (#811) * Add machine learning config options * machine_learning.enabled * machine_learning.inference_events.enable * machine_learning.inference_events.value.enabled * event_harvest_config.harvest_limits.ml_event_data Co-authored-by: Lalleh Rafeei Co-authored-by: Uma Annamalai * Replace all TODOs w/ new config settings * [Mega-Linter] Apply linters fixes * Trigger tests * Add insights settings & tests * Remove collect_custom_events & inference_events.enabled * Revert inference_events_value=>.value * Remove TODO * Fixup: format docstring * Remove file * Add tests for machine_learning.enabled --------- Co-authored-by: Lalleh Rafeei Co-authored-by: Uma Annamalai Co-authored-by: hmstepanek --- newrelic/api/transaction.py | 16 +--- newrelic/config.py | 30 ++++--- newrelic/core/agent_protocol.py | 3 +- newrelic/core/application.py | 6 +- newrelic/core/config.py | 35 ++++++-- newrelic/core/stats_engine.py | 9 +- newrelic/hooks/mlmodel_sklearn.py | 15 +++- .../agent_features/test_high_security_mode.py | 85 +++++++++++++------ tests/agent_features/test_ml_events.py | 47 +++++----- .../mlmodel_sklearn/test_inference_events.py | 49 ++++++++++- 10 files changed, 195 insertions(+), 100 deletions(-) diff --git a/newrelic/api/transaction.py b/newrelic/api/transaction.py index db29c9748..4e2ddecb8 100644 --- a/newrelic/api/transaction.py +++ b/newrelic/api/transaction.py @@ -163,13 +163,11 @@ def path(self): class Transaction(object): - STATE_PENDING = 0 STATE_RUNNING = 1 STATE_STOPPED = 2 def __init__(self, application, enabled=None, source=None): - self._application = application self._source = source @@ -334,8 +332,7 @@ def __init__(self, application, enabled=None, source=None): self._custom_events = SampledDataSet( capacity=self._settings.event_harvest_config.harvest_limits.custom_event_data ) - # TODO Fix this with actual setting - self._ml_events = SampledDataSet(capacity=ML_EVENT_RESERVOIR_SIZE) + self._ml_events = SampledDataSet(capacity=self._settings.event_harvest_config.harvest_limits.ml_event_data) self._log_events = SampledDataSet( capacity=self._settings.event_harvest_config.harvest_limits.log_event_data ) @@ -350,7 +347,6 @@ def __del__(self): self.__exit__(None, None, None) def __enter__(self): - assert self._state == self.STATE_PENDING # Bail out if the transaction is not enabled. @@ -410,7 +406,6 @@ def __enter__(self): return self def __exit__(self, exc, value, tb): - # Bail out if the transaction is not enabled. if not self.enabled: @@ -644,7 +639,6 @@ def __exit__(self, exc, value, tb): # new samples can cause an error. if not self.ignore_transaction: - self._application.record_transaction(node) @property @@ -937,9 +931,7 @@ def filter_request_parameters(self, params): @property def request_parameters(self): if (self.capture_params is None) or self.capture_params: - if self._request_params: - r_attrs = {} for k, v in self._request_params.items(): @@ -1103,7 +1095,6 @@ def _generate_distributed_trace_headers(self, data=None): try: data = data or self._create_distributed_trace_data() if data: - traceparent = W3CTraceParent(data).text() yield ("traceparent", traceparent) @@ -1390,7 +1381,6 @@ def _generate_response_headers(self, read_length=None): # process web external calls. if self.client_cross_process_id is not None: - # Need to work out queueing time and duration up to this # point for inclusion in metrics and response header. If the # recording of the transaction had been prematurely stopped @@ -1455,7 +1445,6 @@ def process_request_metadata(self, cat_linking_value): return self._process_incoming_cat_headers(encoded_cross_process_id, encoded_txn_header) def set_transaction_name(self, name, group=None, priority=None): - # Always perform this operation even if the transaction # is not active at the time as will be called from # constructor. If path has been frozen do not allow @@ -1627,8 +1616,7 @@ def record_ml_event(self, event_type, params): if not settings: return - # TODO Fix settings - if not settings.custom_insights_events.enabled: + if not settings.ml_insights_events.enabled: return event = create_custom_event(event_type, params) diff --git a/newrelic/config.py b/newrelic/config.py index 153dd960c..7c3fa2279 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -433,6 +433,7 @@ def _process_configuration(section): ) _process_setting(section, "custom_insights_events.enabled", "getboolean", None) _process_setting(section, "custom_insights_events.max_samples_stored", "getint", None) + _process_setting(section, "ml_insights_events.enabled", "getboolean", None) _process_setting(section, "distributed_tracing.enabled", "getboolean", None) _process_setting(section, "distributed_tracing.exclude_newrelic_header", "getboolean", None) _process_setting(section, "span_events.enabled", "getboolean", None) @@ -526,6 +527,7 @@ def _process_configuration(section): None, ) _process_setting(section, "event_harvest_config.harvest_limits.custom_event_data", "getint", None) + _process_setting(section, "event_harvest_config.harvest_limits.ml_event_data", "getint", None) _process_setting(section, "event_harvest_config.harvest_limits.span_event_data", "getint", None) _process_setting(section, "event_harvest_config.harvest_limits.error_event_data", "getint", None) _process_setting(section, "event_harvest_config.harvest_limits.log_event_data", "getint", None) @@ -542,7 +544,8 @@ def _process_configuration(section): _process_setting(section, "application_logging.metrics.enabled", "getboolean", None) _process_setting(section, "application_logging.local_decorating.enabled", "getboolean", None) - _process_setting(section, "machine_learning.inference_event_value.enabled", "getboolean", None) + _process_setting(section, "machine_learning.enabled", "getboolean", None) + _process_setting(section, "machine_learning.inference_events_value.enabled", "getboolean", None) # Loading of configuration from specified file and for specified @@ -870,6 +873,10 @@ def apply_local_high_security_mode_setting(settings): settings.custom_insights_events.enabled = False _logger.info(log_template, "custom_insights_events.enabled", True, False) + if settings.ml_insights_events.enabled: + settings.ml_insights_events.enabled = False + _logger.info(log_template, "ml_insights_events.enabled", True, False) + if settings.message_tracer.segment_parameters_enabled: settings.message_tracer.segment_parameters_enabled = False _logger.info(log_template, "message_tracer.segment_parameters_enabled", True, False) @@ -878,9 +885,9 @@ def apply_local_high_security_mode_setting(settings): settings.application_logging.forwarding.enabled = False _logger.info(log_template, "application_logging.forwarding.enabled", True, False) - if settings.machine_learning.inference_event_value.enabled: - settings.machine_learning.inference_event_value.enabled = False - _logger.info(log_template, "machine_learning.inference_event_value.enabled", True, False) + if settings.machine_learning.inference_events_value.enabled: + settings.machine_learning.inference_events_value.enabled = False + _logger.info(log_template, "machine_learning.inference_events_value.enabled", True, False) return settings @@ -2889,6 +2896,12 @@ def _process_module_builtin_defaults(): ) _process_module_definition("tastypie.api", "newrelic.hooks.component_tastypie", "instrument_tastypie_api") + _process_module_definition( + "sklearn.metrics", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_metrics", + ) + _process_module_definition( "sklearn.tree._classes", "newrelic.hooks.mlmodel_sklearn", @@ -2900,11 +2913,6 @@ def _process_module_builtin_defaults(): "newrelic.hooks.mlmodel_sklearn", "instrument_sklearn_tree_models", ) - _process_module_definition( - "sklearn.metrics", - "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_metrics", - ) _process_module_definition( "sklearn.compose._column_transformer", @@ -3670,9 +3678,7 @@ def _process_module_builtin_defaults(): "newrelic.hooks.application_celery", "instrument_celery_worker", ) - # _process_module_definition('celery.loaders.base', - # 'newrelic.hooks.application_celery', - # 'instrument_celery_loaders_base') + _process_module_definition( "celery.execute.trace", "newrelic.hooks.application_celery", diff --git a/newrelic/core/agent_protocol.py b/newrelic/core/agent_protocol.py index c5ac95e23..8b64aed4d 100644 --- a/newrelic/core/agent_protocol.py +++ b/newrelic/core/agent_protocol.py @@ -143,8 +143,9 @@ class AgentProtocol(object): "transaction_tracer.record_sql", "strip_exception_messages.enabled", "custom_insights_events.enabled", + "ml_insights_events.enabled", "application_logging.forwarding.enabled", - "machine_learning.inference_event_value.enabled", + "machine_learning.inference_events_value.enabled", ) LOGGER_FUNC_MAPPING = { diff --git a/newrelic/core/application.py b/newrelic/core/application.py index 3d905d84f..2e7985d18 100644 --- a/newrelic/core/application.py +++ b/newrelic/core/application.py @@ -882,8 +882,7 @@ def record_ml_event(self, event_type, params): settings = self._stats_engine.settings - # TODO Fix this with actual settings - if settings is None or not settings.custom_insights_events.enabled: + if settings is None or not settings.ml_insights_events.enabled: return event = create_custom_event(event_type, params) @@ -1354,8 +1353,7 @@ def harvest(self, shutdown=False, flexible=False): # Send machine learning events - # TODO Fix this with actual settings names - if configuration.collect_custom_events and configuration.custom_insights_events.enabled: + if configuration.ml_insights_events.enabled: ml_events = stats.ml_events if ml_events: diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 6f31ad3d3..ccd9a6132 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -51,10 +51,12 @@ # By default, Transaction Events and Custom Events have the same size # reservoir. Error Events have a different default size. +# Slow harvest (Every 60 seconds) DEFAULT_RESERVOIR_SIZE = 1200 -CUSTOM_EVENT_RESERVOIR_SIZE = 3600 ERROR_EVENT_RESERVOIR_SIZE = 100 SPAN_EVENT_RESERVOIR_SIZE = 2000 +# Fast harvest (Every 5 seconds, so divide by 12 to get average per minute value) +CUSTOM_EVENT_RESERVOIR_SIZE = 3600 LOG_EVENT_RESERVOIR_SIZE = 10000 ML_EVENT_RESERVOIR_SIZE = 100000 @@ -126,7 +128,7 @@ class MachineLearningSettings(Settings): pass -class MachineLearningInferenceEventValueSettings(Settings): +class MachineLearningInferenceEventsValueSettings(Settings): pass @@ -208,6 +210,10 @@ class CustomInsightsEventsSettings(Settings): pass +class MlInsightsEventsSettings(Settings): + pass + + class ProcessHostSettings(Settings): pass @@ -380,7 +386,7 @@ class EventHarvestConfigHarvestLimitSettings(Settings): _settings.application_logging.local_decorating = ApplicationLoggingLocalDecoratingSettings() _settings.application_logging.metrics = ApplicationLoggingMetricsSettings() _settings.machine_learning = MachineLearningSettings() -_settings.machine_learning.inference_event_value = MachineLearningInferenceEventValueSettings() +_settings.machine_learning.inference_events_value = MachineLearningInferenceEventsValueSettings() _settings.attributes = AttributesSettings() _settings.browser_monitoring = BrowserMonitorSettings() _settings.browser_monitoring.attributes = BrowserMonitorAttributesSettings() @@ -388,6 +394,7 @@ class EventHarvestConfigHarvestLimitSettings(Settings): _settings.console = ConsoleSettings() _settings.cross_application_tracer = CrossApplicationTracerSettings() _settings.custom_insights_events = CustomInsightsEventsSettings() +_settings.ml_insights_events = MlInsightsEventsSettings() _settings.datastore_tracer = DatastoreTracerSettings() _settings.datastore_tracer.database_name_reporting = DatastoreTracerDatabaseNameReportingSettings() _settings.datastore_tracer.instance_reporting = DatastoreTracerInstanceReportingSettings() @@ -679,6 +686,7 @@ def default_host(license_key): _settings.transaction_events.attributes.include = [] _settings.custom_insights_events.enabled = True +_settings.ml_insights_events.enabled = True _settings.distributed_tracing.enabled = _environ_as_bool("NEW_RELIC_DISTRIBUTED_TRACING_ENABLED", default=True) _settings.distributed_tracing.exclude_newrelic_header = False @@ -771,6 +779,10 @@ def default_host(license_key): "NEW_RELIC_CUSTOM_INSIGHTS_EVENTS_MAX_SAMPLES_STORED", CUSTOM_EVENT_RESERVOIR_SIZE ) +_settings.event_harvest_config.harvest_limits.ml_event_data = _environ_as_int( + "NEW_RELIC_ML_INSIGHTS_EVENTS_MAX_SAMPLES_STORED", ML_EVENT_RESERVOIR_SIZE +) + _settings.event_harvest_config.harvest_limits.span_event_data = _environ_as_int( "NEW_RELIC_SPAN_EVENTS_MAX_SAMPLES_STORED", SPAN_EVENT_RESERVOIR_SIZE ) @@ -850,7 +862,8 @@ def default_host(license_key): _settings.application_logging.local_decorating.enabled = _environ_as_bool( "NEW_RELIC_APPLICATION_LOGGING_LOCAL_DECORATING_ENABLED", default=False ) -_settings.machine_learning.inference_event_value.enabled = _environ_as_bool( +_settings.machine_learning.enabled = _environ_as_bool("NEW_RELIC_MACHINE_LEARNING_ENABLED", default=True) +_settings.machine_learning.inference_events_value.enabled = _environ_as_bool( "NEW_RELIC_MACHINE_LEARNING_INFERENCE_EVENT_VALUE_ENABLED", default=True ) @@ -1097,8 +1110,8 @@ def apply_server_side_settings(server_side_config=None, settings=_settings): apply_config_setting(settings_snapshot, name, value) # Overlay with global server side configuration settings. - # global server side configuration always takes precedence over the global - # server side configuration settings. + # global server side configuration always takes precedence over the local + # agent configuration settings. for name, value in server_side_config.items(): apply_config_setting(settings_snapshot, name, value) @@ -1115,6 +1128,16 @@ def apply_server_side_settings(server_side_config=None, settings=_settings): settings_snapshot, "event_harvest_config.harvest_limits.span_event_data", span_event_harvest_limit ) + # Since the server does not override this setting as it's an OTLP setting, + # we must override it here manually by converting it into a per harvest cycle + # value. + apply_config_setting( + settings_snapshot, + "event_harvest_config.harvest_limits.ml_event_data", + # override ml_events / (60s/5s) harvest + settings_snapshot.event_harvest_config.harvest_limits.ml_event_data / 12, + ) + # This will be removed at some future point # Special case for account_id which will be sent instead of # cross_process_id in the future diff --git a/newrelic/core/stats_engine.py b/newrelic/core/stats_engine.py index 04c870ae9..17b0d99c1 100644 --- a/newrelic/core/stats_engine.py +++ b/newrelic/core/stats_engine.py @@ -848,8 +848,7 @@ def record_ml_event(self, event): if not settings: return - # TODO Fix this with actual settings - if settings.collect_custom_events and settings.custom_insights_events.enabled: + if settings.ml_insights_events.enabled: self._ml_events.add(event) def record_custom_metric(self, name, value): @@ -1067,8 +1066,7 @@ def record_transaction(self, transaction): # Merge in machine learning events - # TODO Fix this with actual settings - if settings.collect_custom_events and settings.custom_insights_events.enabled: + if settings.ml_insights_events.enabled: self.ml_events.merge(transaction.ml_events) # Merge in span events @@ -1521,8 +1519,7 @@ def reset_custom_events(self): def reset_ml_events(self): if self.__settings is not None: - # TODO fix this with the actual setting - self._ml_events = SampledDataSet(8333) + self._ml_events = SampledDataSet(self.__settings.event_harvest_config.harvest_limits.ml_event_data) else: self._ml_events = SampledDataSet() diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index 8bf8a7d64..80521c735 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -62,6 +62,11 @@ def _nr_wrapper_method(wrapped, instance, args, kwargs): if transaction is None: return wrapped(*args, **kwargs) + settings = transaction.settings if transaction.settings is not None else global_settings() + + if settings and not settings.machine_learning.enabled: + return wrapped(*args, **kwargs) + wrapped_attr_name = "_nr_wrapped_%s" % method # If the method has already been wrapped do not wrap it again. This happens @@ -196,7 +201,7 @@ def create_label_event(transaction, _class, inference_id, instance, return_val): "value": str(value), } # Don't include the raw value when inference_event_value is disabled. - if settings and settings.machine_learning and settings.machine_learning.inference_event_value.enabled: + if settings and settings.machine_learning.inference_events_value.enabled: event["value"] = str(value) transaction.record_custom_event("ML Model Label Event", event) @@ -279,7 +284,6 @@ def create_feature_event(transaction, _class, inference_id, instance, args, kwar final_feature_names = _get_feature_column_names(user_provided_feature_names, data_set) np_casted_data_set = np.array(data_set) _calc_prediction_feature_stats(data_set, _class, final_feature_names) - for col_index, feature in enumerate(np_casted_data_set): for row_index, value in enumerate(feature): value_type = find_type_category(data_set, row_index, col_index) @@ -291,7 +295,7 @@ def create_feature_event(transaction, _class, inference_id, instance, args, kwar "type": value_type, } # Don't include the raw value when inference_event_value is disabled. - if settings and settings.machine_learning and settings.machine_learning.inference_event_value.enabled: + if settings and settings.machine_learning and settings.machine_learning.inference_events_value.enabled: event["value"] = str(value) transaction.record_custom_event("ML Model Feature Event", event) @@ -320,6 +324,11 @@ def wrap_metric_scorer(wrapped, instance, args, kwargs): if not transaction: return wrapped(*args, **kwargs) + settings = transaction.settings if transaction.settings is not None else global_settings() + + if settings and not settings.machine_learning.enabled: + return wrapped(*args, **kwargs) + score = wrapped(*args, **kwargs) y_true, y_pred, args, kwargs = _bind_scorer(*args, **kwargs) diff --git a/tests/agent_features/test_high_security_mode.py b/tests/agent_features/test_high_security_mode.py index e37caac43..51933682f 100644 --- a/tests/agent_features/test_high_security_mode.py +++ b/tests/agent_features/test_high_security_mode.py @@ -77,9 +77,10 @@ def test_hsm_configuration_default(): "transaction_tracer.record_sql": "raw", "strip_exception_messages.enabled": False, "custom_insights_events.enabled": True, + "ml_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": True, "application_logging.forwarding.enabled": True, - "machine_learning.inference_event_value.enabled": True, + "machine_learning.inference_events_value.enabled": True, }, { "high_security": False, @@ -87,9 +88,10 @@ def test_hsm_configuration_default(): "transaction_tracer.record_sql": "raw", "strip_exception_messages.enabled": False, "custom_insights_events.enabled": False, + "ml_insights_events.enabled": False, "message_tracer.segment_parameters_enabled": True, "application_logging.forwarding.enabled": True, - "machine_learning.inference_event_value.enabled": True, + "machine_learning.inference_events_value.enabled": True, }, { "high_security": False, @@ -97,9 +99,10 @@ def test_hsm_configuration_default(): "transaction_tracer.record_sql": "obfuscated", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": True, + "ml_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": False, "application_logging.forwarding.enabled": False, - "machine_learning.inference_event_value.enabled": False, + "machine_learning.inference_events_value.enabled": False, }, { "high_security": False, @@ -107,9 +110,10 @@ def test_hsm_configuration_default(): "transaction_tracer.record_sql": "off", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, + "ml_insights_events.enabled": False, "message_tracer.segment_parameters_enabled": False, "application_logging.forwarding.enabled": False, - "machine_learning.inference_event_value.enabled": False, + "machine_learning.inference_events_value.enabled": False, }, ] @@ -120,9 +124,10 @@ def test_hsm_configuration_default(): "transaction_tracer.record_sql": "raw", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": True, + "ml_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": True, "application_logging.forwarding.enabled": False, - "machine_learning.inference_event_value.enabled": False, + "machine_learning.inference_events_value.enabled": False, }, { "high_security": True, @@ -130,9 +135,10 @@ def test_hsm_configuration_default(): "transaction_tracer.record_sql": "raw", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": True, + "ml_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": True, "application_logging.forwarding.enabled": False, - "machine_learning.inference_event_value.enabled": False, + "machine_learning.inference_events_value.enabled": False, }, { "high_security": True, @@ -140,9 +146,10 @@ def test_hsm_configuration_default(): "transaction_tracer.record_sql": "raw", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": True, + "ml_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": True, "application_logging.forwarding.enabled": False, - "machine_learning.inference_event_value.enabled": False, + "machine_learning.inference_events_value.enabled": False, }, { "high_security": True, @@ -150,9 +157,10 @@ def test_hsm_configuration_default(): "transaction_tracer.record_sql": "raw", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": True, + "ml_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": True, "application_logging.forwarding.enabled": True, - "machine_learning.inference_event_value.enabled": True, + "machine_learning.inference_events_value.enabled": True, }, { "high_security": True, @@ -160,9 +168,10 @@ def test_hsm_configuration_default(): "transaction_tracer.record_sql": "obfuscated", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": True, + "ml_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": True, "application_logging.forwarding.enabled": True, - "machine_learning.inference_event_value.enabled": True, + "machine_learning.inference_events_value.enabled": True, }, { "high_security": True, @@ -170,9 +179,10 @@ def test_hsm_configuration_default(): "transaction_tracer.record_sql": "off", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": True, + "ml_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": False, "application_logging.forwarding.enabled": True, - "machine_learning.inference_event_value.enabled": True, + "machine_learning.inference_events_value.enabled": True, }, { "high_security": True, @@ -180,9 +190,10 @@ def test_hsm_configuration_default(): "transaction_tracer.record_sql": "raw", "strip_exception_messages.enabled": False, "custom_insights_events.enabled": False, + "ml_insights_events.enabled": False, "message_tracer.segment_parameters_enabled": False, "application_logging.forwarding.enabled": True, - "machine_learning.inference_event_value.enabled": True, + "machine_learning.inference_events_value.enabled": True, }, ] @@ -205,9 +216,10 @@ def test_local_config_file_override_hsm_disabled(settings): original_record_sql = settings.transaction_tracer.record_sql original_strip_messages = settings.strip_exception_messages.enabled original_custom_events = settings.custom_insights_events.enabled + original_ml_events = settings.ml_insights_events.enabled original_message_segment_params_enabled = settings.message_tracer.segment_parameters_enabled original_application_logging_forwarding_enabled = settings.application_logging.forwarding.enabled - original_machine_learning_inference_event_value_enabled = settings.machine_learning.inference_event_value.enabled + original_machine_learning_inference_event_value_enabled = settings.machine_learning.inference_events_value.enabled apply_local_high_security_mode_setting(settings) @@ -215,10 +227,11 @@ def test_local_config_file_override_hsm_disabled(settings): assert settings.transaction_tracer.record_sql == original_record_sql assert settings.strip_exception_messages.enabled == original_strip_messages assert settings.custom_insights_events.enabled == original_custom_events + assert settings.ml_insights_events.enabled == original_ml_events assert settings.message_tracer.segment_parameters_enabled == original_message_segment_params_enabled assert settings.application_logging.forwarding.enabled == original_application_logging_forwarding_enabled assert ( - settings.machine_learning.inference_event_value.enabled + settings.machine_learning.inference_events_value.enabled == original_machine_learning_inference_event_value_enabled ) @@ -231,9 +244,10 @@ def test_local_config_file_override_hsm_enabled(settings): assert settings.transaction_tracer.record_sql in ("off", "obfuscated") assert settings.strip_exception_messages.enabled assert settings.custom_insights_events.enabled is False + assert settings.ml_insights_events.enabled is False assert settings.message_tracer.segment_parameters_enabled is False assert settings.application_logging.forwarding.enabled is False - assert settings.machine_learning.inference_event_value.enabled is False + assert settings.machine_learning.inference_events_value.enabled is False _server_side_config_settings_hsm_disabled = [ @@ -244,8 +258,9 @@ def test_local_config_file_override_hsm_enabled(settings): "transaction_tracer.record_sql": "obfuscated", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, + "ml_insights_events.enabled": False, "application_logging.forwarding.enabled": False, - "machine_learning.inference_event_value.enabled": False, + "machine_learning.inference_events_value.enabled": False, }, { "agent_config": { @@ -253,8 +268,9 @@ def test_local_config_file_override_hsm_enabled(settings): "transaction_tracer.record_sql": "raw", "strip_exception_messages.enabled": False, "custom_insights_events.enabled": True, + "ml_insights_events.enabled": True, "application_logging.forwarding.enabled": True, - "machine_learning.inference_event_value.enabled": True, + "machine_learning.inference_events_value.enabled": True, }, }, ), @@ -265,8 +281,9 @@ def test_local_config_file_override_hsm_enabled(settings): "transaction_tracer.record_sql": "raw", "strip_exception_messages.enabled": False, "custom_insights_events.enabled": True, + "ml_insights_events.enabled": True, "application_logging.forwarding.enabled": True, - "machine_learning.inference_event_value.enabled": True, + "machine_learning.inference_events_value.enabled": True, }, { "agent_config": { @@ -274,8 +291,9 @@ def test_local_config_file_override_hsm_enabled(settings): "transaction_tracer.record_sql": "off", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, + "ml_insights_events.enabled": False, "application_logging.forwarding.enabled": False, - "machine_learning.inference_event_value.enabled": False, + "machine_learning.inference_events_value.enabled": False, }, }, ), @@ -289,8 +307,9 @@ def test_local_config_file_override_hsm_enabled(settings): "transaction_tracer.record_sql": "obfuscated", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, + "ml_insights_events.enabled": False, "application_logging.forwarding.enabled": False, - "machine_learning.inference_event_value.enabled": False, + "machine_learning.inference_events_value.enabled": False, }, { "high_security": True, @@ -298,15 +317,17 @@ def test_local_config_file_override_hsm_enabled(settings): "transaction_tracer.record_sql": "obfuscated", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, + "ml_insights_events.enabled": False, "application_logging.forwarding.enabled": False, - "machine_learning.inference_event_value.enabled": False, + "machine_learning.inference_events_value.enabled": False, "agent_config": { "capture_params": False, "transaction_tracer.record_sql": "obfuscated", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, + "ml_insights_events.enabled": False, "application_logging.forwarding.enabled": False, - "machine_learning.inference_event_value.enabled": False, + "machine_learning.inference_events_value.enabled": False, }, }, ), @@ -317,8 +338,9 @@ def test_local_config_file_override_hsm_enabled(settings): "transaction_tracer.record_sql": "obfuscated", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, + "ml_insights_events.enabled": False, "application_logging.forwarding.enabled": False, - "machine_learning.inference_event_value.enabled": False, + "machine_learning.inference_events_value.enabled": False, }, { "high_security": True, @@ -326,15 +348,17 @@ def test_local_config_file_override_hsm_enabled(settings): "transaction_tracer.record_sql": "obfuscated", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, + "ml_insights_events.enabled": False, "application_logging.forwarding.enabled": False, - "machine_learning.inference_event_value.enabled": False, + "machine_learning.inference_events_value.enabled": False, "agent_config": { "capture_params": True, "transaction_tracer.record_sql": "raw", "strip_exception_messages.enabled": False, "custom_insights_events.enabled": True, + "ml_insights_events.enabled": True, "application_logging.forwarding.enabled": True, - "machine_learning.inference_event_value.enabled": True, + "machine_learning.inference_events_value.enabled": True, }, }, ), @@ -354,8 +378,9 @@ def test_remote_config_fixups_hsm_disabled(local_settings, server_settings): original_record_sql = agent_config["transaction_tracer.record_sql"] original_strip_messages = agent_config["strip_exception_messages.enabled"] original_custom_events = agent_config["custom_insights_events.enabled"] + original_ml_events = agent_config["ml_insights_events.enabled"] original_log_forwarding = agent_config["application_logging.forwarding.enabled"] - original_machine_learning_events = agent_config["machine_learning.inference_event_value.enabled"] + original_machine_learning_events = agent_config["machine_learning.inference_events_value.enabled"] _settings = global_settings() settings = override_generic_settings(_settings, local_settings)(AgentProtocol._apply_high_security_mode_fixups)( @@ -370,8 +395,9 @@ def test_remote_config_fixups_hsm_disabled(local_settings, server_settings): assert agent_config["transaction_tracer.record_sql"] == original_record_sql assert agent_config["strip_exception_messages.enabled"] == original_strip_messages assert agent_config["custom_insights_events.enabled"] == original_custom_events + assert agent_config["ml_insights_events.enabled"] == original_ml_events assert agent_config["application_logging.forwarding.enabled"] == original_log_forwarding - assert agent_config["machine_learning.inference_event_value.enabled"] == original_machine_learning_events + assert agent_config["machine_learning.inference_events_value.enabled"] == original_machine_learning_events @pytest.mark.parametrize("local_settings,server_settings", _server_side_config_settings_hsm_enabled) @@ -393,15 +419,17 @@ def test_remote_config_fixups_hsm_enabled(local_settings, server_settings): assert "transaction_tracer.record_sql" not in settings assert "strip_exception_messages.enabled" not in settings assert "custom_insights_events.enabled" not in settings + assert "ml_insights_events.enabled" not in settings assert "application_logging.forwarding.enabled" not in settings - assert "machine_learning.inference_event_value.enabled" not in settings + assert "machine_learning.inference_events_value.enabled" not in settings assert "capture_params" not in agent_config assert "transaction_tracer.record_sql" not in agent_config assert "strip_exception_messages.enabled" not in agent_config assert "custom_insights_events.enabled" not in agent_config + assert "ml_insights_events.enabled" not in agent_config assert "application_logging.forwarding.enabled" not in agent_config - assert "machine_learning.inference_event_value.enabled" not in agent_config + assert "machine_learning.inference_events_value.enabled" not in agent_config def test_remote_config_hsm_fixups_server_side_disabled(): @@ -426,6 +454,7 @@ def test_remote_config_hsm_fixups_server_side_disabled(): "high_security": True, "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, + "ml_insights_events.enabled": False, } diff --git a/tests/agent_features/test_ml_events.py b/tests/agent_features/test_ml_events.py index 9e616887d..f0dcf33c6 100644 --- a/tests/agent_features/test_ml_events.py +++ b/tests/agent_features/test_ml_events.py @@ -16,6 +16,8 @@ import pytest from testing_support.fixtures import ( # function_not_called,; override_application_settings, + function_not_called, + override_application_settings, reset_core_stats_engine, ) from testing_support.validators.validate_ml_event_count import validate_ml_event_count @@ -101,21 +103,14 @@ def test_record_ml_event_outside_transaction_params_not_a_dict(): # Tests for ML Events configuration settings -# TODO Test config settings here -# @override_application_settings({'collect_ml_events': False}) -# @reset_core_stats_engine() -# @validate_ml_event_count(count=0) -# @background_task() -# def test_ml_event_settings_check_collector_flag(): -# record_ml_event('FooEvent', _user_params) +@override_application_settings({"ml_insights_events.enabled": False}) +@reset_core_stats_engine() +@validate_ml_event_count(count=0) +@background_task() +def test_ml_event_settings_check_ml_insights_enabled(): + record_ml_event("FooEvent", {"foo": "bar"}) -# @override_application_settings({'ml_insights_events.enabled': False}) -# @reset_core_stats_engine() -# @validate_ml_event_count(count=0) -# @background_task() -# def test_ml_event_settings_check_ml_insights_enabled(): -# record_ml_event('FooEvent', _user_params) # Test that record_ml_event() methods will short-circuit. # @@ -123,15 +118,17 @@ def test_record_ml_event_outside_transaction_params_not_a_dict(): # `create_ml_event()` function is not called, in order to avoid the # event_type and attribute processing. -# @override_application_settings({'ml_insights_events.enabled': False}) -# @function_not_called('newrelic.api.transaction', 'create_ml_event') -# @background_task() -# def test_transaction_create_ml_event_not_called(): -# record_ml_event('FooEvent', _user_params) - -# @override_application_settings({'ml_insights_events.enabled': False}) -# @function_not_called('newrelic.core.application', 'create_ml_event') -# @background_task() -# def test_application_create_ml_event_not_called(): -# app = application() -# record_ml_event('FooEvent', _user_params, application=app) + +@override_application_settings({"ml_insights_events.enabled": False}) +@function_not_called("newrelic.api.transaction", "create_custom_event") +@background_task() +def test_transaction_create_ml_event_not_called(): + record_ml_event("FooEvent", {"foo": "bar"}) + + +@override_application_settings({"ml_insights_events.enabled": False}) +@function_not_called("newrelic.core.application", "create_custom_event") +@background_task() +def test_application_create_ml_event_not_called(): + app = application() + record_ml_event("FooEvent", {"foo": "bar"}, application=app) diff --git a/tests/mlmodel_sklearn/test_inference_events.py b/tests/mlmodel_sklearn/test_inference_events.py index be38f36ce..873892e4e 100644 --- a/tests/mlmodel_sklearn/test_inference_events.py +++ b/tests/mlmodel_sklearn/test_inference_events.py @@ -401,7 +401,7 @@ def _test(): @reset_core_stats_engine() -@override_application_settings({"machine_learning.inference_event_value.enabled": False}) +@override_application_settings({"machine_learning.inference_events_value.enabled": False}) def test_does_not_include_value_when_inference_event_value_enabled_is_false(): @validate_custom_events(numpy_str_recorded_custom_events_no_value) @validate_custom_event_count(count=3) @@ -422,6 +422,53 @@ def _test(): _test() +@reset_core_stats_engine() +@override_application_settings({"custom_insights_events.enabled": False}) +def test_does_not_include_events_when_custom_insights_events_enabled_is_false(): + """ + Verifies that all ml events can be disabled by setting + custom_insights_events.enabled. + """ + + @validate_custom_event_count(count=0) + @background_task() + def _test(): + import sklearn.tree + + x_train = np.array([[20, 20], [21, 21]], dtype=" Date: Thu, 8 Jun 2023 13:17:28 -0700 Subject: [PATCH 33/54] Dimensional Metrics (#815) * Wiring dimensional metrics * Squashed commit of the following: commit c2d4629dfd7787354b6607160bb952913975d5f7 Author: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Wed May 10 15:59:13 2023 -0700 Add required option for tox v4 (#795) * Add required option for tox v4 * Update tox in GHA * Remove py27 no-cache-dir commit a9636498ab5c20c266fb044a08359c0c9bbcf826 Author: Hannah Stepanek Date: Tue May 9 10:46:39 2023 -0700 Run coverage around pytest (#813) * Run coverage around pytest * Trigger tests * Fixup * Add redis client_no_touch to ignore list * Temporarily remove kafka from coverage * Remove coverage for old libs commit 3d8284540e0acd867c2cf680f43449bc128c0779 Author: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Wed May 3 14:50:30 2023 -0700 Omit some frameworks from coverage analysis (#810) * Omit some frameworks from coverage analysis * Remove commas * Change format of omit * Add relative_files option to coverage * Add absolute directory * Add envsitepackagedir * Add coveragerc file * Add codecov.yml * [Mega-Linter] Apply linters fixes * Revert coveragerc file settings * Add files in packages and more frameworks * Remove commented line --------- Co-authored-by: lrafeei Co-authored-by: Hannah Stepanek commit fd0fa35466b630e34e8476cc53ad0e163564e2de Author: Uma Annamalai Date: Tue May 2 10:55:36 2023 -0700 Add testing for genshi and mako. (#799) * Add testing for genshi and mako. * [Mega-Linter] Apply linters fixes --------- Co-authored-by: umaannamalai Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> commit be4fb3dda0e734889acd6bc53cf91f26c18c2118 Author: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Mon May 1 16:01:09 2023 -0700 Add tests for Waitress (#797) * Change import format * Initial commit * Add more tests to adapter_waitress * Remove commented out code * [Mega-Linter] Apply linters fixes * Add assertions to all tests * Add more NR testing to waitress --------- Co-authored-by: lrafeei Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> commit 7103506ca5639d339e3e47dfb9e4affb546c839b Author: Hannah Stepanek Date: Mon May 1 14:12:31 2023 -0700 Add tests for pyodbc (#796) * Add tests for pyodbc * Move imports into tests to get import coverage * Fixup: remove time import * Trigger tests --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> commit 363122a0efe0ad9f4784fc1f67fda046cb9bb7e8 Author: Hannah Stepanek Date: Mon May 1 13:34:35 2023 -0700 Pin virtualenv, fix pip arg deprecation & disable kafka tests (#803) * Pin virtualenv * Fixup: use 20.21.1 instead * Replace install-options with config-settings See https://github.com/pypa/pip/issues/11358. * Temporarily disable kafka tests * Add dimensional stats table to stats engine * Add attribute processing to metric identity * Add testing for dimensional metrics * Cover tags as list not dict * Commit suggestions from code review --- newrelic/api/application.py | 8 + newrelic/api/transaction.py | 52 +++++- newrelic/common/metric_utils.py | 35 ++++ newrelic/core/agent.py | 27 +++ newrelic/core/application.py | 50 ++++++ newrelic/core/data_collector.py | 21 +++ newrelic/core/stats_engine.py | 163 +++++++++++++++++- newrelic/core/transaction_node.py | 1 + .../test_dimensional_metrics.py | 106 ++++++++++++ tests/agent_unittests/test_harvest_loop.py | 8 +- tests/testing_support/fixtures.py | 5 +- ...dimensional_metrics_outside_transaction.py | 93 ++++++++++ .../validate_transaction_metrics.py | 20 ++- 13 files changed, 583 insertions(+), 6 deletions(-) create mode 100644 newrelic/common/metric_utils.py create mode 100644 tests/agent_features/test_dimensional_metrics.py create mode 100644 tests/testing_support/validators/validate_dimensional_metrics_outside_transaction.py diff --git a/newrelic/api/application.py b/newrelic/api/application.py index 1ff425a7b..e2e7be139 100644 --- a/newrelic/api/application.py +++ b/newrelic/api/application.py @@ -142,6 +142,14 @@ def record_custom_metrics(self, metrics): if self.active and metrics: self._agent.record_custom_metrics(self._name, metrics) + def record_dimensional_metric(self, name, value, tags=None): + if self.active: + self._agent.record_dimensional_metric(self._name, name, value, tags) + + def record_dimensional_metrics(self, metrics): + if self.active and metrics: + self._agent.record_dimensional_metrics(self._name, metrics) + def record_custom_event(self, event_type, params): if self.active: self._agent.record_custom_event(self._name, event_type, params) diff --git a/newrelic/api/transaction.py b/newrelic/api/transaction.py index 4e2ddecb8..9afd49da1 100644 --- a/newrelic/api/transaction.py +++ b/newrelic/api/transaction.py @@ -68,7 +68,7 @@ from newrelic.core.custom_event import create_custom_event from newrelic.core.log_event_node import LogEventNode from newrelic.core.stack_trace import exception_stack -from newrelic.core.stats_engine import CustomMetrics, SampledDataSet +from newrelic.core.stats_engine import CustomMetrics, DimensionalMetrics, SampledDataSet from newrelic.core.thread_utilization import utilization_tracker from newrelic.core.trace_cache import ( TraceCacheActiveTraceError, @@ -309,6 +309,7 @@ def __init__(self, application, enabled=None, source=None): self.synthetics_header = None self._custom_metrics = CustomMetrics() + self._dimensional_metrics = DimensionalMetrics() global_settings = application.global_settings @@ -591,6 +592,7 @@ def __exit__(self, exc, value, tb): apdex_t=self.apdex, suppress_apdex=self.suppress_apdex, custom_metrics=self._custom_metrics, + dimensional_metrics=self._dimensional_metrics, guid=self.guid, cpu_time=self._cpu_user_time_value, suppress_transaction_trace=self.suppress_transaction_trace, @@ -1597,6 +1599,16 @@ def record_custom_metrics(self, metrics): for name, value in metrics: self._custom_metrics.record_custom_metric(name, value) + def record_dimensional_metric(self, name, value, tags=None): + self._dimensional_metrics.record_dimensional_metric(name, value, tags) + + def record_dimensional_metrics(self, metrics): + for metric in metrics: + name, value = metric[:2] + tags = metric[2] if len(metric) >= 3 else None + + self._dimensional_metrics.record_dimensional_metric(name, value, tags) + def record_custom_event(self, event_type, params): settings = self._settings @@ -1908,6 +1920,44 @@ def record_custom_metrics(metrics, application=None): application.record_custom_metrics(metrics) +def record_dimensional_metric(name, value, tags=None, application=None): + if application is None: + transaction = current_transaction() + if transaction: + transaction.record_dimensional_metric(name, value, tags) + else: + _logger.debug( + "record_dimensional_metric has been called but no " + "transaction was running. As a result, the following metric " + "has not been recorded. Name: %r Value: %r Tags: %r. To correct this " + "problem, supply an application object as a parameter to this " + "record_dimensional_metrics call.", + name, + value, + tags, + ) + elif application.enabled: + application.record_dimensional_metric(name, value, tags) + + +def record_dimensional_metrics(metrics, application=None): + if application is None: + transaction = current_transaction() + if transaction: + transaction.record_dimensional_metrics(metrics) + else: + _logger.debug( + "record_dimensional_metrics has been called but no " + "transaction was running. As a result, the following metrics " + "have not been recorded: %r. To correct this problem, " + "supply an application object as a parameter to this " + "record_dimensional_metric call.", + list(metrics), + ) + elif application.enabled: + application.record_dimensional_metrics(metrics) + + def record_custom_event(event_type, params, application=None): """Record a custom event. diff --git a/newrelic/common/metric_utils.py b/newrelic/common/metric_utils.py new file mode 100644 index 000000000..ebffe8332 --- /dev/null +++ b/newrelic/common/metric_utils.py @@ -0,0 +1,35 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module implements functions for creating a unique identity from a name and set of tags for use in dimensional metrics. +""" + +from newrelic.core.attribute import process_user_attribute + + +def create_metric_identity(name, tags=None): + if tags: + # Convert dicts to an iterable of tuples, other iterables should already be in this form + if isinstance(tags, dict): + tags = tags.items() + + # Apply attribute system sanitization. + # process_user_attribute returns (None, None) for results that fail sanitization. + # The filter removes these results from the iterable before creating the frozenset. + tags = frozenset(filter(lambda args: args[0] is not None, map(lambda args: process_user_attribute(*args), tags))) + + tags = tags or None # Set empty iterables after filtering to None + + return (name, tags) diff --git a/newrelic/core/agent.py b/newrelic/core/agent.py index 8aab80d7e..9d9aadab1 100644 --- a/newrelic/core/agent.py +++ b/newrelic/core/agent.py @@ -524,6 +524,33 @@ def record_custom_metrics(self, app_name, metrics): application.record_custom_metrics(metrics) + def record_dimensional_metric(self, app_name, name, value, tags=None): + """Records a basic metric for the named application. If there has + been no prior request to activate the application, the metric is + discarded. + + """ + + application = self._applications.get(app_name, None) + if application is None or not application.active: + return + + application.record_dimensional_metric(name, value, tags) + + def record_dimensional_metrics(self, app_name, metrics): + """Records the metrics for the named application. If there has + been no prior request to activate the application, the metric is + discarded. The metrics should be an iterable yielding tuples + consisting of the name and value. + + """ + + application = self._applications.get(app_name, None) + if application is None or not application.active: + return + + application.record_dimensional_metrics(metrics) + def record_custom_event(self, app_name, event_type, params): application = self._applications.get(app_name, None) if application is None or not application.active: diff --git a/newrelic/core/application.py b/newrelic/core/application.py index 2e7985d18..82cdf8a9a 100644 --- a/newrelic/core/application.py +++ b/newrelic/core/application.py @@ -510,6 +510,9 @@ def connect_to_data_collector(self, activate_agent): with self._stats_custom_lock: self._stats_custom_engine.reset_stats(configuration) + with self._stats_lock: + self._stats_engine.reset_stats(configuration) + # Record an initial start time for the reporting period and # clear record of last transaction processed. @@ -860,6 +863,50 @@ def record_custom_metrics(self, metrics): self._global_events_account += 1 self._stats_custom_engine.record_custom_metric(name, value) + def record_dimensional_metric(self, name, value, tags=None): + """Record a dimensional metric against the application independent + of a specific transaction. + + NOTE that this will require locking of the stats engine for + dimensional metrics and so under heavy use will have performance + issues. It is better to record the dimensional metric against an + active transaction as they will then be aggregated at the end of + the transaction when all other metrics are aggregated and so no + additional locking will be required. + + """ + + if not self._active_session: + return + + with self._stats_lock: + self._global_events_account += 1 + self._stats_engine.record_dimensional_metric(name, value, tags) + + def record_dimensional_metrics(self, metrics): + """Record a set of dimensional metrics against the application + independent of a specific transaction. + + NOTE that this will require locking of the stats engine for + dimensional metrics and so under heavy use will have performance + issues. It is better to record the dimensional metric against an + active transaction as they will then be aggregated at the end of + the transaction when all other metrics are aggregated and so no + additional locking will be required. + + """ + + if not self._active_session: + return + + with self._stats_lock: + for metric in metrics: + name, value = metric[:2] + tags = metric[2] if len(metric) >= 3 else None + + self._global_events_account += 1 + self._stats_engine.record_dimensional_metric(name, value, tags) + def record_custom_event(self, event_type, params): if not self._active_session: return @@ -1452,11 +1499,14 @@ def harvest(self, shutdown=False, flexible=False): _logger.debug("Normalizing metrics for harvest of %r.", self._app_name) metric_data = stats.metric_data(metric_normalizer) + dimensional_metric_data = stats.dimensional_metric_data(metric_normalizer) _logger.debug("Sending metric data for harvest of %r.", self._app_name) # Send metrics self._active_session.send_metric_data(self._period_start, period_end, metric_data) + if dimensional_metric_data: + self._active_session.send_dimensional_metric_data(self._period_start, period_end, dimensional_metric_data) _logger.debug("Done sending data for harvest of %r.", self._app_name) diff --git a/newrelic/core/data_collector.py b/newrelic/core/data_collector.py index 44539020e..e75368bee 100644 --- a/newrelic/core/data_collector.py +++ b/newrelic/core/data_collector.py @@ -31,6 +31,8 @@ _logger = logging.getLogger(__name__) +DIMENSIONAL_METRIC_DATA_TEMP = [] # TODO: REMOVE THIS + class Session(object): PROTOCOL = AgentProtocol @@ -135,6 +137,25 @@ def send_metric_data(self, start_time, end_time, metric_data): payload = (self.agent_run_id, start_time, end_time, metric_data) return self._protocol.send("metric_data", payload) + def send_dimensional_metric_data(self, start_time, end_time, metric_data): + """Called to submit dimensional metric data for specified period of time. + Time values are seconds since UNIX epoch as returned by the + time.time() function. The metric data should be iterable of + specific metrics. + + NOTE: This data is sent not sent to the normal agent endpoints but is sent + to the MELT API endpoints to keep the entity separate. This is for use + with the machine learning integration only. + """ + + payload = (self.agent_run_id, start_time, end_time, metric_data) + # return self._protocol.send("metric_data", payload) + + # TODO: REMOVE THIS. Replace with actual protocol. + DIMENSIONAL_METRIC_DATA_TEMP.append(payload) + _logger.debug("Dimensional Metrics: %r" % metric_data) + return 200 + def send_log_events(self, sampling_info, log_event_data): """Called to submit sample set for log events.""" diff --git a/newrelic/core/stats_engine.py b/newrelic/core/stats_engine.py index 17b0d99c1..9d59efd49 100644 --- a/newrelic/core/stats_engine.py +++ b/newrelic/core/stats_engine.py @@ -35,6 +35,7 @@ from newrelic.api.settings import STRIP_EXCEPTION_MESSAGE from newrelic.api.time_trace import get_linking_metadata from newrelic.common.encoding_utils import json_encode +from newrelic.common.metric_utils import create_metric_identity from newrelic.common.object_names import parse_exc_info from newrelic.common.streaming_utils import StreamBuffer from newrelic.core.attribute import ( @@ -180,6 +181,11 @@ def merge_custom_metric(self, value): self.merge_raw_time_metric(value) + def merge_dimensional_metric(self, value): + """Merge data value.""" + + self.merge_raw_time_metric(value) + class CountStats(TimeStats): def merge_stats(self, other): @@ -234,6 +240,35 @@ def reset_metric_stats(self): """ self.__stats_table = {} +class DimensionalMetrics(CustomMetrics): + + """Extends CustomMetrics to allow a set of tags for metrics.""" + + def __contains__(self, key): + if not isinstance(key[1], frozenset): + # Convert tags dict to a frozen set for proper comparisons + key = create_metric_identity(*key) + return key in self.__stats_table + + def record_dimensional_metric(self, name, value, tags=None): + """Record a single value metric, merging the data with any data + from prior value metrics with the same name. + + """ + + key = create_metric_identity(name, tags) + self.record_custom_metric(key, value) + +class DimensionalStatsTable(dict): + + """Extends dict to coerce a set of tags to a hashable identity.""" + + def __contains__(self, key): + if key[1] is not None and not isinstance(key[1], frozenset): + # Convert tags dict to a frozen set for proper comparisons + key = create_metric_identity(*key) + return super(DimensionalStatsTable, self).__contains__(key) + class SlowSqlStats(list): def __init__(self): @@ -433,6 +468,7 @@ class StatsEngine(object): def __init__(self): self.__settings = None self.__stats_table = {} + self.__dimensional_stats_table = DimensionalStatsTable() self._transaction_events = SampledDataSet() self._error_events = SampledDataSet() self._custom_events = SampledDataSet() @@ -457,6 +493,10 @@ def settings(self): def stats_table(self): return self.__stats_table + @property + def dimensional_stats_table(self): + return self.__dimensional_stats_table + @property def transaction_events(self): return self._transaction_events @@ -499,7 +539,7 @@ def metrics_count(self): """ - return len(self.__stats_table) + return len(self.__stats_table) + len(self.__dimensional_stats_table) def record_apdex_metric(self, metric): """Record a single apdex metric, merging the data with any data @@ -887,6 +927,44 @@ def record_custom_metrics(self, metrics): for name, value in metrics: self.record_custom_metric(name, value) + def record_dimensional_metric(self, name, value, tags=None): + """Record a single value metric, merging the data with any data + from prior value metrics with the same name. + + """ + if isinstance(value, dict): + if len(value) == 1 and "count" in value: + new_stats = CountStats(call_count=value["count"]) + else: + new_stats = TimeStats(*c2t(**value)) + else: + new_stats = TimeStats(1, value, value, value, value, value**2) + + key = create_metric_identity(name, tags) + stats = self.__dimensional_stats_table.get(key) + if stats is None: + self.__dimensional_stats_table[key] = new_stats + else: + stats.merge_stats(new_stats) + + return key + + def record_dimensional_metrics(self, metrics): + """Record the value metrics supplied by the iterable, merging + the data with any data from prior value metrics with the same + name. + + """ + + if not self.__settings: + return + + for metric in metrics: + name, value = metric[:2] + tags = metric[2] if len(metric) >= 3 else None + + self.record_dimensional_metric(name, value, tags) + def record_slow_sql_node(self, node): """Record a single sql metric, merging the data with any data from prior sql metrics for the same sql key. @@ -997,6 +1075,8 @@ def record_transaction(self, transaction): self.merge_custom_metrics(transaction.custom_metrics.metrics()) + self.merge_dimensional_metrics(transaction.dimensional_metrics.metrics()) + self.record_time_metrics(transaction.time_metrics(self)) # Capture any errors if error collection is enabled. @@ -1186,6 +1266,67 @@ def metric_data_count(self): return len(self.__stats_table) + def dimensional_metric_data(self, normalizer=None): + """Returns a list containing the low level metric data for + sending to the core application pertaining to the reporting + period. This consists of tuple pairs where first is dictionary + with name and scope keys with corresponding values, or integer + identifier if metric had an entry in dictionary mapping metric + (name, tags) as supplied from core application. The second is + the list of accumulated metric data, the list always being of + length 6. + + """ + + if not self.__settings: + return [] + + result = [] + normalized_stats = {} + + # Metric Renaming and Re-Aggregation. After applying the metric + # renaming rules, the metrics are re-aggregated to collapse the + # metrics with same names after the renaming. + + if self.__settings.debug.log_raw_metric_data: + _logger.info( + "Raw dimensional metric data for harvest of %r is %r.", + self.__settings.app_name, + list(six.iteritems(self.__dimensional_stats_table)), + ) + + if normalizer is not None: + for key, value in six.iteritems(self.__dimensional_stats_table): + key = (normalizer(key[0])[0], key[1]) + stats = normalized_stats.get(key) + if stats is None: + normalized_stats[key] = copy.copy(value) + else: + stats.merge_stats(value) + else: + normalized_stats = self.__dimensional_stats_table + + if self.__settings.debug.log_normalized_metric_data: + _logger.info( + "Normalized metric data for harvest of %r is %r.", + self.__settings.app_name, + list(six.iteritems(normalized_stats)), + ) + + for key, value in six.iteritems(normalized_stats): + key = dict(name=key[0], scope=key[1]) + result.append((key, value)) + + return result + + def dimensional_metric_data_count(self): + """Returns a count of the number of unique metrics.""" + + if not self.__settings: + return 0 + + return len(self.__dimensional_stats_table) + def error_data(self): """Returns a to a list containing any errors collected during the reporting period. @@ -1464,6 +1605,7 @@ def reset_stats(self, settings, reset_stream=False): self.__settings = settings self.__stats_table = {} + self.__dimensional_stats_table = {} self.__sql_stats_table = {} self.__slow_transaction = None self.__slow_transaction_map = {} @@ -1491,6 +1633,7 @@ def reset_metric_stats(self): """ self.__stats_table = {} + self.__dimensional_stats_table = {} def reset_transaction_events(self): """Resets the accumulated statistics back to initial state for @@ -1827,6 +1970,24 @@ def merge_custom_metrics(self, metrics): else: stats.merge_stats(other) + def merge_dimensional_metrics(self, metrics): + """ + Merges in a set of dimensional metrics. The metrics should be + provide as an iterable where each item is a tuple of the metric + key and the accumulated stats for the metric. The metric key should + also be a tuple, containing a name and attribute filtered frozenset of tags. + """ + + if not self.__settings: + return + + for key, other in metrics: + stats = self.__dimensional_stats_table.get(key) + if not stats: + self.__dimensional_stats_table[key] = other + else: + stats.merge_stats(other) + def _snapshot(self): copy = object.__new__(StatsEngineSnapshot) copy.__dict__.update(self.__dict__) diff --git a/newrelic/core/transaction_node.py b/newrelic/core/transaction_node.py index 056d45a48..d63d7f9b6 100644 --- a/newrelic/core/transaction_node.py +++ b/newrelic/core/transaction_node.py @@ -65,6 +65,7 @@ "apdex_t", "suppress_apdex", "custom_metrics", + "dimensional_metrics", "guid", "cpu_time", "suppress_transaction_trace", diff --git a/tests/agent_features/test_dimensional_metrics.py b/tests/agent_features/test_dimensional_metrics.py new file mode 100644 index 000000000..82ddfad89 --- /dev/null +++ b/tests/agent_features/test_dimensional_metrics.py @@ -0,0 +1,106 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from newrelic.api.application import application_instance +from newrelic.api.background_task import background_task +from newrelic.api.transaction import record_dimensional_metric, record_dimensional_metrics +from newrelic.common.metric_utils import create_metric_identity + +from testing_support.fixtures import reset_core_stats_engine +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics +from testing_support.validators.validate_dimensional_metrics_outside_transaction import validate_dimensional_metrics_outside_transaction + + +_test_tags_examples = [ + (None, None), + ({}, None), + ([], None), + ({"str": "a"}, frozenset({("str", "a")})), + ({"int": 1}, frozenset({("int", 1)})), + ({"float": 1.0}, frozenset({("float", 1.0)})), + ({"bool": True}, frozenset({("bool", True)})), + ({"list": [1]}, frozenset({("list", "[1]")})), + ({"dict": {"subtag": 1}}, frozenset({("dict", "{'subtag': 1}")})), + ([("tags-as-list", 1)], frozenset({("tags-as-list", 1)})), +] + + +@pytest.mark.parametrize("tags,expected", _test_tags_examples) +def test_create_metric_identity(tags, expected): + name = "Metric" + output_name, output_tags = create_metric_identity(name, tags=tags) + assert output_name == name, "Name does not match." + assert output_tags == expected, "Output tags do not match." + + +@pytest.mark.parametrize("tags,expected", _test_tags_examples) +def test_record_dimensional_metric_inside_transaction(tags, expected): + @validate_transaction_metrics("test_record_dimensional_metric_inside_transaction", background_task=True, dimensional_metrics=[ + ("Metric", expected, 1), + ]) + @background_task(name="test_record_dimensional_metric_inside_transaction") + def _test(): + record_dimensional_metric("Metric", 1, tags=tags) + + _test() + + +@pytest.mark.parametrize("tags,expected", _test_tags_examples) +@reset_core_stats_engine() +def test_record_dimensional_metric_outside_transaction(tags, expected): + @validate_dimensional_metrics_outside_transaction([("Metric", expected, 1)]) + def _test(): + app = application_instance() + record_dimensional_metric("Metric", 1, tags=tags, application=app) + + _test() + + +@pytest.mark.parametrize("tags,expected", _test_tags_examples) +def test_record_dimensional_metrics_inside_transaction(tags, expected): + @validate_transaction_metrics("test_record_dimensional_metrics_inside_transaction", background_task=True, dimensional_metrics=[("Metric/1", expected, 1), ("Metric/2", expected, 1)]) + @background_task(name="test_record_dimensional_metrics_inside_transaction") + def _test(): + record_dimensional_metrics([("Metric/1", 1, tags), ("Metric/2", 1, tags)]) + + _test() + + +@pytest.mark.parametrize("tags,expected", _test_tags_examples) +@reset_core_stats_engine() +def test_record_dimensional_metrics_outside_transaction(tags, expected): + @validate_dimensional_metrics_outside_transaction([("Metric/1", expected, 1), ("Metric/2", expected, 1)]) + def _test(): + app = application_instance() + record_dimensional_metrics([("Metric/1", 1, tags), ("Metric/2", 1, tags)], application=app) + + _test() + + +def test_dimensional_metrics_different_tags(): + @validate_transaction_metrics("test_dimensional_metrics_different_tags", background_task=True, dimensional_metrics=[ + ("Metric", frozenset({("tag", 1)}), 1), + ("Metric", frozenset({("tag", 2)}), 2), + ]) + @background_task(name="test_dimensional_metrics_different_tags") + def _test(): + record_dimensional_metrics([ + ("Metric", 1, {"tag": 1}), + ("Metric", 1, {"tag": 2}), + ]) + record_dimensional_metric("Metric", 1, {"tag": 2}) + + _test() diff --git a/tests/agent_unittests/test_harvest_loop.py b/tests/agent_unittests/test_harvest_loop.py index 5f14b270c..15b67a81e 100644 --- a/tests/agent_unittests/test_harvest_loop.py +++ b/tests/agent_unittests/test_harvest_loop.py @@ -32,7 +32,7 @@ from newrelic.core.function_node import FunctionNode from newrelic.core.log_event_node import LogEventNode from newrelic.core.root_node import RootNode -from newrelic.core.stats_engine import CustomMetrics, SampledDataSet +from newrelic.core.stats_engine import CustomMetrics, SampledDataSet, DimensionalMetrics from newrelic.core.transaction_node import TransactionNode from newrelic.network.exceptions import RetryDataForRequest @@ -132,6 +132,7 @@ def transaction_node(request): apdex_t=0.5, suppress_apdex=False, custom_metrics=CustomMetrics(), + dimensional_metrics=DimensionalMetrics(), guid="4485b89db608aece", cpu_time=0.0, suppress_transaction_trace=False, @@ -824,6 +825,7 @@ def test_flexible_events_harvested(allowlist_event): app._stats_engine.log_events.add(LogEventNode(1653609717, "WARNING", "A", {})) app._stats_engine.span_events.add("span event") app._stats_engine.record_custom_metric("CustomMetric/Int", 1) + app._stats_engine.record_dimensional_metric("DimensionalMetric/Int", 1, tags={"tag": "tag"}) assert app._stats_engine.transaction_events.num_seen == 1 assert app._stats_engine.error_events.num_seen == 1 @@ -831,6 +833,7 @@ def test_flexible_events_harvested(allowlist_event): assert app._stats_engine.log_events.num_seen == 1 assert app._stats_engine.span_events.num_seen == 1 assert app._stats_engine.record_custom_metric("CustomMetric/Int", 1) + assert app._stats_engine.record_dimensional_metric("DimensionalMetric/Int", 1, tags={"tag": "tag"}) app.harvest(flexible=True) @@ -850,7 +853,8 @@ def test_flexible_events_harvested(allowlist_event): assert app._stats_engine.span_events.num_seen == num_seen assert ("CustomMetric/Int", "") in app._stats_engine.stats_table - assert app._stats_engine.metrics_count() > 1 + assert ("DimensionalMetric/Int", frozenset({("tag", "tag")})) in app._stats_engine.dimensional_stats_table + assert app._stats_engine.metrics_count() > 3 @pytest.mark.parametrize( diff --git a/tests/testing_support/fixtures.py b/tests/testing_support/fixtures.py index 07de22cf0..bf0e80a67 100644 --- a/tests/testing_support/fixtures.py +++ b/tests/testing_support/fixtures.py @@ -176,7 +176,10 @@ def wrap_shutdown_agent(wrapped, instance, args, kwargs): def wrap_record_custom_metric(wrapped, instance, args, kwargs): def _bind_params(name, value, *args, **kwargs): - return name + if isinstance(name, tuple): + return name[0] + else: + return name metric_name = _bind_params(*args, **kwargs) if ( diff --git a/tests/testing_support/validators/validate_dimensional_metrics_outside_transaction.py b/tests/testing_support/validators/validate_dimensional_metrics_outside_transaction.py new file mode 100644 index 000000000..7a3272bad --- /dev/null +++ b/tests/testing_support/validators/validate_dimensional_metrics_outside_transaction.py @@ -0,0 +1,93 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy + +from testing_support.fixtures import catch_background_exceptions +from newrelic.common.object_wrapper import transient_function_wrapper, function_wrapper + + +def validate_dimensional_metrics_outside_transaction(dimensional_metrics=None): + dimensional_metrics = dimensional_metrics or [] + + @function_wrapper + def _validate_wrapper(wrapped, instance, args, kwargs): + + record_dimensional_metric_called = [] + recorded_metrics = [None] + + @transient_function_wrapper("newrelic.core.stats_engine", "StatsEngine.record_dimensional_metric") + @catch_background_exceptions + def _validate_dimensional_metrics_outside_transaction(wrapped, instance, args, kwargs): + record_dimensional_metric_called.append(True) + try: + result = wrapped(*args, **kwargs) + except: + raise + else: + metrics = instance.dimensional_stats_table + # Record a copy of the metric value so that the values aren't + # merged in the future + _metrics = {} + for k, v in metrics.items(): + _metrics[k] = copy.copy(v) + recorded_metrics[0] = _metrics + + return result + + def _validate(metrics, name, tags, count): + key = (name, tags) + metric = metrics.get(key) + + def _metrics_table(): + out = [""] + out.append("Expected: {0}: {1}".format(key, count)) + for metric_key, metric_value in metrics.items(): + out.append("{0}: {1}".format(metric_key, metric_value[0])) + return "\n".join(out) + + def _metric_details(): + return "metric=%r, count=%r" % (key, metric.call_count) + + if count is not None: + assert metric is not None, _metrics_table() + if count == "present": + assert metric.call_count > 0, _metric_details() + else: + assert metric.call_count == count, _metric_details() + + assert metric.total_call_time >= 0, (key, metric) + assert metric.total_exclusive_call_time >= 0, (key, metric) + assert metric.min_call_time >= 0, (key, metric) + assert metric.sum_of_squares >= 0, (key, metric) + + else: + assert metric is None, _metrics_table() + + _new_wrapper = _validate_dimensional_metrics_outside_transaction(wrapped) + val = _new_wrapper(*args, **kwargs) + assert record_dimensional_metric_called + metrics = recorded_metrics[0] + + record_dimensional_metric_called[:] = [] + recorded_metrics[:] = [] + + for dimensional_metric, dimensional_tags, count in dimensional_metrics: + if isinstance(dimensional_tags, dict): + dimensional_tags = frozenset(dimensional_tags.items()) + _validate(metrics, dimensional_metric, dimensional_tags, count) + + return val + + return _validate_wrapper diff --git a/tests/testing_support/validators/validate_transaction_metrics.py b/tests/testing_support/validators/validate_transaction_metrics.py index 7122b009a..63c5b3551 100644 --- a/tests/testing_support/validators/validate_transaction_metrics.py +++ b/tests/testing_support/validators/validate_transaction_metrics.py @@ -27,11 +27,13 @@ def validate_transaction_metrics( scoped_metrics=None, rollup_metrics=None, custom_metrics=None, + dimensional_metrics=None, index=-1, ): scoped_metrics = scoped_metrics or [] rollup_metrics = rollup_metrics or [] custom_metrics = custom_metrics or [] + dimensional_metrics = dimensional_metrics or [] if background_task: unscoped_metrics = [ @@ -56,6 +58,7 @@ def _validate_wrapper(wrapped, instance, args, kwargs): record_transaction_called = [] recorded_metrics = [] + recorded_dimensional_metrics = [] @transient_function_wrapper("newrelic.core.stats_engine", "StatsEngine.record_transaction") @catch_background_exceptions @@ -74,6 +77,14 @@ def _validate_transaction_metrics(wrapped, instance, args, kwargs): _metrics[k] = copy.copy(v) recorded_metrics.append(_metrics) + metrics = instance.dimensional_stats_table + # Record a copy of the metric value so that the values aren't + # merged in the future + _metrics = {} + for k, v in metrics.items(): + _metrics[k] = copy.copy(v) + recorded_dimensional_metrics.append(_metrics) + return result def _validate(metrics, name, scope, count): @@ -109,9 +120,11 @@ def _metric_details(): val = _new_wrapper(*args, **kwargs) assert record_transaction_called metrics = recorded_metrics[index] + captured_dimensional_metrics = recorded_dimensional_metrics[index] record_transaction_called[:] = [] recorded_metrics[:] = [] + recorded_dimensional_metrics[:] = [] for unscoped_metric in unscoped_metrics: _validate(metrics, unscoped_metric, "", 1) @@ -125,6 +138,11 @@ def _metric_details(): for custom_name, custom_count in custom_metrics: _validate(metrics, custom_name, "", custom_count) + for dimensional_name, dimensional_tags, dimensional_count in dimensional_metrics: + if isinstance(dimensional_tags, dict): + dimensional_tags = frozenset(dimensional_tags.items()) + _validate(captured_dimensional_metrics, dimensional_name, dimensional_tags, dimensional_count) + custom_metric_names = {name for name, _ in custom_metrics} for name, _ in metrics: if name not in custom_metric_names: @@ -132,4 +150,4 @@ def _metric_details(): return val - return _validate_wrapper \ No newline at end of file + return _validate_wrapper From 30f0bf5ce27f239f70b236c639a49715f33ce948 Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Fri, 9 Jun 2023 16:12:09 -0700 Subject: [PATCH 34/54] Add OTLP protocol class & protos (#821) * Add protos under packages for otlp * Add common otlp proto payload methods * Add new oltp protocol class * Remove ML event from log message * Remove params, add api-key header & expose path The params are not relevant to OTLP so remove these. The api-key header is how we provide the license key to OTLP so add this. The path to upload dimensional metrics and events are different in OTLP so expose the path so it can be overriden inside the coresponding data_collector methods. * Add otlp_port and otlp_host settings * Default to JSON if protobuf not available & warn * Move otlp_utils to core * Call encode in protocol class * Patch issues with data collector * Move resource to utils & add log proto imports --------- Co-authored-by: Tim Pansino --- MANIFEST.in | 1 + newrelic/common/agent_http.py | 13 +- newrelic/config.py | 2 + newrelic/core/agent_protocol.py | 93 +++++++- newrelic/core/config.py | 31 +++ newrelic/core/data_collector.py | 10 +- newrelic/core/otlp_utils.py | 107 +++++++++ .../packages/opentelemetry_proto/LICENSE.txt | 201 ++++++++++++++++ .../packages/opentelemetry_proto/__init__.py | 0 .../opentelemetry_proto/common_pb2.py | 87 +++++++ .../packages/opentelemetry_proto/logs_pb2.py | 117 ++++++++++ .../opentelemetry_proto/metrics_pb2.py | 217 ++++++++++++++++++ .../opentelemetry_proto/resource_pb2.py | 36 +++ setup.py | 1 + tests/agent_features/test_configuration.py | 2 + .../test_utilization_settings.py | 16 ++ tox.ini | 1 + 17 files changed, 926 insertions(+), 9 deletions(-) create mode 100644 newrelic/core/otlp_utils.py create mode 100644 newrelic/packages/opentelemetry_proto/LICENSE.txt create mode 100644 newrelic/packages/opentelemetry_proto/__init__.py create mode 100644 newrelic/packages/opentelemetry_proto/common_pb2.py create mode 100644 newrelic/packages/opentelemetry_proto/logs_pb2.py create mode 100644 newrelic/packages/opentelemetry_proto/metrics_pb2.py create mode 100644 newrelic/packages/opentelemetry_proto/resource_pb2.py diff --git a/MANIFEST.in b/MANIFEST.in index 0a75ce752..ed6efde88 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,3 +8,4 @@ include newrelic/common/cacert.pem include newrelic/packages/wrapt/LICENSE include newrelic/packages/wrapt/README include newrelic/packages/urllib3/LICENSE.txt +include newrelic/packages/opentelemetry_proto/LICENSE.txt diff --git a/newrelic/common/agent_http.py b/newrelic/common/agent_http.py index e9d9a00aa..555816796 100644 --- a/newrelic/common/agent_http.py +++ b/newrelic/common/agent_http.py @@ -92,6 +92,7 @@ def __init__( compression_method="gzip", max_payload_size_in_bytes=1000000, audit_log_fp=None, + default_content_encoding_header="Identity", ): self._audit_log_fp = audit_log_fp @@ -240,6 +241,7 @@ def __init__( compression_method="gzip", max_payload_size_in_bytes=1000000, audit_log_fp=None, + default_content_encoding_header="Identity", ): self._host = host port = self._port = port @@ -248,6 +250,7 @@ def __init__( self._compression_method = compression_method self._max_payload_size_in_bytes = max_payload_size_in_bytes self._audit_log_fp = audit_log_fp + self._default_content_encoding_header = default_content_encoding_header self._prefix = "" @@ -419,11 +422,9 @@ def send_request( method=self._compression_method, level=self._compression_level, ) - content_encoding = self._compression_method - else: - content_encoding = "Identity" - - merged_headers["Content-Encoding"] = content_encoding + merged_headers["Content-Encoding"] = self._compression_method + elif self._default_content_encoding_header: + merged_headers["Content-Encoding"] = self._default_content_encoding_header request_id = self.log_request( self._audit_log_fp, @@ -489,6 +490,7 @@ def __init__( compression_method="gzip", max_payload_size_in_bytes=1000000, audit_log_fp=None, + default_content_encoding_header="Identity", ): proxy = self._parse_proxy(proxy_scheme, proxy_host, None, None, None) if proxy and proxy.scheme == "https": @@ -515,6 +517,7 @@ def __init__( compression_method, max_payload_size_in_bytes, audit_log_fp, + default_content_encoding_header, ) diff --git a/newrelic/config.py b/newrelic/config.py index 7c3fa2279..5602efb3c 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -320,6 +320,8 @@ def _process_configuration(section): _process_setting(section, "api_key", "get", None) _process_setting(section, "host", "get", None) _process_setting(section, "port", "getint", None) + _process_setting(section, "otlp_host", "get", None) + _process_setting(section, "otlp_port", "getint", None) _process_setting(section, "ssl", "getboolean", None) _process_setting(section, "proxy_scheme", "get", None) _process_setting(section, "proxy_host", "get", None) diff --git a/newrelic/core/agent_protocol.py b/newrelic/core/agent_protocol.py index 8b64aed4d..b661fd0ca 100644 --- a/newrelic/core/agent_protocol.py +++ b/newrelic/core/agent_protocol.py @@ -38,6 +38,7 @@ global_settings_dump, ) from newrelic.core.internal_metrics import internal_count_metric +from newrelic.core.otlp_utils import OTLP_CONTENT_TYPE, otlp_encode from newrelic.network.exceptions import ( DiscardDataForRequest, ForceAgentDisconnect, @@ -217,11 +218,16 @@ def __exit__(self, exc, value, tb): def close_connection(self): self.client.close_connection() - def send(self, method, payload=()): + def send( + self, + method, + payload=(), + path="/agent_listener/invoke_raw_method", + ): params, headers, payload = self._to_http(method, payload) try: - response = self.client.send_request(params=params, headers=headers, payload=payload) + response = self.client.send_request(path=path, params=params, headers=headers, payload=payload) except NetworkInterfaceException: # All HTTP errors are currently retried raise RetryDataForRequest @@ -253,7 +259,10 @@ def send(self, method, payload=()): exception = self.STATUS_CODE_RESPONSE.get(status, DiscardDataForRequest) raise exception if status == 200: - return json_decode(data.decode("utf-8"))["return_value"] + return self.decode_response(data) + + def decode_response(self, response): + return json_decode(response.decode("utf-8"))["return_value"] def _to_http(self, method, payload=()): params = dict(self._params) @@ -516,3 +525,81 @@ def connect( # can be modified later settings.aws_lambda_metadata = aws_lambda_metadata return cls(settings, client_cls=client_cls) + + +class OtlpProtocol(AgentProtocol): + def __init__(self, settings, host=None, client_cls=ApplicationModeClient): + if settings.audit_log_file: + audit_log_fp = open(settings.audit_log_file, "a") + else: + audit_log_fp = None + + self.client = client_cls( + host=host or settings.otlp_host, + port=settings.otlp_port or 4318, + proxy_scheme=settings.proxy_scheme, + proxy_host=settings.proxy_host, + proxy_port=settings.proxy_port, + proxy_user=settings.proxy_user, + proxy_pass=settings.proxy_pass, + timeout=settings.agent_limits.data_collector_timeout, + ca_bundle_path=settings.ca_bundle_path, + disable_certificate_validation=settings.debug.disable_certificate_validation, + compression_threshold=settings.agent_limits.data_compression_threshold, + compression_level=settings.agent_limits.data_compression_level, + compression_method=settings.compressed_content_encoding, + max_payload_size_in_bytes=1000000, + audit_log_fp=audit_log_fp, + default_content_encoding_header=None, + ) + + self._params = {} + self._headers = { + "api-key": settings.license_key, + } + + # In Python 2, the JSON is loaded with unicode keys and values; + # however, the header name must be a non-unicode value when given to + # the HTTP library. This code converts the header name from unicode to + # non-unicode. + if settings.request_headers_map: + for k, v in settings.request_headers_map.items(): + if not isinstance(k, str): + k = k.encode("utf-8") + self._headers[k] = v + + # Content-Type should be protobuf, but falls back to JSON if protobuf is not installed. + self._headers["Content-Type"] = OTLP_CONTENT_TYPE + self._run_token = settings.agent_run_id + + # Logging + self._proxy_host = settings.proxy_host + self._proxy_port = settings.proxy_port + self._proxy_user = settings.proxy_user + + # Do not access configuration anywhere inside the class + self.configuration = settings + + @classmethod + def connect( + cls, + app_name, + linked_applications, + environment, + settings, + client_cls=ApplicationModeClient, + ): + with cls(settings, client_cls=client_cls) as protocol: + pass + + return protocol + + def _to_http(self, method, payload=()): + params = dict(self._params) + params["method"] = method + if self._run_token: + params["run_id"] = self._run_token + return params, self._headers, otlp_encode(payload) + + def decode_response(self, response): + return response.decode("utf-8") diff --git a/newrelic/core/config.py b/newrelic/core/config.py index ccd9a6132..55b359174 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -104,6 +104,7 @@ def create_settings(nested): class TopLevelSettings(Settings): _host = None + _otlp_host = None @property def host(self): @@ -115,6 +116,16 @@ def host(self): def host(self, value): self._host = value + @property + def otlp_host(self): + if self._otlp_host: + return self._otlp_host + return default_otlp_host(self.host) + + @otlp_host.setter + def otlp_host(self, value): + self._otlp_host = value + class AttributesSettings(Settings): pass @@ -560,6 +571,24 @@ def default_host(license_key): return host +def default_otlp_host(host): + HOST_MAP = { + "collector.newrelic.com": "otlp.nr-data.net", + "collector.eu.newrelic.com": "otlp.eu01.nr-data.net", + "gov-collector.newrelic.com": "gov-otlp.nr-data.net", + "staging-collector.newrelic.com": "staging-otlp.nr-data.net", + "staging-collector.eu.newrelic.com": "staging-otlp.eu01.nr-data.net", + "staging-gov-collector.newrelic.com": "staging-gov-otlp.nr-data.net", + "fake-collector.newrelic.com": "fake-otlp.nr-data.net", + } + otlp_host = HOST_MAP.get(host, None) + if not otlp_host: + default = HOST_MAP["collector.newrelic.com"] + _logger.warn("Unable to find corresponding OTLP host using default %s" % default) + otlp_host = default + return otlp_host + + _LOG_LEVEL = { "CRITICAL": logging.CRITICAL, "ERROR": logging.ERROR, @@ -585,7 +614,9 @@ def default_host(license_key): _settings.ssl = _environ_as_bool("NEW_RELIC_SSL", True) _settings.host = os.environ.get("NEW_RELIC_HOST") +_settings.otlp_host = os.environ.get("NEW_RELIC_OTLP_HOST") _settings.port = int(os.environ.get("NEW_RELIC_PORT", "0")) +_settings.otlp_port = int(os.environ.get("NEW_RELIC_OTLP_PORT", "0")) _settings.agent_run_id = None _settings.entity_guid = None diff --git a/newrelic/core/data_collector.py b/newrelic/core/data_collector.py index e75368bee..441f587dc 100644 --- a/newrelic/core/data_collector.py +++ b/newrelic/core/data_collector.py @@ -25,7 +25,11 @@ DeveloperModeClient, ServerlessModeClient, ) -from newrelic.core.agent_protocol import AgentProtocol, ServerlessModeProtocol +from newrelic.core.agent_protocol import ( + AgentProtocol, + OtlpProtocol, + ServerlessModeProtocol, +) from newrelic.core.agent_streaming import StreamingRpc from newrelic.core.config import global_settings @@ -36,12 +40,16 @@ class Session(object): PROTOCOL = AgentProtocol + OTLP_PROTOCOL = OtlpProtocol CLIENT = ApplicationModeClient def __init__(self, app_name, linked_applications, environment, settings): self._protocol = self.PROTOCOL.connect( app_name, linked_applications, environment, settings, client_cls=self.CLIENT ) + self._otlp_protocol = self.OTLP_PROTOCOL.connect( + app_name, linked_applications, environment, settings, client_cls=self.CLIENT + ) self._rpc = None @property diff --git a/newrelic/core/otlp_utils.py b/newrelic/core/otlp_utils.py new file mode 100644 index 000000000..6a44cb4e3 --- /dev/null +++ b/newrelic/core/otlp_utils.py @@ -0,0 +1,107 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module provides common utilities for interacting with OTLP protocol buffers.""" + +import logging + +_logger = logging.getLogger(__name__) + +try: + from newrelic.packages.opentelemetry_proto.common_pb2 import AnyValue, KeyValue + from newrelic.packages.opentelemetry_proto.logs_pb2 import ( + LogRecord, + ResourceLogs, + ScopeLogs, + ) + from newrelic.packages.opentelemetry_proto.metrics_pb2 import ( + AggregationTemporality, + Metric, + MetricsData, + NumberDataPoint, + ResourceMetrics, + ScopeMetrics, + Sum, + Summary, + SummaryDataPoint, + ) + from newrelic.packages.opentelemetry_proto.resource_pb2 import Resource + + AGGREGATION_TEMPORALITY_DELTA = AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA + ValueAtQuantile = SummaryDataPoint.ValueAtQuantile + + otlp_encode = lambda payload: payload.SerializeToString() + OTLP_CONTENT_TYPE = "application/x-protobuf" + +except ImportError: + from newrelic.common.encoding_utils import json_encode + + def otlp_encode(*args, **kwargs): + _logger.warn( + "Using OTLP integration while protobuf is not installed. This may result in larger payload sizes and data loss." + ) + return json_encode(*args, **kwargs) + + Resource = dict + ValueAtQuantile = dict + AnyValue = dict + KeyValue = dict + NumberDataPoint = dict + SummaryDataPoint = dict + Sum = dict + Summary = dict + Metric = dict + MetricsData = dict + ScopeMetrics = dict + ResourceMetrics = dict + AGGREGATION_TEMPORALITY_DELTA = 1 + ResourceLogs = dict + ScopeLogs = dict + LogRecord = dict + OTLP_CONTENT_TYPE = "application/json" + + +def create_key_value(key, value): + if isinstance(value, bool): + return KeyValue(key=key, value=AnyValue(bool_value=value)) + elif isinstance(value, int): + return KeyValue(key=key, value=AnyValue(int_value=value)) + elif isinstance(value, float): + return KeyValue(key=key, value=AnyValue(double_value=value)) + elif isinstance(value, str): + return KeyValue(key=key, value=AnyValue(string_value=value)) + # Technically AnyValue accepts array, kvlist, and bytes however, since + # those are not valid custom attribute types according to our api spec, + # we will not bother to support them here either. + else: + _logger.warn("Unsupported attribute value type %s: %s." % (key, value)) + + +def create_key_values_from_iterable(iterable): + if isinstance(iterable, dict): + iterable = iterable.items() + + # The create_key_value list may return None if the value is an unsupported type + # so filter None values out before returning. + return list( + filter( + lambda i: i is not None, + (create_key_value(key, value) for key, value in iterable), + ) + ) + + +def create_resource(attributes=None): + attributes = attributes or {"instrumentation.provider": "nr_performance_monitoring"} + return Resource(attributes=create_key_values_from_iterable(attributes)) diff --git a/newrelic/packages/opentelemetry_proto/LICENSE.txt b/newrelic/packages/opentelemetry_proto/LICENSE.txt new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/newrelic/packages/opentelemetry_proto/LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/newrelic/packages/opentelemetry_proto/__init__.py b/newrelic/packages/opentelemetry_proto/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/newrelic/packages/opentelemetry_proto/common_pb2.py b/newrelic/packages/opentelemetry_proto/common_pb2.py new file mode 100644 index 000000000..a38431a58 --- /dev/null +++ b/newrelic/packages/opentelemetry_proto/common_pb2.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: opentelemetry/proto/common/v1/common.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n*opentelemetry/proto/common/v1/common.proto\x12\x1dopentelemetry.proto.common.v1\"\x8c\x02\n\x08\x41nyValue\x12\x16\n\x0cstring_value\x18\x01 \x01(\tH\x00\x12\x14\n\nbool_value\x18\x02 \x01(\x08H\x00\x12\x13\n\tint_value\x18\x03 \x01(\x03H\x00\x12\x16\n\x0c\x64ouble_value\x18\x04 \x01(\x01H\x00\x12@\n\x0b\x61rray_value\x18\x05 \x01(\x0b\x32).opentelemetry.proto.common.v1.ArrayValueH\x00\x12\x43\n\x0ckvlist_value\x18\x06 \x01(\x0b\x32+.opentelemetry.proto.common.v1.KeyValueListH\x00\x12\x15\n\x0b\x62ytes_value\x18\x07 \x01(\x0cH\x00\x42\x07\n\x05value\"E\n\nArrayValue\x12\x37\n\x06values\x18\x01 \x03(\x0b\x32\'.opentelemetry.proto.common.v1.AnyValue\"G\n\x0cKeyValueList\x12\x37\n\x06values\x18\x01 \x03(\x0b\x32\'.opentelemetry.proto.common.v1.KeyValue\"O\n\x08KeyValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x36\n\x05value\x18\x02 \x01(\x0b\x32\'.opentelemetry.proto.common.v1.AnyValue\";\n\x16InstrumentationLibrary\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t:\x02\x18\x01\"5\n\x14InstrumentationScope\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\tB[\n io.opentelemetry.proto.common.v1B\x0b\x43ommonProtoP\x01Z(go.opentelemetry.io/proto/otlp/common/v1b\x06proto3') + + + +_ANYVALUE = DESCRIPTOR.message_types_by_name['AnyValue'] +_ARRAYVALUE = DESCRIPTOR.message_types_by_name['ArrayValue'] +_KEYVALUELIST = DESCRIPTOR.message_types_by_name['KeyValueList'] +_KEYVALUE = DESCRIPTOR.message_types_by_name['KeyValue'] +_INSTRUMENTATIONLIBRARY = DESCRIPTOR.message_types_by_name['InstrumentationLibrary'] +_INSTRUMENTATIONSCOPE = DESCRIPTOR.message_types_by_name['InstrumentationScope'] +AnyValue = _reflection.GeneratedProtocolMessageType('AnyValue', (_message.Message,), { + 'DESCRIPTOR' : _ANYVALUE, + '__module__' : 'opentelemetry.proto.common.v1.common_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.common.v1.AnyValue) + }) +_sym_db.RegisterMessage(AnyValue) + +ArrayValue = _reflection.GeneratedProtocolMessageType('ArrayValue', (_message.Message,), { + 'DESCRIPTOR' : _ARRAYVALUE, + '__module__' : 'opentelemetry.proto.common.v1.common_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.common.v1.ArrayValue) + }) +_sym_db.RegisterMessage(ArrayValue) + +KeyValueList = _reflection.GeneratedProtocolMessageType('KeyValueList', (_message.Message,), { + 'DESCRIPTOR' : _KEYVALUELIST, + '__module__' : 'opentelemetry.proto.common.v1.common_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.common.v1.KeyValueList) + }) +_sym_db.RegisterMessage(KeyValueList) + +KeyValue = _reflection.GeneratedProtocolMessageType('KeyValue', (_message.Message,), { + 'DESCRIPTOR' : _KEYVALUE, + '__module__' : 'opentelemetry.proto.common.v1.common_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.common.v1.KeyValue) + }) +_sym_db.RegisterMessage(KeyValue) + +InstrumentationLibrary = _reflection.GeneratedProtocolMessageType('InstrumentationLibrary', (_message.Message,), { + 'DESCRIPTOR' : _INSTRUMENTATIONLIBRARY, + '__module__' : 'opentelemetry.proto.common.v1.common_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.common.v1.InstrumentationLibrary) + }) +_sym_db.RegisterMessage(InstrumentationLibrary) + +InstrumentationScope = _reflection.GeneratedProtocolMessageType('InstrumentationScope', (_message.Message,), { + 'DESCRIPTOR' : _INSTRUMENTATIONSCOPE, + '__module__' : 'opentelemetry.proto.common.v1.common_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.common.v1.InstrumentationScope) + }) +_sym_db.RegisterMessage(InstrumentationScope) + +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'\n io.opentelemetry.proto.common.v1B\013CommonProtoP\001Z(go.opentelemetry.io/proto/otlp/common/v1' + _INSTRUMENTATIONLIBRARY._options = None + _INSTRUMENTATIONLIBRARY._serialized_options = b'\030\001' + _ANYVALUE._serialized_start=78 + _ANYVALUE._serialized_end=346 + _ARRAYVALUE._serialized_start=348 + _ARRAYVALUE._serialized_end=417 + _KEYVALUELIST._serialized_start=419 + _KEYVALUELIST._serialized_end=490 + _KEYVALUE._serialized_start=492 + _KEYVALUE._serialized_end=571 + _INSTRUMENTATIONLIBRARY._serialized_start=573 + _INSTRUMENTATIONLIBRARY._serialized_end=632 + _INSTRUMENTATIONSCOPE._serialized_start=634 + _INSTRUMENTATIONSCOPE._serialized_end=687 +# @@protoc_insertion_point(module_scope) diff --git a/newrelic/packages/opentelemetry_proto/logs_pb2.py b/newrelic/packages/opentelemetry_proto/logs_pb2.py new file mode 100644 index 000000000..bb6a55d66 --- /dev/null +++ b/newrelic/packages/opentelemetry_proto/logs_pb2.py @@ -0,0 +1,117 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: opentelemetry/proto/logs/v1/logs.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from . import common_pb2 as opentelemetry_dot_proto_dot_common_dot_v1_dot_common__pb2 +from . import resource_pb2 as opentelemetry_dot_proto_dot_resource_dot_v1_dot_resource__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n&opentelemetry/proto/logs/v1/logs.proto\x12\x1bopentelemetry.proto.logs.v1\x1a*opentelemetry/proto/common/v1/common.proto\x1a.opentelemetry/proto/resource/v1/resource.proto\"L\n\x08LogsData\x12@\n\rresource_logs\x18\x01 \x03(\x0b\x32).opentelemetry.proto.logs.v1.ResourceLogs\"\xff\x01\n\x0cResourceLogs\x12;\n\x08resource\x18\x01 \x01(\x0b\x32).opentelemetry.proto.resource.v1.Resource\x12:\n\nscope_logs\x18\x02 \x03(\x0b\x32&.opentelemetry.proto.logs.v1.ScopeLogs\x12\x62\n\x1cinstrumentation_library_logs\x18\xe8\x07 \x03(\x0b\x32\x37.opentelemetry.proto.logs.v1.InstrumentationLibraryLogsB\x02\x18\x01\x12\x12\n\nschema_url\x18\x03 \x01(\t\"\xa0\x01\n\tScopeLogs\x12\x42\n\x05scope\x18\x01 \x01(\x0b\x32\x33.opentelemetry.proto.common.v1.InstrumentationScope\x12;\n\x0blog_records\x18\x02 \x03(\x0b\x32&.opentelemetry.proto.logs.v1.LogRecord\x12\x12\n\nschema_url\x18\x03 \x01(\t\"\xc9\x01\n\x1aInstrumentationLibraryLogs\x12V\n\x17instrumentation_library\x18\x01 \x01(\x0b\x32\x35.opentelemetry.proto.common.v1.InstrumentationLibrary\x12;\n\x0blog_records\x18\x02 \x03(\x0b\x32&.opentelemetry.proto.logs.v1.LogRecord\x12\x12\n\nschema_url\x18\x03 \x01(\t:\x02\x18\x01\"\xef\x02\n\tLogRecord\x12\x16\n\x0etime_unix_nano\x18\x01 \x01(\x06\x12\x1f\n\x17observed_time_unix_nano\x18\x0b \x01(\x06\x12\x44\n\x0fseverity_number\x18\x02 \x01(\x0e\x32+.opentelemetry.proto.logs.v1.SeverityNumber\x12\x15\n\rseverity_text\x18\x03 \x01(\t\x12\x35\n\x04\x62ody\x18\x05 \x01(\x0b\x32\'.opentelemetry.proto.common.v1.AnyValue\x12;\n\nattributes\x18\x06 \x03(\x0b\x32\'.opentelemetry.proto.common.v1.KeyValue\x12 \n\x18\x64ropped_attributes_count\x18\x07 \x01(\r\x12\r\n\x05\x66lags\x18\x08 \x01(\x07\x12\x10\n\x08trace_id\x18\t \x01(\x0c\x12\x0f\n\x07span_id\x18\n \x01(\x0cJ\x04\x08\x04\x10\x05*\xc3\x05\n\x0eSeverityNumber\x12\x1f\n\x1bSEVERITY_NUMBER_UNSPECIFIED\x10\x00\x12\x19\n\x15SEVERITY_NUMBER_TRACE\x10\x01\x12\x1a\n\x16SEVERITY_NUMBER_TRACE2\x10\x02\x12\x1a\n\x16SEVERITY_NUMBER_TRACE3\x10\x03\x12\x1a\n\x16SEVERITY_NUMBER_TRACE4\x10\x04\x12\x19\n\x15SEVERITY_NUMBER_DEBUG\x10\x05\x12\x1a\n\x16SEVERITY_NUMBER_DEBUG2\x10\x06\x12\x1a\n\x16SEVERITY_NUMBER_DEBUG3\x10\x07\x12\x1a\n\x16SEVERITY_NUMBER_DEBUG4\x10\x08\x12\x18\n\x14SEVERITY_NUMBER_INFO\x10\t\x12\x19\n\x15SEVERITY_NUMBER_INFO2\x10\n\x12\x19\n\x15SEVERITY_NUMBER_INFO3\x10\x0b\x12\x19\n\x15SEVERITY_NUMBER_INFO4\x10\x0c\x12\x18\n\x14SEVERITY_NUMBER_WARN\x10\r\x12\x19\n\x15SEVERITY_NUMBER_WARN2\x10\x0e\x12\x19\n\x15SEVERITY_NUMBER_WARN3\x10\x0f\x12\x19\n\x15SEVERITY_NUMBER_WARN4\x10\x10\x12\x19\n\x15SEVERITY_NUMBER_ERROR\x10\x11\x12\x1a\n\x16SEVERITY_NUMBER_ERROR2\x10\x12\x12\x1a\n\x16SEVERITY_NUMBER_ERROR3\x10\x13\x12\x1a\n\x16SEVERITY_NUMBER_ERROR4\x10\x14\x12\x19\n\x15SEVERITY_NUMBER_FATAL\x10\x15\x12\x1a\n\x16SEVERITY_NUMBER_FATAL2\x10\x16\x12\x1a\n\x16SEVERITY_NUMBER_FATAL3\x10\x17\x12\x1a\n\x16SEVERITY_NUMBER_FATAL4\x10\x18*X\n\x0eLogRecordFlags\x12\x1f\n\x1bLOG_RECORD_FLAG_UNSPECIFIED\x10\x00\x12%\n LOG_RECORD_FLAG_TRACE_FLAGS_MASK\x10\xff\x01\x42U\n\x1eio.opentelemetry.proto.logs.v1B\tLogsProtoP\x01Z&go.opentelemetry.io/proto/otlp/logs/v1b\x06proto3') + +_SEVERITYNUMBER = DESCRIPTOR.enum_types_by_name['SeverityNumber'] +SeverityNumber = enum_type_wrapper.EnumTypeWrapper(_SEVERITYNUMBER) +_LOGRECORDFLAGS = DESCRIPTOR.enum_types_by_name['LogRecordFlags'] +LogRecordFlags = enum_type_wrapper.EnumTypeWrapper(_LOGRECORDFLAGS) +SEVERITY_NUMBER_UNSPECIFIED = 0 +SEVERITY_NUMBER_TRACE = 1 +SEVERITY_NUMBER_TRACE2 = 2 +SEVERITY_NUMBER_TRACE3 = 3 +SEVERITY_NUMBER_TRACE4 = 4 +SEVERITY_NUMBER_DEBUG = 5 +SEVERITY_NUMBER_DEBUG2 = 6 +SEVERITY_NUMBER_DEBUG3 = 7 +SEVERITY_NUMBER_DEBUG4 = 8 +SEVERITY_NUMBER_INFO = 9 +SEVERITY_NUMBER_INFO2 = 10 +SEVERITY_NUMBER_INFO3 = 11 +SEVERITY_NUMBER_INFO4 = 12 +SEVERITY_NUMBER_WARN = 13 +SEVERITY_NUMBER_WARN2 = 14 +SEVERITY_NUMBER_WARN3 = 15 +SEVERITY_NUMBER_WARN4 = 16 +SEVERITY_NUMBER_ERROR = 17 +SEVERITY_NUMBER_ERROR2 = 18 +SEVERITY_NUMBER_ERROR3 = 19 +SEVERITY_NUMBER_ERROR4 = 20 +SEVERITY_NUMBER_FATAL = 21 +SEVERITY_NUMBER_FATAL2 = 22 +SEVERITY_NUMBER_FATAL3 = 23 +SEVERITY_NUMBER_FATAL4 = 24 +LOG_RECORD_FLAG_UNSPECIFIED = 0 +LOG_RECORD_FLAG_TRACE_FLAGS_MASK = 255 + + +_LOGSDATA = DESCRIPTOR.message_types_by_name['LogsData'] +_RESOURCELOGS = DESCRIPTOR.message_types_by_name['ResourceLogs'] +_SCOPELOGS = DESCRIPTOR.message_types_by_name['ScopeLogs'] +_INSTRUMENTATIONLIBRARYLOGS = DESCRIPTOR.message_types_by_name['InstrumentationLibraryLogs'] +_LOGRECORD = DESCRIPTOR.message_types_by_name['LogRecord'] +LogsData = _reflection.GeneratedProtocolMessageType('LogsData', (_message.Message,), { + 'DESCRIPTOR' : _LOGSDATA, + '__module__' : 'opentelemetry.proto.logs.v1.logs_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.logs.v1.LogsData) + }) +_sym_db.RegisterMessage(LogsData) + +ResourceLogs = _reflection.GeneratedProtocolMessageType('ResourceLogs', (_message.Message,), { + 'DESCRIPTOR' : _RESOURCELOGS, + '__module__' : 'opentelemetry.proto.logs.v1.logs_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.logs.v1.ResourceLogs) + }) +_sym_db.RegisterMessage(ResourceLogs) + +ScopeLogs = _reflection.GeneratedProtocolMessageType('ScopeLogs', (_message.Message,), { + 'DESCRIPTOR' : _SCOPELOGS, + '__module__' : 'opentelemetry.proto.logs.v1.logs_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.logs.v1.ScopeLogs) + }) +_sym_db.RegisterMessage(ScopeLogs) + +InstrumentationLibraryLogs = _reflection.GeneratedProtocolMessageType('InstrumentationLibraryLogs', (_message.Message,), { + 'DESCRIPTOR' : _INSTRUMENTATIONLIBRARYLOGS, + '__module__' : 'opentelemetry.proto.logs.v1.logs_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.logs.v1.InstrumentationLibraryLogs) + }) +_sym_db.RegisterMessage(InstrumentationLibraryLogs) + +LogRecord = _reflection.GeneratedProtocolMessageType('LogRecord', (_message.Message,), { + 'DESCRIPTOR' : _LOGRECORD, + '__module__' : 'opentelemetry.proto.logs.v1.logs_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.logs.v1.LogRecord) + }) +_sym_db.RegisterMessage(LogRecord) + +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'\n\036io.opentelemetry.proto.logs.v1B\tLogsProtoP\001Z&go.opentelemetry.io/proto/otlp/logs/v1' + _RESOURCELOGS.fields_by_name['instrumentation_library_logs']._options = None + _RESOURCELOGS.fields_by_name['instrumentation_library_logs']._serialized_options = b'\030\001' + _INSTRUMENTATIONLIBRARYLOGS._options = None + _INSTRUMENTATIONLIBRARYLOGS._serialized_options = b'\030\001' + _SEVERITYNUMBER._serialized_start=1237 + _SEVERITYNUMBER._serialized_end=1944 + _LOGRECORDFLAGS._serialized_start=1946 + _LOGRECORDFLAGS._serialized_end=2034 + _LOGSDATA._serialized_start=163 + _LOGSDATA._serialized_end=239 + _RESOURCELOGS._serialized_start=242 + _RESOURCELOGS._serialized_end=497 + _SCOPELOGS._serialized_start=500 + _SCOPELOGS._serialized_end=660 + _INSTRUMENTATIONLIBRARYLOGS._serialized_start=663 + _INSTRUMENTATIONLIBRARYLOGS._serialized_end=864 + _LOGRECORD._serialized_start=867 + _LOGRECORD._serialized_end=1234 +# @@protoc_insertion_point(module_scope) diff --git a/newrelic/packages/opentelemetry_proto/metrics_pb2.py b/newrelic/packages/opentelemetry_proto/metrics_pb2.py new file mode 100644 index 000000000..dea77c7de --- /dev/null +++ b/newrelic/packages/opentelemetry_proto/metrics_pb2.py @@ -0,0 +1,217 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: opentelemetry/proto/metrics/v1/metrics.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from . import common_pb2 as opentelemetry_dot_proto_dot_common_dot_v1_dot_common__pb2 +from . import resource_pb2 as opentelemetry_dot_proto_dot_resource_dot_v1_dot_resource__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n,opentelemetry/proto/metrics/v1/metrics.proto\x12\x1eopentelemetry.proto.metrics.v1\x1a*opentelemetry/proto/common/v1/common.proto\x1a.opentelemetry/proto/resource/v1/resource.proto\"X\n\x0bMetricsData\x12I\n\x10resource_metrics\x18\x01 \x03(\x0b\x32/.opentelemetry.proto.metrics.v1.ResourceMetrics\"\x94\x02\n\x0fResourceMetrics\x12;\n\x08resource\x18\x01 \x01(\x0b\x32).opentelemetry.proto.resource.v1.Resource\x12\x43\n\rscope_metrics\x18\x02 \x03(\x0b\x32,.opentelemetry.proto.metrics.v1.ScopeMetrics\x12k\n\x1finstrumentation_library_metrics\x18\xe8\x07 \x03(\x0b\x32=.opentelemetry.proto.metrics.v1.InstrumentationLibraryMetricsB\x02\x18\x01\x12\x12\n\nschema_url\x18\x03 \x01(\t\"\x9f\x01\n\x0cScopeMetrics\x12\x42\n\x05scope\x18\x01 \x01(\x0b\x32\x33.opentelemetry.proto.common.v1.InstrumentationScope\x12\x37\n\x07metrics\x18\x02 \x03(\x0b\x32&.opentelemetry.proto.metrics.v1.Metric\x12\x12\n\nschema_url\x18\x03 \x01(\t\"\xc8\x01\n\x1dInstrumentationLibraryMetrics\x12V\n\x17instrumentation_library\x18\x01 \x01(\x0b\x32\x35.opentelemetry.proto.common.v1.InstrumentationLibrary\x12\x37\n\x07metrics\x18\x02 \x03(\x0b\x32&.opentelemetry.proto.metrics.v1.Metric\x12\x12\n\nschema_url\x18\x03 \x01(\t:\x02\x18\x01\"\x92\x03\n\x06Metric\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x0c\n\x04unit\x18\x03 \x01(\t\x12\x36\n\x05gauge\x18\x05 \x01(\x0b\x32%.opentelemetry.proto.metrics.v1.GaugeH\x00\x12\x32\n\x03sum\x18\x07 \x01(\x0b\x32#.opentelemetry.proto.metrics.v1.SumH\x00\x12>\n\thistogram\x18\t \x01(\x0b\x32).opentelemetry.proto.metrics.v1.HistogramH\x00\x12U\n\x15\x65xponential_histogram\x18\n \x01(\x0b\x32\x34.opentelemetry.proto.metrics.v1.ExponentialHistogramH\x00\x12:\n\x07summary\x18\x0b \x01(\x0b\x32\'.opentelemetry.proto.metrics.v1.SummaryH\x00\x42\x06\n\x04\x64\x61taJ\x04\x08\x04\x10\x05J\x04\x08\x06\x10\x07J\x04\x08\x08\x10\t\"M\n\x05Gauge\x12\x44\n\x0b\x64\x61ta_points\x18\x01 \x03(\x0b\x32/.opentelemetry.proto.metrics.v1.NumberDataPoint\"\xba\x01\n\x03Sum\x12\x44\n\x0b\x64\x61ta_points\x18\x01 \x03(\x0b\x32/.opentelemetry.proto.metrics.v1.NumberDataPoint\x12W\n\x17\x61ggregation_temporality\x18\x02 \x01(\x0e\x32\x36.opentelemetry.proto.metrics.v1.AggregationTemporality\x12\x14\n\x0cis_monotonic\x18\x03 \x01(\x08\"\xad\x01\n\tHistogram\x12G\n\x0b\x64\x61ta_points\x18\x01 \x03(\x0b\x32\x32.opentelemetry.proto.metrics.v1.HistogramDataPoint\x12W\n\x17\x61ggregation_temporality\x18\x02 \x01(\x0e\x32\x36.opentelemetry.proto.metrics.v1.AggregationTemporality\"\xc3\x01\n\x14\x45xponentialHistogram\x12R\n\x0b\x64\x61ta_points\x18\x01 \x03(\x0b\x32=.opentelemetry.proto.metrics.v1.ExponentialHistogramDataPoint\x12W\n\x17\x61ggregation_temporality\x18\x02 \x01(\x0e\x32\x36.opentelemetry.proto.metrics.v1.AggregationTemporality\"P\n\x07Summary\x12\x45\n\x0b\x64\x61ta_points\x18\x01 \x03(\x0b\x32\x30.opentelemetry.proto.metrics.v1.SummaryDataPoint\"\x86\x02\n\x0fNumberDataPoint\x12;\n\nattributes\x18\x07 \x03(\x0b\x32\'.opentelemetry.proto.common.v1.KeyValue\x12\x1c\n\x14start_time_unix_nano\x18\x02 \x01(\x06\x12\x16\n\x0etime_unix_nano\x18\x03 \x01(\x06\x12\x13\n\tas_double\x18\x04 \x01(\x01H\x00\x12\x10\n\x06\x61s_int\x18\x06 \x01(\x10H\x00\x12;\n\texemplars\x18\x05 \x03(\x0b\x32(.opentelemetry.proto.metrics.v1.Exemplar\x12\r\n\x05\x66lags\x18\x08 \x01(\rB\x07\n\x05valueJ\x04\x08\x01\x10\x02\"\xe6\x02\n\x12HistogramDataPoint\x12;\n\nattributes\x18\t \x03(\x0b\x32\'.opentelemetry.proto.common.v1.KeyValue\x12\x1c\n\x14start_time_unix_nano\x18\x02 \x01(\x06\x12\x16\n\x0etime_unix_nano\x18\x03 \x01(\x06\x12\r\n\x05\x63ount\x18\x04 \x01(\x06\x12\x10\n\x03sum\x18\x05 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\rbucket_counts\x18\x06 \x03(\x06\x12\x17\n\x0f\x65xplicit_bounds\x18\x07 \x03(\x01\x12;\n\texemplars\x18\x08 \x03(\x0b\x32(.opentelemetry.proto.metrics.v1.Exemplar\x12\r\n\x05\x66lags\x18\n \x01(\r\x12\x10\n\x03min\x18\x0b \x01(\x01H\x01\x88\x01\x01\x12\x10\n\x03max\x18\x0c \x01(\x01H\x02\x88\x01\x01\x42\x06\n\x04_sumB\x06\n\x04_minB\x06\n\x04_maxJ\x04\x08\x01\x10\x02\"\xb5\x04\n\x1d\x45xponentialHistogramDataPoint\x12;\n\nattributes\x18\x01 \x03(\x0b\x32\'.opentelemetry.proto.common.v1.KeyValue\x12\x1c\n\x14start_time_unix_nano\x18\x02 \x01(\x06\x12\x16\n\x0etime_unix_nano\x18\x03 \x01(\x06\x12\r\n\x05\x63ount\x18\x04 \x01(\x06\x12\x0b\n\x03sum\x18\x05 \x01(\x01\x12\r\n\x05scale\x18\x06 \x01(\x11\x12\x12\n\nzero_count\x18\x07 \x01(\x06\x12W\n\x08positive\x18\x08 \x01(\x0b\x32\x45.opentelemetry.proto.metrics.v1.ExponentialHistogramDataPoint.Buckets\x12W\n\x08negative\x18\t \x01(\x0b\x32\x45.opentelemetry.proto.metrics.v1.ExponentialHistogramDataPoint.Buckets\x12\r\n\x05\x66lags\x18\n \x01(\r\x12;\n\texemplars\x18\x0b \x03(\x0b\x32(.opentelemetry.proto.metrics.v1.Exemplar\x12\x10\n\x03min\x18\x0c \x01(\x01H\x00\x88\x01\x01\x12\x10\n\x03max\x18\r \x01(\x01H\x01\x88\x01\x01\x1a\x30\n\x07\x42uckets\x12\x0e\n\x06offset\x18\x01 \x01(\x11\x12\x15\n\rbucket_counts\x18\x02 \x03(\x04\x42\x06\n\x04_minB\x06\n\x04_max\"\xc5\x02\n\x10SummaryDataPoint\x12;\n\nattributes\x18\x07 \x03(\x0b\x32\'.opentelemetry.proto.common.v1.KeyValue\x12\x1c\n\x14start_time_unix_nano\x18\x02 \x01(\x06\x12\x16\n\x0etime_unix_nano\x18\x03 \x01(\x06\x12\r\n\x05\x63ount\x18\x04 \x01(\x06\x12\x0b\n\x03sum\x18\x05 \x01(\x01\x12Y\n\x0fquantile_values\x18\x06 \x03(\x0b\x32@.opentelemetry.proto.metrics.v1.SummaryDataPoint.ValueAtQuantile\x12\r\n\x05\x66lags\x18\x08 \x01(\r\x1a\x32\n\x0fValueAtQuantile\x12\x10\n\x08quantile\x18\x01 \x01(\x01\x12\r\n\x05value\x18\x02 \x01(\x01J\x04\x08\x01\x10\x02\"\xc1\x01\n\x08\x45xemplar\x12\x44\n\x13\x66iltered_attributes\x18\x07 \x03(\x0b\x32\'.opentelemetry.proto.common.v1.KeyValue\x12\x16\n\x0etime_unix_nano\x18\x02 \x01(\x06\x12\x13\n\tas_double\x18\x03 \x01(\x01H\x00\x12\x10\n\x06\x61s_int\x18\x06 \x01(\x10H\x00\x12\x0f\n\x07span_id\x18\x04 \x01(\x0c\x12\x10\n\x08trace_id\x18\x05 \x01(\x0c\x42\x07\n\x05valueJ\x04\x08\x01\x10\x02*\x8c\x01\n\x16\x41ggregationTemporality\x12\'\n#AGGREGATION_TEMPORALITY_UNSPECIFIED\x10\x00\x12!\n\x1d\x41GGREGATION_TEMPORALITY_DELTA\x10\x01\x12&\n\"AGGREGATION_TEMPORALITY_CUMULATIVE\x10\x02*;\n\x0e\x44\x61taPointFlags\x12\r\n\tFLAG_NONE\x10\x00\x12\x1a\n\x16\x46LAG_NO_RECORDED_VALUE\x10\x01\x42^\n!io.opentelemetry.proto.metrics.v1B\x0cMetricsProtoP\x01Z)go.opentelemetry.io/proto/otlp/metrics/v1b\x06proto3') + +_AGGREGATIONTEMPORALITY = DESCRIPTOR.enum_types_by_name['AggregationTemporality'] +AggregationTemporality = enum_type_wrapper.EnumTypeWrapper(_AGGREGATIONTEMPORALITY) +_DATAPOINTFLAGS = DESCRIPTOR.enum_types_by_name['DataPointFlags'] +DataPointFlags = enum_type_wrapper.EnumTypeWrapper(_DATAPOINTFLAGS) +AGGREGATION_TEMPORALITY_UNSPECIFIED = 0 +AGGREGATION_TEMPORALITY_DELTA = 1 +AGGREGATION_TEMPORALITY_CUMULATIVE = 2 +FLAG_NONE = 0 +FLAG_NO_RECORDED_VALUE = 1 + + +_METRICSDATA = DESCRIPTOR.message_types_by_name['MetricsData'] +_RESOURCEMETRICS = DESCRIPTOR.message_types_by_name['ResourceMetrics'] +_SCOPEMETRICS = DESCRIPTOR.message_types_by_name['ScopeMetrics'] +_INSTRUMENTATIONLIBRARYMETRICS = DESCRIPTOR.message_types_by_name['InstrumentationLibraryMetrics'] +_METRIC = DESCRIPTOR.message_types_by_name['Metric'] +_GAUGE = DESCRIPTOR.message_types_by_name['Gauge'] +_SUM = DESCRIPTOR.message_types_by_name['Sum'] +_HISTOGRAM = DESCRIPTOR.message_types_by_name['Histogram'] +_EXPONENTIALHISTOGRAM = DESCRIPTOR.message_types_by_name['ExponentialHistogram'] +_SUMMARY = DESCRIPTOR.message_types_by_name['Summary'] +_NUMBERDATAPOINT = DESCRIPTOR.message_types_by_name['NumberDataPoint'] +_HISTOGRAMDATAPOINT = DESCRIPTOR.message_types_by_name['HistogramDataPoint'] +_EXPONENTIALHISTOGRAMDATAPOINT = DESCRIPTOR.message_types_by_name['ExponentialHistogramDataPoint'] +_EXPONENTIALHISTOGRAMDATAPOINT_BUCKETS = _EXPONENTIALHISTOGRAMDATAPOINT.nested_types_by_name['Buckets'] +_SUMMARYDATAPOINT = DESCRIPTOR.message_types_by_name['SummaryDataPoint'] +_SUMMARYDATAPOINT_VALUEATQUANTILE = _SUMMARYDATAPOINT.nested_types_by_name['ValueAtQuantile'] +_EXEMPLAR = DESCRIPTOR.message_types_by_name['Exemplar'] +MetricsData = _reflection.GeneratedProtocolMessageType('MetricsData', (_message.Message,), { + 'DESCRIPTOR' : _METRICSDATA, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.MetricsData) + }) +_sym_db.RegisterMessage(MetricsData) + +ResourceMetrics = _reflection.GeneratedProtocolMessageType('ResourceMetrics', (_message.Message,), { + 'DESCRIPTOR' : _RESOURCEMETRICS, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.ResourceMetrics) + }) +_sym_db.RegisterMessage(ResourceMetrics) + +ScopeMetrics = _reflection.GeneratedProtocolMessageType('ScopeMetrics', (_message.Message,), { + 'DESCRIPTOR' : _SCOPEMETRICS, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.ScopeMetrics) + }) +_sym_db.RegisterMessage(ScopeMetrics) + +InstrumentationLibraryMetrics = _reflection.GeneratedProtocolMessageType('InstrumentationLibraryMetrics', (_message.Message,), { + 'DESCRIPTOR' : _INSTRUMENTATIONLIBRARYMETRICS, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.InstrumentationLibraryMetrics) + }) +_sym_db.RegisterMessage(InstrumentationLibraryMetrics) + +Metric = _reflection.GeneratedProtocolMessageType('Metric', (_message.Message,), { + 'DESCRIPTOR' : _METRIC, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.Metric) + }) +_sym_db.RegisterMessage(Metric) + +Gauge = _reflection.GeneratedProtocolMessageType('Gauge', (_message.Message,), { + 'DESCRIPTOR' : _GAUGE, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.Gauge) + }) +_sym_db.RegisterMessage(Gauge) + +Sum = _reflection.GeneratedProtocolMessageType('Sum', (_message.Message,), { + 'DESCRIPTOR' : _SUM, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.Sum) + }) +_sym_db.RegisterMessage(Sum) + +Histogram = _reflection.GeneratedProtocolMessageType('Histogram', (_message.Message,), { + 'DESCRIPTOR' : _HISTOGRAM, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.Histogram) + }) +_sym_db.RegisterMessage(Histogram) + +ExponentialHistogram = _reflection.GeneratedProtocolMessageType('ExponentialHistogram', (_message.Message,), { + 'DESCRIPTOR' : _EXPONENTIALHISTOGRAM, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.ExponentialHistogram) + }) +_sym_db.RegisterMessage(ExponentialHistogram) + +Summary = _reflection.GeneratedProtocolMessageType('Summary', (_message.Message,), { + 'DESCRIPTOR' : _SUMMARY, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.Summary) + }) +_sym_db.RegisterMessage(Summary) + +NumberDataPoint = _reflection.GeneratedProtocolMessageType('NumberDataPoint', (_message.Message,), { + 'DESCRIPTOR' : _NUMBERDATAPOINT, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.NumberDataPoint) + }) +_sym_db.RegisterMessage(NumberDataPoint) + +HistogramDataPoint = _reflection.GeneratedProtocolMessageType('HistogramDataPoint', (_message.Message,), { + 'DESCRIPTOR' : _HISTOGRAMDATAPOINT, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.HistogramDataPoint) + }) +_sym_db.RegisterMessage(HistogramDataPoint) + +ExponentialHistogramDataPoint = _reflection.GeneratedProtocolMessageType('ExponentialHistogramDataPoint', (_message.Message,), { + + 'Buckets' : _reflection.GeneratedProtocolMessageType('Buckets', (_message.Message,), { + 'DESCRIPTOR' : _EXPONENTIALHISTOGRAMDATAPOINT_BUCKETS, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.ExponentialHistogramDataPoint.Buckets) + }) + , + 'DESCRIPTOR' : _EXPONENTIALHISTOGRAMDATAPOINT, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.ExponentialHistogramDataPoint) + }) +_sym_db.RegisterMessage(ExponentialHistogramDataPoint) +_sym_db.RegisterMessage(ExponentialHistogramDataPoint.Buckets) + +SummaryDataPoint = _reflection.GeneratedProtocolMessageType('SummaryDataPoint', (_message.Message,), { + + 'ValueAtQuantile' : _reflection.GeneratedProtocolMessageType('ValueAtQuantile', (_message.Message,), { + 'DESCRIPTOR' : _SUMMARYDATAPOINT_VALUEATQUANTILE, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.SummaryDataPoint.ValueAtQuantile) + }) + , + 'DESCRIPTOR' : _SUMMARYDATAPOINT, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.SummaryDataPoint) + }) +_sym_db.RegisterMessage(SummaryDataPoint) +_sym_db.RegisterMessage(SummaryDataPoint.ValueAtQuantile) + +Exemplar = _reflection.GeneratedProtocolMessageType('Exemplar', (_message.Message,), { + 'DESCRIPTOR' : _EXEMPLAR, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.Exemplar) + }) +_sym_db.RegisterMessage(Exemplar) + +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'\n!io.opentelemetry.proto.metrics.v1B\014MetricsProtoP\001Z)go.opentelemetry.io/proto/otlp/metrics/v1' + _RESOURCEMETRICS.fields_by_name['instrumentation_library_metrics']._options = None + _RESOURCEMETRICS.fields_by_name['instrumentation_library_metrics']._serialized_options = b'\030\001' + _INSTRUMENTATIONLIBRARYMETRICS._options = None + _INSTRUMENTATIONLIBRARYMETRICS._serialized_options = b'\030\001' + _AGGREGATIONTEMPORALITY._serialized_start=3754 + _AGGREGATIONTEMPORALITY._serialized_end=3894 + _DATAPOINTFLAGS._serialized_start=3896 + _DATAPOINTFLAGS._serialized_end=3955 + _METRICSDATA._serialized_start=172 + _METRICSDATA._serialized_end=260 + _RESOURCEMETRICS._serialized_start=263 + _RESOURCEMETRICS._serialized_end=539 + _SCOPEMETRICS._serialized_start=542 + _SCOPEMETRICS._serialized_end=701 + _INSTRUMENTATIONLIBRARYMETRICS._serialized_start=704 + _INSTRUMENTATIONLIBRARYMETRICS._serialized_end=904 + _METRIC._serialized_start=907 + _METRIC._serialized_end=1309 + _GAUGE._serialized_start=1311 + _GAUGE._serialized_end=1388 + _SUM._serialized_start=1391 + _SUM._serialized_end=1577 + _HISTOGRAM._serialized_start=1580 + _HISTOGRAM._serialized_end=1753 + _EXPONENTIALHISTOGRAM._serialized_start=1756 + _EXPONENTIALHISTOGRAM._serialized_end=1951 + _SUMMARY._serialized_start=1953 + _SUMMARY._serialized_end=2033 + _NUMBERDATAPOINT._serialized_start=2036 + _NUMBERDATAPOINT._serialized_end=2298 + _HISTOGRAMDATAPOINT._serialized_start=2301 + _HISTOGRAMDATAPOINT._serialized_end=2659 + _EXPONENTIALHISTOGRAMDATAPOINT._serialized_start=2662 + _EXPONENTIALHISTOGRAMDATAPOINT._serialized_end=3227 + _EXPONENTIALHISTOGRAMDATAPOINT_BUCKETS._serialized_start=3163 + _EXPONENTIALHISTOGRAMDATAPOINT_BUCKETS._serialized_end=3211 + _SUMMARYDATAPOINT._serialized_start=3230 + _SUMMARYDATAPOINT._serialized_end=3555 + _SUMMARYDATAPOINT_VALUEATQUANTILE._serialized_start=3499 + _SUMMARYDATAPOINT_VALUEATQUANTILE._serialized_end=3549 + _EXEMPLAR._serialized_start=3558 + _EXEMPLAR._serialized_end=3751 +# @@protoc_insertion_point(module_scope) \ No newline at end of file diff --git a/newrelic/packages/opentelemetry_proto/resource_pb2.py b/newrelic/packages/opentelemetry_proto/resource_pb2.py new file mode 100644 index 000000000..8cc64e352 --- /dev/null +++ b/newrelic/packages/opentelemetry_proto/resource_pb2.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: opentelemetry/proto/resource/v1/resource.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from . import common_pb2 as opentelemetry_dot_proto_dot_common_dot_v1_dot_common__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.opentelemetry/proto/resource/v1/resource.proto\x12\x1fopentelemetry.proto.resource.v1\x1a*opentelemetry/proto/common/v1/common.proto\"i\n\x08Resource\x12;\n\nattributes\x18\x01 \x03(\x0b\x32\'.opentelemetry.proto.common.v1.KeyValue\x12 \n\x18\x64ropped_attributes_count\x18\x02 \x01(\rBa\n\"io.opentelemetry.proto.resource.v1B\rResourceProtoP\x01Z*go.opentelemetry.io/proto/otlp/resource/v1b\x06proto3') + + + +_RESOURCE = DESCRIPTOR.message_types_by_name['Resource'] +Resource = _reflection.GeneratedProtocolMessageType('Resource', (_message.Message,), { + 'DESCRIPTOR' : _RESOURCE, + '__module__' : 'opentelemetry.proto.resource.v1.resource_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.resource.v1.Resource) + }) +_sym_db.RegisterMessage(Resource) + +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'\n\"io.opentelemetry.proto.resource.v1B\rResourceProtoP\001Z*go.opentelemetry.io/proto/otlp/resource/v1' + _RESOURCE._serialized_start=127 + _RESOURCE._serialized_end=232 +# @@protoc_insertion_point(module_scope) diff --git a/setup.py b/setup.py index 044125a23..93bcf74e6 100644 --- a/setup.py +++ b/setup.py @@ -109,6 +109,7 @@ def build_extension(self, ext): "newrelic/packages/urllib3/packages", "newrelic/packages/urllib3/packages/backports", "newrelic/packages/wrapt", + "newrelic/packages/opentelemetry_proto", "newrelic.samplers", ] diff --git a/tests/agent_features/test_configuration.py b/tests/agent_features/test_configuration.py index 5df69d71e..547a0eeb6 100644 --- a/tests/agent_features/test_configuration.py +++ b/tests/agent_features/test_configuration.py @@ -577,6 +577,8 @@ def test_translate_deprecated_ignored_params_with_new_setting(): ("agent_run_id", None), ("entity_guid", None), ("distributed_tracing.exclude_newrelic_header", False), + ("otlp_host", "otlp.nr-data.net"), + ("otlp_port", 0), ), ) def test_default_values(name, expected_value): diff --git a/tests/agent_unittests/test_utilization_settings.py b/tests/agent_unittests/test_utilization_settings.py index 8af4bcbf1..96cf47669 100644 --- a/tests/agent_unittests/test_utilization_settings.py +++ b/tests/agent_unittests/test_utilization_settings.py @@ -118,6 +118,22 @@ def reset(wrapped, instance, args, kwargs): return reset +@reset_agent_config(INI_FILE_WITHOUT_UTIL_CONF, ENV_WITHOUT_UTIL_CONF) +def test_otlp_host_port_default(): + settings = global_settings() + assert settings.otlp_host == "otlp.nr-data.net" + assert settings.otlp_port == 0 + + +@reset_agent_config( + INI_FILE_WITHOUT_UTIL_CONF, {"NEW_RELIC_OTLP_HOST": "custom-otlp.nr-data.net", "NEW_RELIC_OTLP_PORT": 443} +) +def test_otlp_port_override(): + settings = global_settings() + assert settings.otlp_host == "custom-otlp.nr-data.net" + assert settings.otlp_port == 443 + + @reset_agent_config(INI_FILE_WITHOUT_UTIL_CONF, ENV_WITHOUT_UTIL_CONF) def test_heroku_default(): settings = global_settings() diff --git a/tox.ini b/tox.ini index 33cf11774..adc3206f8 100644 --- a/tox.ini +++ b/tox.ini @@ -207,6 +207,7 @@ deps = application_celery-py{py37,37}: importlib-metadata<5.0 application_gearman: gearman<3.0.0 mlmodel_sklearn: pandas + mlmodel_sklearn: protobuf mlmodel_sklearn-scikitlearnlatest: scikit-learn mlmodel_sklearn-scikitlearn0101: scikit-learn<1.1 mlmodel_sklearn-scikitlearn0020: scikit-learn<0.21 From 4876a98d4a84b323918ebc4035b7e4df539c05b9 Mon Sep 17 00:00:00 2001 From: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Mon, 12 Jun 2023 16:15:54 -0700 Subject: [PATCH 35/54] Fix Testing Failures (#828) * Fix tastypie tests * Adjust asgiref pinned version * Make aioredis key PID unique * Pin more asgiref versions --- tests/datastore_aioredis/conftest.py | 5 +++ tests/datastore_aioredis/test_get_and_set.py | 14 ++++---- tests/datastore_aioredis/test_transactions.py | 36 +++++++++---------- tox.ini | 2 ++ 4 files changed, 31 insertions(+), 26 deletions(-) diff --git a/tests/datastore_aioredis/conftest.py b/tests/datastore_aioredis/conftest.py index d50129255..e1cea4c01 100644 --- a/tests/datastore_aioredis/conftest.py +++ b/tests/datastore_aioredis/conftest.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import pytest from newrelic.common.package_version_utils import get_package_version_tuple @@ -67,3 +68,7 @@ def client(request, loop): pytest.skip("StrictRedis not implemented.") else: raise NotImplementedError() + +@pytest.fixture(scope="session") +def key(): + return "AIOREDIS-TEST-" + str(os.getpid()) diff --git a/tests/datastore_aioredis/test_get_and_set.py b/tests/datastore_aioredis/test_get_and_set.py index 180f32578..cbddf6091 100644 --- a/tests/datastore_aioredis/test_get_and_set.py +++ b/tests/datastore_aioredis/test_get_and_set.py @@ -64,9 +64,9 @@ _disable_rollup_metrics.append((_instance_metric_name, None)) -async def exercise_redis(client): - await client.set("key", "value") - await client.get("key") +async def exercise_redis(client, key): + await client.set(key, "value") + await client.get(key) @override_application_settings(_enable_instance_settings) @@ -77,8 +77,8 @@ async def exercise_redis(client): background_task=True, ) @background_task() -def test_redis_client_operation_enable_instance(client, loop): - loop.run_until_complete(exercise_redis(client)) +def test_redis_client_operation_enable_instance(client, loop, key): + loop.run_until_complete(exercise_redis(client, key)) @override_application_settings(_disable_instance_settings) @@ -89,5 +89,5 @@ def test_redis_client_operation_enable_instance(client, loop): background_task=True, ) @background_task() -def test_redis_client_operation_disable_instance(client, loop): - loop.run_until_complete(exercise_redis(client)) +def test_redis_client_operation_disable_instance(client, loop, key): + loop.run_until_complete(exercise_redis(client, key)) diff --git a/tests/datastore_aioredis/test_transactions.py b/tests/datastore_aioredis/test_transactions.py index 0f84ca684..ced922022 100644 --- a/tests/datastore_aioredis/test_transactions.py +++ b/tests/datastore_aioredis/test_transactions.py @@ -23,42 +23,46 @@ @background_task() @pytest.mark.parametrize("in_transaction", (True, False)) -def test_pipelines_no_harm(client, in_transaction, loop): +def test_pipelines_no_harm(client, in_transaction, loop, key): async def exercise(): if AIOREDIS_VERSION >= (2,): pipe = client.pipeline(transaction=in_transaction) else: pipe = client.pipeline() # Transaction kwarg unsupported - pipe.set("TXN", 1) + pipe.set(key, 1) return await pipe.execute() status = loop.run_until_complete(exercise()) assert status == [True] -def exercise_transaction_sync(pipe): - pipe.set("TXN", 1) +def exercise_transaction_sync(key): + def _run(pipe): + pipe.set(key, 1) + return _run -async def exercise_transaction_async(pipe): - await pipe.set("TXN", 1) +def exercise_transaction_async(key): + async def _run(pipe): + await pipe.set(key, 1) + return _run @SKIPIF_AIOREDIS_V1 @pytest.mark.parametrize("exercise", (exercise_transaction_sync, exercise_transaction_async)) @background_task() -def test_transactions_no_harm(client, loop, exercise): - status = loop.run_until_complete(client.transaction(exercise)) +def test_transactions_no_harm(client, loop, key, exercise): + status = loop.run_until_complete(client.transaction(exercise(key))) assert status == [True] @SKIPIF_AIOREDIS_V2 @background_task() -def test_multi_exec_no_harm(client, loop): +def test_multi_exec_no_harm(client, loop, key): async def exercise(): pipe = client.multi_exec() - pipe.set("key", "value") + pipe.set(key, "value") status = await pipe.execute() assert status == [True] @@ -67,9 +71,7 @@ async def exercise(): @SKIPIF_AIOREDIS_V1 @background_task() -def test_pipeline_immediate_execution_no_harm(client, loop): - key = "TXN_WATCH" - +def test_pipeline_immediate_execution_no_harm(client, loop, key): async def exercise(): await client.set(key, 1) @@ -94,9 +96,7 @@ async def exercise(): @SKIPIF_AIOREDIS_V1 @background_task() -def test_transaction_immediate_execution_no_harm(client, loop): - key = "TXN_WATCH" - +def test_transaction_immediate_execution_no_harm(client, loop, key): async def exercise(): async def exercise_transaction(pipe): value = int(await pipe.get(key)) @@ -119,9 +119,7 @@ async def exercise_transaction(pipe): @SKIPIF_AIOREDIS_V1 @validate_transaction_errors([]) @background_task() -def test_transaction_watch_error_no_harm(client, loop): - key = "TXN_WATCH" - +def test_transaction_watch_error_no_harm(client, loop, key): async def exercise(): async def exercise_transaction(pipe): value = int(await pipe.get(key)) diff --git a/tox.ini b/tox.ini index adc3206f8..30215f162 100644 --- a/tox.ini +++ b/tox.ini @@ -229,8 +229,10 @@ deps = component_tastypie-tastypie0143: django-tastypie<0.14.4 component_tastypie-{py27,pypy}-tastypie0143: django<1.12 component_tastypie-{py37,py38,py39,py310,py311,pypy37}-tastypie0143: django<3.0.1 + component_tastypie-{py37,py38,py39,py310,py311,pypy37}-tastypie0143: asgiref<3.7.1 # asgiref==3.7.1 only suppport Python 3.10+ component_tastypie-tastypielatest: django-tastypie component_tastypie-tastypielatest: django<4.1 + component_tastypie-tastypielatest: asgiref<3.7.1 # asgiref==3.7.1 only suppport Python 3.10+ coroutines_asyncio-{py37,py38,py39,py310,py311}: uvloop cross_agent: mock==1.0.1 cross_agent: requests From 2f6581f1295af6cf88ae98c14ee80ca98727375d Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Mon, 12 Jun 2023 16:53:06 -0700 Subject: [PATCH 36/54] Fix pytest test filtering when running tox (#823) Co-authored-by: Uma Annamalai --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 30215f162..f077e09ec 100644 --- a/tox.ini +++ b/tox.ini @@ -416,7 +416,7 @@ commands = framework_grpc: /{toxinidir}/tests/framework_grpc/sample_application/sample_application.proto libcurl: pip install --ignore-installed --config-settings="--build-option=--with-openssl" pycurl - coverage run -m pytest -v + coverage run -m pytest -v [] allowlist_externals={toxinidir}/.github/scripts/* From ad05014f8d07db799cbb664f3b137a1f3e4ec541 Mon Sep 17 00:00:00 2001 From: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Fri, 23 Jun 2023 11:24:37 -0700 Subject: [PATCH 37/54] OTLP Serialization for Dimensional Metrics (#826) * Add protos under packages for otlp * Add common otlp proto payload methods * Add new oltp protocol class * Remove ML event from log message * Remove params, add api-key header & expose path The params are not relevant to OTLP so remove these. The api-key header is how we provide the license key to OTLP so add this. The path to upload dimensional metrics and events are different in OTLP so expose the path so it can be overriden inside the coresponding data_collector methods. * Add metric protos * Use protos to create payload * Squashed commit of the following: commit 6f15520cea6a1098915c9ca340dbe42de6a5de1d Author: Tim Pansino Date: Mon May 15 14:28:50 2023 -0700 TEMP commit 1a28d36f86dd3f1fa5ca7a8f56357d168aac69db Author: Tim Pansino Date: Thu May 11 17:28:27 2023 -0700 Cover tags as list not dict commit 71261e3d468320569742a72c690f6ff4e9b3e621 Merge: 459e08567 c2d4629df Author: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Thu May 11 16:59:11 2023 -0700 Merge branch 'main' into feature-dimensional-metrics commit 459e08567102cfadce398b57d245ecf99408400d Author: Tim Pansino Date: Thu May 11 16:57:16 2023 -0700 Add testing for dimensional metrics commit ed33957cd2b20bc1f6e9759a0bad5e4f4a86a38c Author: Tim Pansino Date: Thu May 11 16:56:31 2023 -0700 Add attribute processing to metric identity commit 6caf71ef4386395d950060e0e996f80dbcbfbc32 Author: Tim Pansino Date: Thu May 11 16:56:16 2023 -0700 Add dimensional stats table to stats engine commit 5e1cc9dea6d0d9623130dedd0f787408a8439388 Author: Tim Pansino Date: Wed May 10 16:00:42 2023 -0700 Squashed commit of the following: commit c2d4629dfd7787354b6607160bb952913975d5f7 Author: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Wed May 10 15:59:13 2023 -0700 Add required option for tox v4 (#795) * Add required option for tox v4 * Update tox in GHA * Remove py27 no-cache-dir commit a9636498ab5c20c266fb044a08359c0c9bbcf826 Author: Hannah Stepanek Date: Tue May 9 10:46:39 2023 -0700 Run coverage around pytest (#813) * Run coverage around pytest * Trigger tests * Fixup * Add redis client_no_touch to ignore list * Temporarily remove kafka from coverage * Remove coverage for old libs commit 3d8284540e0acd867c2cf680f43449bc128c0779 Author: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Wed May 3 14:50:30 2023 -0700 Omit some frameworks from coverage analysis (#810) * Omit some frameworks from coverage analysis * Remove commas * Change format of omit * Add relative_files option to coverage * Add absolute directory * Add envsitepackagedir * Add coveragerc file * Add codecov.yml * [Mega-Linter] Apply linters fixes * Revert coveragerc file settings * Add files in packages and more frameworks * Remove commented line --------- Co-authored-by: lrafeei Co-authored-by: Hannah Stepanek commit fd0fa35466b630e34e8476cc53ad0e163564e2de Author: Uma Annamalai Date: Tue May 2 10:55:36 2023 -0700 Add testing for genshi and mako. (#799) * Add testing for genshi and mako. * [Mega-Linter] Apply linters fixes --------- Co-authored-by: umaannamalai Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> commit be4fb3dda0e734889acd6bc53cf91f26c18c2118 Author: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Mon May 1 16:01:09 2023 -0700 Add tests for Waitress (#797) * Change import format * Initial commit * Add more tests to adapter_waitress * Remove commented out code * [Mega-Linter] Apply linters fixes * Add assertions to all tests * Add more NR testing to waitress --------- Co-authored-by: lrafeei Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> commit 7103506ca5639d339e3e47dfb9e4affb546c839b Author: Hannah Stepanek Date: Mon May 1 14:12:31 2023 -0700 Add tests for pyodbc (#796) * Add tests for pyodbc * Move imports into tests to get import coverage * Fixup: remove time import * Trigger tests --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> commit 363122a0efe0ad9f4784fc1f67fda046cb9bb7e8 Author: Hannah Stepanek Date: Mon May 1 13:34:35 2023 -0700 Pin virtualenv, fix pip arg deprecation & disable kafka tests (#803) * Pin virtualenv * Fixup: use 20.21.1 instead * Replace install-options with config-settings See https://github.com/pypa/pip/issues/11358. * Temporarily disable kafka tests commit c2d4629dfd7787354b6607160bb952913975d5f7 Author: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Wed May 10 15:59:13 2023 -0700 Add required option for tox v4 (#795) * Add required option for tox v4 * Update tox in GHA * Remove py27 no-cache-dir commit a9636498ab5c20c266fb044a08359c0c9bbcf826 Author: Hannah Stepanek Date: Tue May 9 10:46:39 2023 -0700 Run coverage around pytest (#813) * Run coverage around pytest * Trigger tests * Fixup * Add redis client_no_touch to ignore list * Temporarily remove kafka from coverage * Remove coverage for old libs commit dc81a50a9fc5f2a5ce6978aa064fdfab1618328b Author: Tim Pansino Date: Sat May 6 14:16:14 2023 -0700 Wiring dimensional metrics commit 3d8284540e0acd867c2cf680f43449bc128c0779 Author: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Wed May 3 14:50:30 2023 -0700 Omit some frameworks from coverage analysis (#810) * Omit some frameworks from coverage analysis * Remove commas * Change format of omit * Add relative_files option to coverage * Add absolute directory * Add envsitepackagedir * Add coveragerc file * Add codecov.yml * [Mega-Linter] Apply linters fixes * Revert coveragerc file settings * Add files in packages and more frameworks * Remove commented line --------- Co-authored-by: lrafeei Co-authored-by: Hannah Stepanek commit fd0fa35466b630e34e8476cc53ad0e163564e2de Author: Uma Annamalai Date: Tue May 2 10:55:36 2023 -0700 Add testing for genshi and mako. (#799) * Add testing for genshi and mako. * [Mega-Linter] Apply linters fixes --------- Co-authored-by: umaannamalai Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> commit be4fb3dda0e734889acd6bc53cf91f26c18c2118 Author: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Mon May 1 16:01:09 2023 -0700 Add tests for Waitress (#797) * Change import format * Initial commit * Add more tests to adapter_waitress * Remove commented out code * [Mega-Linter] Apply linters fixes * Add assertions to all tests * Add more NR testing to waitress --------- Co-authored-by: lrafeei Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> commit 7103506ca5639d339e3e47dfb9e4affb546c839b Author: Hannah Stepanek Date: Mon May 1 14:12:31 2023 -0700 Add tests for pyodbc (#796) * Add tests for pyodbc * Move imports into tests to get import coverage * Fixup: remove time import * Trigger tests --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * Squashed commit of the following: commit 7a384c5935f8d6d24db9b488a7e48a6854efedd6 Author: Tim Pansino Date: Thu Jun 1 12:10:59 2023 -0700 Cleaning out agent protocol commit c87d31d5d3a91eb7c584f32f9831fbebc1ffe378 Author: Tim Pansino Date: Thu Jun 1 12:10:46 2023 -0700 Change content-type header commit 5750e546797b16f96e71161f794cb34a253418a6 Author: Tim Pansino Date: Thu Jun 1 12:05:52 2023 -0700 Add common utilities for OTLP * Remove testing logic * Adding metric serialization helpers * Squashed commit of the following: commit a47e209925a210e85bb6c57f0a2efa9e99630b7f Author: Tim Pansino Date: Tue Jun 6 11:11:30 2023 -0700 Commit suggestions from code review commit 1a28d36f86dd3f1fa5ca7a8f56357d168aac69db Author: Tim Pansino Date: Thu May 11 17:28:27 2023 -0700 Cover tags as list not dict commit 71261e3d468320569742a72c690f6ff4e9b3e621 Merge: 459e08567 c2d4629df Author: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Thu May 11 16:59:11 2023 -0700 Merge branch 'main' into feature-dimensional-metrics commit 459e08567102cfadce398b57d245ecf99408400d Author: Tim Pansino Date: Thu May 11 16:57:16 2023 -0700 Add testing for dimensional metrics commit ed33957cd2b20bc1f6e9759a0bad5e4f4a86a38c Author: Tim Pansino Date: Thu May 11 16:56:31 2023 -0700 Add attribute processing to metric identity commit 6caf71ef4386395d950060e0e996f80dbcbfbc32 Author: Tim Pansino Date: Thu May 11 16:56:16 2023 -0700 Add dimensional stats table to stats engine commit 5e1cc9dea6d0d9623130dedd0f787408a8439388 Author: Tim Pansino Date: Wed May 10 16:00:42 2023 -0700 Squashed commit of the following: commit c2d4629dfd7787354b6607160bb952913975d5f7 Author: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Wed May 10 15:59:13 2023 -0700 Add required option for tox v4 (#795) * Add required option for tox v4 * Update tox in GHA * Remove py27 no-cache-dir commit a9636498ab5c20c266fb044a08359c0c9bbcf826 Author: Hannah Stepanek Date: Tue May 9 10:46:39 2023 -0700 Run coverage around pytest (#813) * Run coverage around pytest * Trigger tests * Fixup * Add redis client_no_touch to ignore list * Temporarily remove kafka from coverage * Remove coverage for old libs commit 3d8284540e0acd867c2cf680f43449bc128c0779 Author: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Wed May 3 14:50:30 2023 -0700 Omit some frameworks from coverage analysis (#810) * Omit some frameworks from coverage analysis * Remove commas * Change format of omit * Add relative_files option to coverage * Add absolute directory * Add envsitepackagedir * Add coveragerc file * Add codecov.yml * [Mega-Linter] Apply linters fixes * Revert coveragerc file settings * Add files in packages and more frameworks * Remove commented line --------- Co-authored-by: lrafeei Co-authored-by: Hannah Stepanek commit fd0fa35466b630e34e8476cc53ad0e163564e2de Author: Uma Annamalai Date: Tue May 2 10:55:36 2023 -0700 Add testing for genshi and mako. (#799) * Add testing for genshi and mako. * [Mega-Linter] Apply linters fixes --------- Co-authored-by: umaannamalai Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> commit be4fb3dda0e734889acd6bc53cf91f26c18c2118 Author: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Mon May 1 16:01:09 2023 -0700 Add tests for Waitress (#797) * Change import format * Initial commit * Add more tests to adapter_waitress * Remove commented out code * [Mega-Linter] Apply linters fixes * Add assertions to all tests * Add more NR testing to waitress --------- Co-authored-by: lrafeei Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> commit 7103506ca5639d339e3e47dfb9e4affb546c839b Author: Hannah Stepanek Date: Mon May 1 14:12:31 2023 -0700 Add tests for pyodbc (#796) * Add tests for pyodbc * Move imports into tests to get import coverage * Fixup: remove time import * Trigger tests --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> commit 363122a0efe0ad9f4784fc1f67fda046cb9bb7e8 Author: Hannah Stepanek Date: Mon May 1 13:34:35 2023 -0700 Pin virtualenv, fix pip arg deprecation & disable kafka tests (#803) * Pin virtualenv * Fixup: use 20.21.1 instead * Replace install-options with config-settings See https://github.com/pypa/pip/issues/11358. * Temporarily disable kafka tests commit c2d4629dfd7787354b6607160bb952913975d5f7 Author: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Wed May 10 15:59:13 2023 -0700 Add required option for tox v4 (#795) * Add required option for tox v4 * Update tox in GHA * Remove py27 no-cache-dir commit a9636498ab5c20c266fb044a08359c0c9bbcf826 Author: Hannah Stepanek Date: Tue May 9 10:46:39 2023 -0700 Run coverage around pytest (#813) * Run coverage around pytest * Trigger tests * Fixup * Add redis client_no_touch to ignore list * Temporarily remove kafka from coverage * Remove coverage for old libs commit dc81a50a9fc5f2a5ce6978aa064fdfab1618328b Author: Tim Pansino Date: Sat May 6 14:16:14 2023 -0700 Wiring dimensional metrics commit 3d8284540e0acd867c2cf680f43449bc128c0779 Author: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Wed May 3 14:50:30 2023 -0700 Omit some frameworks from coverage analysis (#810) * Omit some frameworks from coverage analysis * Remove commas * Change format of omit * Add relative_files option to coverage * Add absolute directory * Add envsitepackagedir * Add coveragerc file * Add codecov.yml * [Mega-Linter] Apply linters fixes * Revert coveragerc file settings * Add files in packages and more frameworks * Remove commented line --------- Co-authored-by: lrafeei Co-authored-by: Hannah Stepanek commit fd0fa35466b630e34e8476cc53ad0e163564e2de Author: Uma Annamalai Date: Tue May 2 10:55:36 2023 -0700 Add testing for genshi and mako. (#799) * Add testing for genshi and mako. * [Mega-Linter] Apply linters fixes --------- Co-authored-by: umaannamalai Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> commit be4fb3dda0e734889acd6bc53cf91f26c18c2118 Author: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Mon May 1 16:01:09 2023 -0700 Add tests for Waitress (#797) * Change import format * Initial commit * Add more tests to adapter_waitress * Remove commented out code * [Mega-Linter] Apply linters fixes * Add assertions to all tests * Add more NR testing to waitress --------- Co-authored-by: lrafeei Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> commit 7103506ca5639d339e3e47dfb9e4affb546c839b Author: Hannah Stepanek Date: Mon May 1 14:12:31 2023 -0700 Add tests for pyodbc (#796) * Add tests for pyodbc * Move imports into tests to get import coverage * Fixup: remove time import * Trigger tests --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * Add protobuf to agent features tests * Proper bucket dimensional metric serialization * Wiring up OTLP protocol for metrics * Correct metrics payloads * Make default content-encoding header configurable * Clean up otlp encoding * Expand OTLP metrics testing * Squashed commit of the following: commit 30f0bf5ce27f239f70b236c639a49715f33ce948 Author: Hannah Stepanek Date: Fri Jun 9 16:12:09 2023 -0700 Add OTLP protocol class & protos (#821) * Add protos under packages for otlp * Add common otlp proto payload methods * Add new oltp protocol class * Remove ML event from log message * Remove params, add api-key header & expose path The params are not relevant to OTLP so remove these. The api-key header is how we provide the license key to OTLP so add this. The path to upload dimensional metrics and events are different in OTLP so expose the path so it can be overriden inside the coresponding data_collector methods. * Add otlp_port and otlp_host settings * Default to JSON if protobuf not available & warn * Move otlp_utils to core * Call encode in protocol class * Patch issues with data collector * Move resource to utils & add log proto imports --------- Co-authored-by: Tim Pansino commit e970884dac0e1f9c703c6fdbff408fb923502f51 Author: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Thu Jun 8 13:17:28 2023 -0700 Dimensional Metrics (#815) * Wiring dimensional metrics * Squashed commit of the following: commit c2d4629dfd7787354b6607160bb952913975d5f7 Author: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Wed May 10 15:59:13 2023 -0700 Add required option for tox v4 (#795) * Add required option for tox v4 * Update tox in GHA * Remove py27 no-cache-dir commit a9636498ab5c20c266fb044a08359c0c9bbcf826 Author: Hannah Stepanek Date: Tue May 9 10:46:39 2023 -0700 Run coverage around pytest (#813) * Run coverage around pytest * Trigger tests * Fixup * Add redis client_no_touch to ignore list * Temporarily remove kafka from coverage * Remove coverage for old libs commit 3d8284540e0acd867c2cf680f43449bc128c0779 Author: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Wed May 3 14:50:30 2023 -0700 Omit some frameworks from coverage analysis (#810) * Omit some frameworks from coverage analysis * Remove commas * Change format of omit * Add relative_files option to coverage * Add absolute directory * Add envsitepackagedir * Add coveragerc file * Add codecov.yml * [Mega-Linter] Apply linters fixes * Revert coveragerc file settings * Add files in packages and more frameworks * Remove commented line --------- Co-authored-by: lrafeei Co-authored-by: Hannah Stepanek commit fd0fa35466b630e34e8476cc53ad0e163564e2de Author: Uma Annamalai Date: Tue May 2 10:55:36 2023 -0700 Add testing for genshi and mako. (#799) * Add testing for genshi and mako. * [Mega-Linter] Apply linters fixes --------- Co-authored-by: umaannamalai Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> commit be4fb3dda0e734889acd6bc53cf91f26c18c2118 Author: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Mon May 1 16:01:09 2023 -0700 Add tests for Waitress (#797) * Change import format * Initial commit * Add more tests to adapter_waitress * Remove commented out code * [Mega-Linter] Apply linters fixes * Add assertions to all tests * Add more NR testing to waitress --------- Co-authored-by: lrafeei Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> commit 7103506ca5639d339e3e47dfb9e4affb546c839b Author: Hannah Stepanek Date: Mon May 1 14:12:31 2023 -0700 Add tests for pyodbc (#796) * Add tests for pyodbc * Move imports into tests to get import coverage * Fixup: remove time import * Trigger tests --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> commit 363122a0efe0ad9f4784fc1f67fda046cb9bb7e8 Author: Hannah Stepanek Date: Mon May 1 13:34:35 2023 -0700 Pin virtualenv, fix pip arg deprecation & disable kafka tests (#803) * Pin virtualenv * Fixup: use 20.21.1 instead * Replace install-options with config-settings See https://github.com/pypa/pip/issues/11358. * Temporarily disable kafka tests * Add dimensional stats table to stats engine * Add attribute processing to metric identity * Add testing for dimensional metrics * Cover tags as list not dict * Commit suggestions from code review * Fix missing resource error * Add global settings override for otlp_host test * Fix unbound local variable * Remove redundant and miscategorized tests * Migrate and merge otlp utils to core. * Fix virtualenv for newer tox versions and Py27 * Fix validator for Py27 * Fix dimensional metric normalization * Fix lint errors * Fix pypy 27 naming * Add debug override for metric serialization * Fix exit code passthrough in tox script * Make otlp_encode more robust * Add json vs protobuf testing fixture * Remove sklearn py27 testing * Validate resource in OTLP * Revert unrelated changes from code review * Fixup: service.provider assertion --------- Co-authored-by: Hannah Stepanek --- .github/scripts/retry.sh | 2 +- newrelic/config.py | 1 + newrelic/core/config.py | 1 + newrelic/core/data_collector.py | 12 +- newrelic/core/otlp_utils.py | 197 ++++++++++++++---- newrelic/core/stats_engine.py | 149 ++++++++----- .../test_dimensional_metrics.py | 127 +++++++++-- .../test_metric_normalization.py | 78 +++++++ .../validate_dimensional_metric_payload.py | 187 +++++++++++++++++ ...dimensional_metrics_outside_transaction.py | 16 +- .../validate_transaction_metrics.py | 21 +- tox.ini | 6 +- 12 files changed, 665 insertions(+), 132 deletions(-) create mode 100644 tests/agent_features/test_metric_normalization.py create mode 100644 tests/testing_support/validators/validate_dimensional_metric_payload.py diff --git a/.github/scripts/retry.sh b/.github/scripts/retry.sh index 1cb17836e..f4aaca39b 100755 --- a/.github/scripts/retry.sh +++ b/.github/scripts/retry.sh @@ -25,4 +25,4 @@ for i in $(seq 1 $retries); do done # Exit with status code of wrapped command -exit $? +exit $result diff --git a/newrelic/config.py b/newrelic/config.py index 3749f81bd..7b5da4dd4 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -495,6 +495,7 @@ def _process_configuration(section): _process_setting(section, "debug.disable_certificate_validation", "getboolean", None) _process_setting(section, "debug.disable_harvest_until_shutdown", "getboolean", None) _process_setting(section, "debug.connect_span_stream_in_developer_mode", "getboolean", None) + _process_setting(section, "debug.otlp_content_encoding", "get", None) _process_setting(section, "cross_application_tracer.enabled", "getboolean", None) _process_setting(section, "message_tracer.segment_parameters_enabled", "getboolean", None) _process_setting(section, "process_host.display_name", "get", None) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 55b359174..6f32ea80b 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -851,6 +851,7 @@ def default_otlp_host(host): _settings.debug.log_untrusted_distributed_trace_keys = False _settings.debug.disable_harvest_until_shutdown = False _settings.debug.connect_span_stream_in_developer_mode = False +_settings.debug.otlp_content_encoding = None _settings.message_tracer.segment_parameters_enabled = True diff --git a/newrelic/core/data_collector.py b/newrelic/core/data_collector.py index 441f587dc..0df6fc3fe 100644 --- a/newrelic/core/data_collector.py +++ b/newrelic/core/data_collector.py @@ -32,6 +32,7 @@ ) from newrelic.core.agent_streaming import StreamingRpc from newrelic.core.config import global_settings +from newrelic.core.otlp_utils import encode_metric_data _logger = logging.getLogger(__name__) @@ -152,17 +153,12 @@ def send_dimensional_metric_data(self, start_time, end_time, metric_data): specific metrics. NOTE: This data is sent not sent to the normal agent endpoints but is sent - to the MELT API endpoints to keep the entity separate. This is for use + to the OTLP API endpoints to keep the entity separate. This is for use with the machine learning integration only. """ - payload = (self.agent_run_id, start_time, end_time, metric_data) - # return self._protocol.send("metric_data", payload) - - # TODO: REMOVE THIS. Replace with actual protocol. - DIMENSIONAL_METRIC_DATA_TEMP.append(payload) - _logger.debug("Dimensional Metrics: %r" % metric_data) - return 200 + payload = encode_metric_data(metric_data, start_time, end_time) + return self._otlp_protocol.send("dimensional_metric_data", payload, path="/v1/metrics") def send_log_events(self, sampling_info, log_event_data): """Called to submit sample set for log events.""" diff --git a/newrelic/core/otlp_utils.py b/newrelic/core/otlp_utils.py index 6a44cb4e3..cd0328bcc 100644 --- a/newrelic/core/otlp_utils.py +++ b/newrelic/core/otlp_utils.py @@ -12,66 +12,87 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""This module provides common utilities for interacting with OTLP protocol buffers.""" +""" +This module provides common utilities for interacting with OTLP protocol buffers. + +The serialization implemented here attempts to use protobuf as an encoding, but falls +back to JSON when encoutering exceptions unless the content type is explicitly set in debug settings. +""" import logging -_logger = logging.getLogger(__name__) +from newrelic.common.encoding_utils import json_encode +from newrelic.core.stats_engine import CountStats, TimeStats +from newrelic.core.config import global_settings -try: - from newrelic.packages.opentelemetry_proto.common_pb2 import AnyValue, KeyValue - from newrelic.packages.opentelemetry_proto.logs_pb2 import ( - LogRecord, - ResourceLogs, - ScopeLogs, - ) - from newrelic.packages.opentelemetry_proto.metrics_pb2 import ( - AggregationTemporality, - Metric, - MetricsData, - NumberDataPoint, - ResourceMetrics, - ScopeMetrics, - Sum, - Summary, - SummaryDataPoint, - ) - from newrelic.packages.opentelemetry_proto.resource_pb2 import Resource +_logger = logging.getLogger(__name__) - AGGREGATION_TEMPORALITY_DELTA = AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA - ValueAtQuantile = SummaryDataPoint.ValueAtQuantile +_settings = global_settings() +otlp_content_setting = _settings.debug.otlp_content_encoding +if not otlp_content_setting or otlp_content_setting == "protobuf": + try: + from newrelic.packages.opentelemetry_proto.common_pb2 import AnyValue, KeyValue + from newrelic.packages.opentelemetry_proto.logs_pb2 import ( + LogRecord, + ResourceLogs, + ScopeLogs, + ) + from newrelic.packages.opentelemetry_proto.metrics_pb2 import ( + AggregationTemporality, + Metric, + MetricsData, + NumberDataPoint, + ResourceMetrics, + ScopeMetrics, + Sum, + Summary, + SummaryDataPoint, + ) + from newrelic.packages.opentelemetry_proto.resource_pb2 import Resource - otlp_encode = lambda payload: payload.SerializeToString() - OTLP_CONTENT_TYPE = "application/x-protobuf" + ValueAtQuantile = SummaryDataPoint.ValueAtQuantile + AGGREGATION_TEMPORALITY_DELTA = AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA + OTLP_CONTENT_TYPE = "application/x-protobuf" -except ImportError: - from newrelic.common.encoding_utils import json_encode + otlp_content_setting = "protobuf" # Explicitly set to overwrite None values + except Exception: + if otlp_content_setting == "protobuf": + raise # Reraise exception if content type explicitly set + else: # Fallback to JSON + otlp_content_setting = "json" - def otlp_encode(*args, **kwargs): - _logger.warn( - "Using OTLP integration while protobuf is not installed. This may result in larger payload sizes and data loss." - ) - return json_encode(*args, **kwargs) - Resource = dict - ValueAtQuantile = dict +if otlp_content_setting == "json": AnyValue = dict KeyValue = dict - NumberDataPoint = dict - SummaryDataPoint = dict - Sum = dict - Summary = dict Metric = dict MetricsData = dict - ScopeMetrics = dict + NumberDataPoint = dict + Resource = dict ResourceMetrics = dict - AGGREGATION_TEMPORALITY_DELTA = 1 + ScopeMetrics = dict + Sum = dict + Summary = dict + SummaryDataPoint = dict + ValueAtQuantile = dict ResourceLogs = dict ScopeLogs = dict LogRecord = dict + + AGGREGATION_TEMPORALITY_DELTA = 1 OTLP_CONTENT_TYPE = "application/json" +def otlp_encode(payload): + if type(payload) is dict: + _logger.warning( + "Using OTLP integration while protobuf is not installed. This may result in larger payload sizes and data loss." + ) + return json_encode(payload) + else: + return payload.SerializeToString() + + def create_key_value(key, value): if isinstance(value, bool): return KeyValue(key=key, value=AnyValue(bool_value=value)) @@ -85,11 +106,13 @@ def create_key_value(key, value): # those are not valid custom attribute types according to our api spec, # we will not bother to support them here either. else: - _logger.warn("Unsupported attribute value type %s: %s." % (key, value)) + _logger.warning("Unsupported attribute value type %s: %s." % (key, value)) def create_key_values_from_iterable(iterable): - if isinstance(iterable, dict): + if not iterable: + return None + elif isinstance(iterable, dict): iterable = iterable.items() # The create_key_value list may return None if the value is an unsupported type @@ -103,5 +126,93 @@ def create_key_values_from_iterable(iterable): def create_resource(attributes=None): - attributes = attributes or {"instrumentation.provider": "nr_performance_monitoring"} + attributes = attributes or {"instrumentation.provider": "newrelic-opentelemetry-python-ml"} return Resource(attributes=create_key_values_from_iterable(attributes)) + + +def TimeStats_to_otlp_data_point(self, start_time, end_time, attributes=None): + data = SummaryDataPoint( + time_unix_nano=int(end_time * 1e9), # Time of current harvest + start_time_unix_nano=int(start_time * 1e9), # Time of last harvest + attributes=attributes, + count=int(self[0]), + sum=float(self[1]), + quantile_values=[ + ValueAtQuantile(quantile=0.0, value=float(self[3])), # Min Value + ValueAtQuantile(quantile=1.0, value=float(self[4])), # Max Value + ], + ) + return data + + +def CountStats_to_otlp_data_point(self, start_time, end_time, attributes=None): + data = NumberDataPoint( + time_unix_nano=int(end_time * 1e9), # Time of current harvest + start_time_unix_nano=int(start_time * 1e9), # Time of last harvest + attributes=attributes, + as_int=int(self[0]), + ) + return data + + +def stats_to_otlp_metrics(metric_data, start_time, end_time): + """ + Generator producing protos for Summary and Sum metrics, for CountStats and TimeStats respectively. + + Individual Metric protos must be entirely one type of metric data point. For mixed metric types we have to + separate the types and report multiple metrics, one for each type. + """ + for name, metric_container in metric_data: + if any(isinstance(metric, CountStats) for metric in metric_container.values()): + # Metric contains Sum metric data points. + yield Metric( + name=name, + sum=Sum( + aggregation_temporality=AGGREGATION_TEMPORALITY_DELTA, + is_monotonic=True, + data_points=[ + CountStats_to_otlp_data_point( + value, + start_time=start_time, + end_time=end_time, + attributes=create_key_values_from_iterable(tags), + ) + for tags, value in metric_container.items() + if isinstance(value, CountStats) + ], + ), + ) + if any(isinstance(metric, TimeStats) for metric in metric_container.values()): + # Metric contains Summary metric data points. + yield Metric( + name=name, + summary=Summary( + data_points=[ + TimeStats_to_otlp_data_point( + value, + start_time=start_time, + end_time=end_time, + attributes=create_key_values_from_iterable(tags), + ) + for tags, value in metric_container.items() + if isinstance(value, TimeStats) + ] + ), + ) + + +def encode_metric_data(metric_data, start_time, end_time, resource=None, scope=None): + resource = resource or create_resource() + return MetricsData( + resource_metrics=[ + ResourceMetrics( + resource=resource, + scope_metrics=[ + ScopeMetrics( + scope=scope, + metrics=list(stats_to_otlp_metrics(metric_data, start_time, end_time)), + ) + ], + ) + ] + ) diff --git a/newrelic/core/stats_engine.py b/newrelic/core/stats_engine.py index 9d59efd49..615f2b11a 100644 --- a/newrelic/core/stats_engine.py +++ b/newrelic/core/stats_engine.py @@ -188,6 +188,7 @@ def merge_dimensional_metric(self, value): class CountStats(TimeStats): + def merge_stats(self, other): self[0] += other[0] @@ -240,34 +241,97 @@ def reset_metric_stats(self): """ self.__stats_table = {} -class DimensionalMetrics(CustomMetrics): +class DimensionalMetrics(object): + + """Nested dictionary table for collecting a set of metrics broken down by tags.""" - """Extends CustomMetrics to allow a set of tags for metrics.""" + def __init__(self): + self.__stats_table = {} def __contains__(self, key): - if not isinstance(key[1], frozenset): - # Convert tags dict to a frozen set for proper comparisons - key = create_metric_identity(*key) - return key in self.__stats_table + if isinstance(key, tuple): + if not isinstance(key[1], frozenset): + # Convert tags dict to a frozen set for proper comparisons + name, tags = create_metric_identity(*key) + else: + name, tags = key + + # Check that both metric name and tags are already present. + stats_container = self.__stats_table.get(name) + return stats_container and tags in stats_container + else: + # Only look for metric name + return key in self.__stats_table def record_dimensional_metric(self, name, value, tags=None): """Record a single value metric, merging the data with any data - from prior value metrics with the same name. + from prior value metrics with the same name and tags. + """ + name, tags = create_metric_identity(name, tags) + + if isinstance(value, dict): + if len(value) == 1 and "count" in value: + new_stats = CountStats(call_count=value["count"]) + else: + new_stats = TimeStats(*c2t(**value)) + else: + new_stats = TimeStats(1, value, value, value, value, value**2) + + stats_container = self.__stats_table.get(name) + if stats_container is None: + # No existing metrics with this name. Set up new stats container. + self.__stats_table[name] = {tags: new_stats} + else: + # Existing metric container found. + stats = stats_container.get(tags) + if stats is None: + # No data points for this set of tags. Add new data. + stats_container[tags] = new_stats + else: + # Existing data points found, merge stats. + stats.merge_stats(new_stats) + + return (name, tags) + def metrics(self): + """Returns an iterator over the set of value metrics. + The items returned are a dictionary of tags for each metric value. + Metric values are each a tuple consisting of the metric name and accumulated + stats for the metric. """ - key = create_metric_identity(name, tags) - self.record_custom_metric(key, value) + return six.iteritems(self.__stats_table) -class DimensionalStatsTable(dict): + def metrics_count(self): + """Returns a count of the number of unique metrics currently + recorded for apdex, time and value metrics. + """ - """Extends dict to coerce a set of tags to a hashable identity.""" + return sum(len(metric) for metric in self.__stats_table.values()) - def __contains__(self, key): - if key[1] is not None and not isinstance(key[1], frozenset): - # Convert tags dict to a frozen set for proper comparisons - key = create_metric_identity(*key) - return super(DimensionalStatsTable, self).__contains__(key) + def reset_metric_stats(self): + """Resets the accumulated statistics back to initial state for + metric data. + """ + self.__stats_table = {} + + def get(self, key, default=None): + return self.__stats_table.get(key, default) + + def __setitem__(self, key, value): + self.__stats_table[key] = value + + def __getitem__(self, key): + return self.__stats_table[key] + + def __str__(self): + return str(self.__stats_table) + + def __repr__(self): + return "%s(%s)" % (__class__.__name__, repr(self.__stats_table)) + + def items(self): + return self.metrics() class SlowSqlStats(list): @@ -468,7 +532,7 @@ class StatsEngine(object): def __init__(self): self.__settings = None self.__stats_table = {} - self.__dimensional_stats_table = DimensionalStatsTable() + self.__dimensional_stats_table = DimensionalMetrics() self._transaction_events = SampledDataSet() self._error_events = SampledDataSet() self._custom_events = SampledDataSet() @@ -539,7 +603,7 @@ def metrics_count(self): """ - return len(self.__stats_table) + len(self.__dimensional_stats_table) + return len(self.__stats_table) + self.__dimensional_stats_table.metrics_count() def record_apdex_metric(self, metric): """Record a single apdex metric, merging the data with any data @@ -929,25 +993,9 @@ def record_custom_metrics(self, metrics): def record_dimensional_metric(self, name, value, tags=None): """Record a single value metric, merging the data with any data - from prior value metrics with the same name. - + from prior value metrics with the same name and tags. """ - if isinstance(value, dict): - if len(value) == 1 and "count" in value: - new_stats = CountStats(call_count=value["count"]) - else: - new_stats = TimeStats(*c2t(**value)) - else: - new_stats = TimeStats(1, value, value, value, value, value**2) - - key = create_metric_identity(name, tags) - stats = self.__dimensional_stats_table.get(key) - if stats is None: - self.__dimensional_stats_table[key] = new_stats - else: - stats.merge_stats(new_stats) - - return key + return self.__dimensional_stats_table.record_dimensional_metric(name, value, tags) def record_dimensional_metrics(self, metrics): """Record the value metrics supplied by the iterable, merging @@ -1292,12 +1340,12 @@ def dimensional_metric_data(self, normalizer=None): _logger.info( "Raw dimensional metric data for harvest of %r is %r.", self.__settings.app_name, - list(six.iteritems(self.__dimensional_stats_table)), + list(self.__dimensional_stats_table.metrics()), ) if normalizer is not None: - for key, value in six.iteritems(self.__dimensional_stats_table): - key = (normalizer(key[0])[0], key[1]) + for key, value in self.__dimensional_stats_table.metrics(): + key = normalizer(key)[0] stats = normalized_stats.get(key) if stats is None: normalized_stats[key] = copy.copy(value) @@ -1310,11 +1358,10 @@ def dimensional_metric_data(self, normalizer=None): _logger.info( "Normalized metric data for harvest of %r is %r.", self.__settings.app_name, - list(six.iteritems(normalized_stats)), + list(normalized_stats.metrics()), ) - for key, value in six.iteritems(normalized_stats): - key = dict(name=key[0], scope=key[1]) + for key, value in normalized_stats.items(): result.append((key, value)) return result @@ -1325,7 +1372,7 @@ def dimensional_metric_data_count(self): if not self.__settings: return 0 - return len(self.__dimensional_stats_table) + return self.__dimensional_stats_table.metrics_count() def error_data(self): """Returns a to a list containing any errors collected during @@ -1604,8 +1651,6 @@ def reset_stats(self, settings, reset_stream=False): """ self.__settings = settings - self.__stats_table = {} - self.__dimensional_stats_table = {} self.__sql_stats_table = {} self.__slow_transaction = None self.__slow_transaction_map = {} @@ -1613,6 +1658,7 @@ def reset_stats(self, settings, reset_stream=False): self.__transaction_errors = [] self.__synthetics_transactions = [] + self.reset_metric_stats() self.reset_transaction_events() self.reset_error_events() self.reset_custom_events() @@ -1633,7 +1679,7 @@ def reset_metric_stats(self): """ self.__stats_table = {} - self.__dimensional_stats_table = {} + self.__dimensional_stats_table.reset_metric_stats() def reset_transaction_events(self): """Resets the accumulated statistics back to initial state for @@ -1982,11 +2028,16 @@ def merge_dimensional_metrics(self, metrics): return for key, other in metrics: - stats = self.__dimensional_stats_table.get(key) - if not stats: + stats_container = self.__dimensional_stats_table.get(key) + if not stats_container: self.__dimensional_stats_table[key] = other else: - stats.merge_stats(other) + for tags, other_value in other.items(): + stats = stats_container.get(tags) + if not stats: + stats_container[tags] = other_value + else: + stats.merge_stats(other_value) def _snapshot(self): copy = object.__new__(StatsEngineSnapshot) diff --git a/tests/agent_features/test_dimensional_metrics.py b/tests/agent_features/test_dimensional_metrics.py index 82ddfad89..b1f746a81 100644 --- a/tests/agent_features/test_dimensional_metrics.py +++ b/tests/agent_features/test_dimensional_metrics.py @@ -13,15 +13,48 @@ # limitations under the License. import pytest +from testing_support.fixtures import reset_core_stats_engine +from testing_support.validators.validate_dimensional_metric_payload import ( + validate_dimensional_metric_payload, +) +from testing_support.validators.validate_dimensional_metrics_outside_transaction import ( + validate_dimensional_metrics_outside_transaction, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) from newrelic.api.application import application_instance from newrelic.api.background_task import background_task -from newrelic.api.transaction import record_dimensional_metric, record_dimensional_metrics +from newrelic.api.transaction import ( + record_dimensional_metric, + record_dimensional_metrics, +) from newrelic.common.metric_utils import create_metric_identity + +import newrelic.core.otlp_utils +from newrelic.core.config import global_settings + + +try: + # python 2.x + reload +except NameError: + # python 3.x + from importlib import reload -from testing_support.fixtures import reset_core_stats_engine -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics -from testing_support.validators.validate_dimensional_metrics_outside_transaction import validate_dimensional_metrics_outside_transaction + +@pytest.fixture(scope="module", autouse=True, params=["protobuf", "json"]) +def otlp_content_encoding(request): + _settings = global_settings() + prev = _settings.debug.otlp_content_encoding + _settings.debug.otlp_content_encoding = request.param + reload(newrelic.core.otlp_utils) + assert newrelic.core.otlp_utils.otlp_content_setting == request.param, "Content encoding mismatch." + + yield + + _settings.debug.otlp_content_encoding = prev _test_tags_examples = [ @@ -47,10 +80,15 @@ def test_create_metric_identity(tags, expected): @pytest.mark.parametrize("tags,expected", _test_tags_examples) +@reset_core_stats_engine() def test_record_dimensional_metric_inside_transaction(tags, expected): - @validate_transaction_metrics("test_record_dimensional_metric_inside_transaction", background_task=True, dimensional_metrics=[ - ("Metric", expected, 1), - ]) + @validate_transaction_metrics( + "test_record_dimensional_metric_inside_transaction", + background_task=True, + dimensional_metrics=[ + ("Metric", expected, 1), + ], + ) @background_task(name="test_record_dimensional_metric_inside_transaction") def _test(): record_dimensional_metric("Metric", 1, tags=tags) @@ -70,11 +108,16 @@ def _test(): @pytest.mark.parametrize("tags,expected", _test_tags_examples) +@reset_core_stats_engine() def test_record_dimensional_metrics_inside_transaction(tags, expected): - @validate_transaction_metrics("test_record_dimensional_metrics_inside_transaction", background_task=True, dimensional_metrics=[("Metric/1", expected, 1), ("Metric/2", expected, 1)]) + @validate_transaction_metrics( + "test_record_dimensional_metrics_inside_transaction", + background_task=True, + dimensional_metrics=[("Metric.1", expected, 1), ("Metric.2", expected, 1)], + ) @background_task(name="test_record_dimensional_metrics_inside_transaction") def _test(): - record_dimensional_metrics([("Metric/1", 1, tags), ("Metric/2", 1, tags)]) + record_dimensional_metrics([("Metric.1", 1, tags), ("Metric.2", 1, tags)]) _test() @@ -82,25 +125,71 @@ def _test(): @pytest.mark.parametrize("tags,expected", _test_tags_examples) @reset_core_stats_engine() def test_record_dimensional_metrics_outside_transaction(tags, expected): - @validate_dimensional_metrics_outside_transaction([("Metric/1", expected, 1), ("Metric/2", expected, 1)]) + @validate_dimensional_metrics_outside_transaction([("Metric.1", expected, 1), ("Metric.2", expected, 1)]) def _test(): app = application_instance() - record_dimensional_metrics([("Metric/1", 1, tags), ("Metric/2", 1, tags)], application=app) + record_dimensional_metrics([("Metric.1", 1, tags), ("Metric.2", 1, tags)], application=app) _test() +@reset_core_stats_engine() def test_dimensional_metrics_different_tags(): - @validate_transaction_metrics("test_dimensional_metrics_different_tags", background_task=True, dimensional_metrics=[ - ("Metric", frozenset({("tag", 1)}), 1), - ("Metric", frozenset({("tag", 2)}), 2), - ]) + @validate_transaction_metrics( + "test_dimensional_metrics_different_tags", + background_task=True, + dimensional_metrics=[ + ("Metric", frozenset({("tag", 1)}), 1), + ("Metric", frozenset({("tag", 2)}), 2), + ], + ) @background_task(name="test_dimensional_metrics_different_tags") def _test(): - record_dimensional_metrics([ - ("Metric", 1, {"tag": 1}), - ("Metric", 1, {"tag": 2}), - ]) + record_dimensional_metrics( + [ + ("Metric", 1, {"tag": 1}), + ("Metric", 1, {"tag": 2}), + ] + ) record_dimensional_metric("Metric", 1, {"tag": 2}) _test() + + +@reset_core_stats_engine() +@validate_dimensional_metric_payload( + summary_metrics=[ + ("Metric.Summary", {"tag": 1}, 1), + ("Metric.Summary", {"tag": 2}, 1), + ("Metric.Summary", None, 1), + ("Metric.Mixed", {"tag": 1}, 1), + ("Metric.NotPresent", None, None), + ], + count_metrics=[ + ("Metric.Count", {"tag": 1}, 1), + ("Metric.Count", {"tag": 2}, 2), + ("Metric.Count", None, 3), + ("Metric.Mixed", {"tag": 2}, 2), + ("Metric.NotPresent", None, None), + ], +) +def test_dimensional_metric_payload(): + @background_task(name="test_dimensional_metric_payload") + def _test(): + record_dimensional_metrics( + [ + ("Metric.Summary", 1, {"tag": 1}), + ("Metric.Summary", 2, {"tag": 2}), + ("Metric.Summary", 3), # No tags + ("Metric.Count", {"count": 1}, {"tag": 1}), + ("Metric.Count", {"count": 2}, {"tag": 2}), + ("Metric.Count", {"count": 3}), # No tags + ("Metric.Mixed", 1, {"tag": 1}), + ("Metric.Mixed", {"count": 2}, {"tag": 2}), + ] + ) + + _test() + app = application_instance() + core_app = app._agent.application(app.name) + core_app.harvest() diff --git a/tests/agent_features/test_metric_normalization.py b/tests/agent_features/test_metric_normalization.py new file mode 100644 index 000000000..65f2903ae --- /dev/null +++ b/tests/agent_features/test_metric_normalization.py @@ -0,0 +1,78 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from testing_support.fixtures import reset_core_stats_engine +from testing_support.validators.validate_dimensional_metric_payload import ( + validate_dimensional_metric_payload, +) +from testing_support.validators.validate_metric_payload import validate_metric_payload + +from newrelic.api.application import application_instance +from newrelic.api.background_task import background_task +from newrelic.api.transaction import record_custom_metric, record_dimensional_metric +from newrelic.core.rules_engine import NormalizationRule, RulesEngine + +RULES = [{"match_expression": "(replace)", "replacement": "expected", "ignore": False, "eval_order": 0}] +EXPECTED_TAGS = frozenset({"tag": 1}.items()) + + +def _prepare_rules(test_rules): + # ensure all keys are present, if not present set to an empty string + for rule in test_rules: + for key in NormalizationRule._fields: + rule[key] = rule.get(key, "") + return test_rules + + +@pytest.fixture(scope="session") +def core_app(collector_agent_registration): + app = collector_agent_registration + return app._agent.application(app.name) + + +@pytest.fixture(scope="function") +def rules_engine_fixture(core_app): + rules_engine = core_app._rules_engine + previous_rules = rules_engine["metric"] + + rules_engine["metric"] = RulesEngine(_prepare_rules(RULES)) + yield + rules_engine["metric"] = previous_rules # Restore after test run + + +@validate_dimensional_metric_payload(summary_metrics=[("Metric/expected", EXPECTED_TAGS, 1)]) +@validate_metric_payload([("Metric/expected", 1)]) +@reset_core_stats_engine() +def test_metric_normalization_inside_transaction(core_app, rules_engine_fixture): + @background_task(name="test_record_dimensional_metric_inside_transaction") + def _test(): + record_dimensional_metric("Metric/replace", 1, tags={"tag": 1}) + record_custom_metric("Metric/replace", 1) + + _test() + core_app.harvest() + + +@validate_dimensional_metric_payload(summary_metrics=[("Metric/expected", EXPECTED_TAGS, 1)]) +@validate_metric_payload([("Metric/expected", 1)]) +@reset_core_stats_engine() +def test_metric_normalization_outside_transaction(core_app, rules_engine_fixture): + def _test(): + app = application_instance() + record_dimensional_metric("Metric/replace", 1, tags={"tag": 1}, application=app) + record_custom_metric("Metric/replace", 1, application=app) + + _test() + core_app.harvest() diff --git a/tests/testing_support/validators/validate_dimensional_metric_payload.py b/tests/testing_support/validators/validate_dimensional_metric_payload.py new file mode 100644 index 000000000..58523e2fd --- /dev/null +++ b/tests/testing_support/validators/validate_dimensional_metric_payload.py @@ -0,0 +1,187 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from newrelic.common.object_wrapper import function_wrapper, transient_function_wrapper +from newrelic.core.otlp_utils import otlp_content_setting + +if otlp_content_setting == "protobuf": + from google.protobuf.json_format import MessageToDict +else: + MessageToDict = None + + +def data_points_to_dict(data_points): + return { + frozenset( + {attr["key"]: attribute_to_value(attr["value"]) for attr in (data_point.get("attributes") or [])}.items() + ) + or None: data_point + for data_point in data_points + } + + +def attribute_to_value(attribute): + attribute_type, attribute_value = next(iter(attribute.items())) + if attribute_type == "int_value": + return int(attribute_value) + elif attribute_type == "double_value": + return float(attribute_value) + elif attribute_type == "bool_value": + return bool(attribute_value) + elif attribute_type == "str_value": + return str(attribute_value) + else: + raise TypeError("Invalid attribute type: %s" % attribute_type) + + +def payload_to_metrics(payload): + if type(payload) is not dict: + message = MessageToDict(payload, use_integers_for_enums=True, preserving_proto_field_name=True) + else: + message = payload + + resource_metrics = message.get("resource_metrics") + assert len(resource_metrics) == 1 + resource_metrics = resource_metrics[0] + + resource = resource_metrics.get("resource") + assert resource and resource.get("attributes")[0] == { + "key": "instrumentation.provider", + "value": {"string_value": "newrelic-opentelemetry-python-ml"}, + } + scope_metrics = resource_metrics.get("scope_metrics") + assert len(scope_metrics) == 1 + scope_metrics = scope_metrics[0] + + scope = scope_metrics.get("scope") + assert scope is None + metrics = scope_metrics.get("metrics") + + sent_summary_metrics = {} + sent_count_metrics = {} + for metric in metrics: + metric_name = metric["name"] + if metric.get("sum"): + sent_count_metrics[metric_name] = metric + elif metric.get("summary"): + sent_summary_metrics[metric_name] = metric + else: + raise TypeError("Unknown metrics type for metric: %s" % metric) + + return sent_summary_metrics, sent_count_metrics + + +def validate_dimensional_metric_payload(summary_metrics=None, count_metrics=None): + # Validates OTLP metrics as they are sent to the collector. + + summary_metrics = summary_metrics or [] + count_metrics = count_metrics or [] + + @function_wrapper + def _validate_wrapper(wrapped, instance, args, kwargs): + recorded_metrics = [] + + @transient_function_wrapper("newrelic.core.agent_protocol", "OtlpProtocol.send") + def send_request_wrapper(wrapped, instance, args, kwargs): + def _bind_params(method, payload=(), *args, **kwargs): + return method, payload + + method, payload = _bind_params(*args, **kwargs) + + if method == "dimensional_metric_data" and payload: + recorded_metrics.append(payload) + + return wrapped(*args, **kwargs) + + wrapped = send_request_wrapper(wrapped) + val = wrapped(*args, **kwargs) + assert recorded_metrics + + decoded_payloads = [payload_to_metrics(payload) for payload in recorded_metrics] + for sent_summary_metrics, sent_count_metrics in decoded_payloads: + for metric, tags, count in summary_metrics: + if isinstance(tags, dict): + tags = frozenset(tags.items()) + + if not count: + if metric in sent_summary_metrics: + data_points = data_points_to_dict(sent_summary_metrics[metric]["summary"]["data_points"]) + assert tags not in data_points, "(%s, %s) Found." % (metric, tags and dict(tags)) + else: + assert metric in sent_summary_metrics, "%s Not Found. Got: %s" % ( + metric, + list(sent_summary_metrics.keys()), + ) + data_points = data_points_to_dict(sent_summary_metrics[metric]["summary"]["data_points"]) + assert tags in data_points, "(%s, %s) Not Found. Got: %s" % ( + metric, + tags and dict(tags), + list(data_points.keys()), + ) + + # Validate metric format + metric_container = data_points[tags] + for key in ("start_time_unix_nano", "time_unix_nano", "count", "sum", "quantile_values"): + assert key in metric_container, "Invalid metric format. Missing key: %s" % key + quantile_values = metric_container["quantile_values"] + assert len(quantile_values) == 2 # Min and Max + + # Validate metric count + if count != "present": + assert int(metric_container["count"]) == count, "(%s, %s): Expected: %s Got: %s" % ( + metric, + tags and dict(tags), + count, + metric_container["count"], + ) + + for metric, tags, count in count_metrics: + if isinstance(tags, dict): + tags = frozenset(tags.items()) + + if not count: + if metric in sent_count_metrics: + data_points = data_points_to_dict(sent_count_metrics[metric]["sum"]["data_points"]) + assert tags not in data_points, "(%s, %s) Found." % (metric, tags and dict(tags)) + else: + assert metric in sent_count_metrics, "%s Not Found. Got: %s" % ( + metric, + list(sent_count_metrics.keys()), + ) + data_points = data_points_to_dict(sent_count_metrics[metric]["sum"]["data_points"]) + assert tags in data_points, "(%s, %s) Not Found. Got: %s" % ( + metric, + tags and dict(tags), + list(data_points.keys()), + ) + + # Validate metric format + assert sent_count_metrics[metric]["sum"].get("is_monotonic") + assert sent_count_metrics[metric]["sum"].get("aggregation_temporality") == 1 + metric_container = data_points[tags] + for key in ("start_time_unix_nano", "time_unix_nano", "as_int"): + assert key in metric_container, "Invalid metric format. Missing key: %s" % key + + # Validate metric count + if count != "present": + assert int(metric_container["as_int"]) == count, "(%s, %s): Expected: %s Got: %s" % ( + metric, + tags and dict(tags), + count, + metric_container["count"], + ) + + return val + + return _validate_wrapper diff --git a/tests/testing_support/validators/validate_dimensional_metrics_outside_transaction.py b/tests/testing_support/validators/validate_dimensional_metrics_outside_transaction.py index 7a3272bad..2854a7478 100644 --- a/tests/testing_support/validators/validate_dimensional_metrics_outside_transaction.py +++ b/tests/testing_support/validators/validate_dimensional_metrics_outside_transaction.py @@ -36,11 +36,11 @@ def _validate_dimensional_metrics_outside_transaction(wrapped, instance, args, k except: raise else: - metrics = instance.dimensional_stats_table + metrics = instance.dimensional_stats_table.metrics() # Record a copy of the metric value so that the values aren't # merged in the future _metrics = {} - for k, v in metrics.items(): + for k, v in metrics: _metrics[k] = copy.copy(v) recorded_metrics[0] = _metrics @@ -48,13 +48,19 @@ def _validate_dimensional_metrics_outside_transaction(wrapped, instance, args, k def _validate(metrics, name, tags, count): key = (name, tags) - metric = metrics.get(key) + # Dimensional metric lookup + metric_container = metrics.get(name, {}) + metric = metric_container.get(tags) def _metrics_table(): out = [""] out.append("Expected: {0}: {1}".format(key, count)) - for metric_key, metric_value in metrics.items(): - out.append("{0}: {1}".format(metric_key, metric_value[0])) + for metric_key, metric_container in metrics.items(): + if isinstance(metric_container, dict): + for metric_tags, metric_value in metric_container.items(): + out.append("{0}: {1}".format((metric_key, metric_tags), metric_value[0])) + else: + out.append("{0}: {1}".format(metric_key, metric_container[0])) return "\n".join(out) def _metric_details(): diff --git a/tests/testing_support/validators/validate_transaction_metrics.py b/tests/testing_support/validators/validate_transaction_metrics.py index 63c5b3551..0cb569d29 100644 --- a/tests/testing_support/validators/validate_transaction_metrics.py +++ b/tests/testing_support/validators/validate_transaction_metrics.py @@ -77,11 +77,11 @@ def _validate_transaction_metrics(wrapped, instance, args, kwargs): _metrics[k] = copy.copy(v) recorded_metrics.append(_metrics) - metrics = instance.dimensional_stats_table + metrics = instance.dimensional_stats_table.metrics() # Record a copy of the metric value so that the values aren't # merged in the future _metrics = {} - for k, v in metrics.items(): + for k, v in metrics: _metrics[k] = copy.copy(v) recorded_dimensional_metrics.append(_metrics) @@ -89,13 +89,24 @@ def _validate_transaction_metrics(wrapped, instance, args, kwargs): def _validate(metrics, name, scope, count): key = (name, scope) - metric = metrics.get(key) + + if isinstance(scope, str): + # Normal metric lookup + metric = metrics.get(key) + else: + # Dimensional metric lookup + metric_container = metrics.get(name, {}) + metric = metric_container.get(scope) def _metrics_table(): out = [""] out.append("Expected: {0}: {1}".format(key, count)) - for metric_key, metric_value in metrics.items(): - out.append("{0}: {1}".format(metric_key, metric_value[0])) + for metric_key, metric_container in metrics.items(): + if isinstance(metric_container, dict): + for metric_tags, metric_value in metric_container.items(): + out.append("{0}: {1}".format((metric_key, metric_tags), metric_value[0])) + else: + out.append("{0}: {1}".format(metric_key, metric_container[0])) return "\n".join(out) def _metric_details(): diff --git a/tox.ini b/tox.ini index 4723a5144..f4c354deb 100644 --- a/tox.ini +++ b/tox.ini @@ -40,6 +40,7 @@ ; - python-adapter_gevent-py27 [tox] +requires = virtualenv<20.22.0 setupdir = {toxinidir} envlist = python-adapter_cheroot-{py37,py38,py39,py310,py311}, @@ -63,7 +64,6 @@ envlist = python-application_celery-{py37,py38,py39,py310,py311,pypy37}, python-mlmodel_sklearn-{py38,py39,py310,py311}-scikitlearnlatest, python-mlmodel_sklearn-{py37,pypy37}-scikitlearn0101, - python-mlmodel_sklearn-{py27,pypy27}-scikitlearn0020, python-component_djangorestframework-{py37,py38,py39,py310,py311}-djangorestframeworklatest, python-component_flask_rest-{py37,py38,py39,pypy37}-flaskrestxlatest, python-component_graphqlserver-{py37,py38,py39,py310,py311}, @@ -188,14 +188,16 @@ deps = adapter_waitress-waitress02: waitress<2.1 adapter_waitress-waitresslatest: waitress agent_features: beautifulsoup4 + agent_features-{py37,py38,py39,py310,py311,pypy37}: protobuf + agent_features-{py27,pypy}: protobuf<3.18.0 application_celery: celery<6.0 application_celery-py{py37,37}: importlib-metadata<5.0 ; application_gearman: gearman<3.0.0 mlmodel_sklearn: pandas mlmodel_sklearn: protobuf + mlmodel_sklearn: numpy mlmodel_sklearn-scikitlearnlatest: scikit-learn mlmodel_sklearn-scikitlearn0101: scikit-learn<1.1 - mlmodel_sklearn-scikitlearn0020: scikit-learn<0.21 component_djangorestframework-djangorestframework0300: Django<1.9 component_djangorestframework-djangorestframework0300: djangorestframework<3.1 component_djangorestframework-djangorestframeworklatest: Django From cccd7f4eb1f741a7a076dd08026f2ebaac27604c Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Fri, 23 Jun 2023 14:37:37 -0700 Subject: [PATCH 38/54] Fix attribute name mismatches from mlops sdk (#845) * Convert numerical -> numeric * Adjust attr names to match mlops sdk * Add feature_/label_ prefix to type & name attrs * model_name -> modelName * Set event type to inferenceData --- newrelic/hooks/mlmodel_sklearn.py | 21 +-- .../mlmodel_sklearn/test_inference_events.py | 146 +++++++++--------- tests/mlmodel_sklearn/test_ml_model.py | 102 ++++++------ 3 files changed, 135 insertions(+), 134 deletions(-) diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index 80521c735..358605788 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -194,16 +194,16 @@ def create_label_event(transaction, _class, inference_id, instance, return_val): event = { "inference_id": inference_id, - "model_name": model_name, "model_version": model_version, "label_name": str(label_names_list[index]), - "type": value_type, - "value": str(value), + "label_type": value_type, + # The following are used for entity synthesis. + "modelName": model_name, } # Don't include the raw value when inference_event_value is disabled. if settings and settings.machine_learning.inference_events_value.enabled: - event["value"] = str(value) - transaction.record_custom_event("ML Model Label Event", event) + event["label_value"] = str(value) + transaction.record_custom_event("inferenceData", event) def _get_label_names(user_defined_label_names, prediction_array): @@ -234,7 +234,7 @@ def find_type_category(data_set, row_index, column_index): def categorize_data_type(python_type): if "int" in python_type or "float" in python_type or "complex" in python_type: - return "numerical" + return "numeric" if "bool" in python_type: return "bool" if "str" in python_type or "unicode" in python_type: @@ -289,15 +289,16 @@ def create_feature_event(transaction, _class, inference_id, instance, args, kwar value_type = find_type_category(data_set, row_index, col_index) event = { "inference_id": inference_id, - "model_name": model_name, "model_version": model_version, "feature_name": str(final_feature_names[row_index]), - "type": value_type, + "feature_type": value_type, + # The following are used for entity synthesis. + "modelName": model_name, } # Don't include the raw value when inference_event_value is disabled. if settings and settings.machine_learning and settings.machine_learning.inference_events_value.enabled: - event["value"] = str(value) - transaction.record_custom_event("ML Model Feature Event", event) + event["feature_value"] = str(value) + transaction.record_custom_event("inferenceData", event) def _nr_instrument_model(module, model_class): diff --git a/tests/mlmodel_sklearn/test_inference_events.py b/tests/mlmodel_sklearn/test_inference_events.py index 873892e4e..2cf71ce94 100644 --- a/tests/mlmodel_sklearn/test_inference_events.py +++ b/tests/mlmodel_sklearn/test_inference_events.py @@ -29,31 +29,31 @@ { "users": { "inference_id": None, - "model_name": "DecisionTreeClassifier", + "modelName": "DecisionTreeClassifier", "model_version": "0.0.0", "feature_name": "col1", - "type": "categorical", - "value": "2.0", + "feature_type": "categorical", + "feature_value": "2.0", } }, { "users": { "inference_id": None, - "model_name": "DecisionTreeClassifier", + "modelName": "DecisionTreeClassifier", "model_version": "0.0.0", "feature_name": "col2", - "type": "categorical", - "value": "4.0", + "feature_type": "categorical", + "feature_value": "4.0", } }, { "users": { "inference_id": None, - "model_name": "DecisionTreeClassifier", + "modelName": "DecisionTreeClassifier", "model_version": "0.0.0", "label_name": "0", - "type": "numerical", - "value": "27.0", + "label_type": "numeric", + "label_value": "27.0", } }, ] @@ -79,38 +79,38 @@ def _test(): _test() -label_type = "bool" if sys.version_info < (3, 8) else "numerical" +label_type = "bool" if sys.version_info < (3, 8) else "numeric" true_label_value = "True" if sys.version_info < (3, 8) else "1.0" false_label_value = "False" if sys.version_info < (3, 8) else "0.0" pandas_df_bool_recorded_custom_events = [ { "users": { "inference_id": None, - "model_name": "DecisionTreeClassifier", + "modelName": "DecisionTreeClassifier", "model_version": "0.0.0", "feature_name": "col1", - "type": "bool", - "value": "True", + "feature_type": "bool", + "feature_value": "True", } }, { "users": { "inference_id": None, - "model_name": "DecisionTreeClassifier", + "modelName": "DecisionTreeClassifier", "model_version": "0.0.0", "feature_name": "col2", - "type": "bool", - "value": "True", + "feature_type": "bool", + "feature_value": "True", } }, { "users": { "inference_id": None, - "model_name": "DecisionTreeClassifier", + "modelName": "DecisionTreeClassifier", "model_version": "0.0.0", "label_name": "0", - "type": label_type, - "value": true_label_value, + "label_type": label_type, + "label_value": true_label_value, } }, ] @@ -142,31 +142,31 @@ def _test(): { "users": { "inference_id": None, - "model_name": "DecisionTreeRegressor", + "modelName": "DecisionTreeRegressor", "model_version": "0.0.0", "feature_name": "col1", - "type": "numerical", - "value": "100.0", + "feature_type": "numeric", + "feature_value": "100.0", } }, { "users": { "inference_id": None, - "model_name": "DecisionTreeRegressor", + "modelName": "DecisionTreeRegressor", "model_version": "0.0.0", "feature_name": "col2", - "type": "numerical", - "value": "300.0", + "feature_type": "numeric", + "feature_value": "300.0", } }, { "users": { "inference_id": None, - "model_name": "DecisionTreeRegressor", + "modelName": "DecisionTreeRegressor", "model_version": "0.0.0", "label_name": "0", - "type": "numerical", - "value": "345.6", + "label_type": "numeric", + "label_value": "345.6", } }, ] @@ -198,31 +198,31 @@ def _test(): { "users": { "inference_id": None, - "model_name": "ExtraTreeRegressor", + "modelName": "ExtraTreeRegressor", "model_version": "0.0.0", "feature_name": "0", - "type": "numerical", - "value": "1", + "feature_type": "numeric", + "feature_value": "1", } }, { "users": { "inference_id": None, - "model_name": "ExtraTreeRegressor", + "modelName": "ExtraTreeRegressor", "model_version": "0.0.0", "feature_name": "1", - "type": "numerical", - "value": "2", + "feature_type": "numeric", + "feature_value": "2", } }, { "users": { "inference_id": None, - "model_name": "ExtraTreeRegressor", + "modelName": "ExtraTreeRegressor", "model_version": "0.0.0", "label_name": "0", - "type": "numerical", - "value": "1.0", + "label_type": "numeric", + "label_value": "1.0", } }, ] @@ -253,31 +253,31 @@ def _test(): { "users": { "inference_id": None, - "model_name": "ExtraTreeRegressor", + "modelName": "ExtraTreeRegressor", "model_version": "0.0.0", "feature_name": "0", - "type": "numerical", - "value": "12", + "feature_type": "numeric", + "feature_value": "12", } }, { "users": { "inference_id": None, - "model_name": "ExtraTreeRegressor", + "modelName": "ExtraTreeRegressor", "model_version": "0.0.0", "feature_name": "1", - "type": "numerical", - "value": "13", + "feature_type": "numeric", + "feature_value": "13", } }, { "users": { "inference_id": None, - "model_name": "ExtraTreeRegressor", + "modelName": "ExtraTreeRegressor", "model_version": "0.0.0", "label_name": "0", - "type": "numerical", - "value": "11.0", + "label_type": "numeric", + "label_value": "11.0", } }, ] @@ -308,41 +308,41 @@ def _test(): { "users": { "inference_id": None, - "model_name": "DecisionTreeClassifier", + "modelName": "DecisionTreeClassifier", "model_version": "0.0.0", "feature_name": "0", - "type": "str", - "value": "20", + "feature_type": "str", + "feature_value": "20", } }, { "users": { "inference_id": None, - "model_name": "DecisionTreeClassifier", + "modelName": "DecisionTreeClassifier", "model_version": "0.0.0", "feature_name": "1", - "type": "str", - "value": "21", + "feature_type": "str", + "feature_value": "21", } }, { "users": { "inference_id": None, - "model_name": "DecisionTreeClassifier", + "modelName": "DecisionTreeClassifier", "model_version": "0.0.0", "feature_name": "0", - "type": "str", - "value": "22", + "feature_type": "str", + "feature_value": "22", } }, { "users": { "inference_id": None, - "model_name": "DecisionTreeClassifier", + "modelName": "DecisionTreeClassifier", "model_version": "0.0.0", "feature_name": "1", - "type": "str", - "value": "23", + "feature_type": "str", + "feature_value": "23", } }, ] @@ -373,28 +373,28 @@ def _test(): { "users": { "inference_id": None, - "model_name": "DecisionTreeClassifier", + "modelName": "DecisionTreeClassifier", "model_version": "0.0.0", "feature_name": "0", - "type": "str", + "feature_type": "str", } }, { "users": { "inference_id": None, - "model_name": "DecisionTreeClassifier", + "modelName": "DecisionTreeClassifier", "model_version": "0.0.0", "feature_name": "1", - "type": "str", + "feature_type": "str", } }, { "users": { "inference_id": None, - "model_name": "DecisionTreeClassifier", + "modelName": "DecisionTreeClassifier", "model_version": "0.0.0", "label_name": "0", - "type": "str", + "label_type": "str", } }, ] @@ -473,31 +473,31 @@ def _test(): { "users": { "inference_id": None, - "model_name": "MultiOutputClassifier", + "modelName": "MultiOutputClassifier", "model_version": "0.0.0", "label_name": "0", - "type": "numerical", - "value": "1", + "label_type": "numeric", + "label_value": "1", } }, { "users": { "inference_id": None, - "model_name": "MultiOutputClassifier", + "modelName": "MultiOutputClassifier", "model_version": "0.0.0", "label_name": "1", - "type": "numerical", - "value": "0", + "label_type": "numeric", + "label_value": "0", } }, { "users": { "inference_id": None, - "model_name": "MultiOutputClassifier", + "modelName": "MultiOutputClassifier", "model_version": "0.0.0", "label_name": "2", - "type": "numerical", - "value": "1", + "label_type": "numeric", + "label_value": "1", } }, ] diff --git a/tests/mlmodel_sklearn/test_ml_model.py b/tests/mlmodel_sklearn/test_ml_model.py index 305188b70..571126e6c 100644 --- a/tests/mlmodel_sklearn/test_ml_model.py +++ b/tests/mlmodel_sklearn/test_ml_model.py @@ -118,31 +118,31 @@ def predict(self, X, check_input=True): { "users": { "inference_id": None, - "model_name": "MyCustomModel", + "modelName": "MyCustomModel", "model_version": "1.2.3", "feature_name": "0", - "type": "numerical", - "value": "1.0", + "feature_type": "numeric", + "feature_value": "1.0", } }, { "users": { "inference_id": None, - "model_name": "MyCustomModel", + "modelName": "MyCustomModel", "model_version": "1.2.3", "feature_name": "1", - "type": "numerical", - "value": "2.0", + "feature_type": "numeric", + "feature_value": "2.0", } }, { "users": { "inference_id": None, - "model_name": "MyCustomModel", + "modelName": "MyCustomModel", "model_version": "1.2.3", "label_name": "0", - "type": "numerical", - "value": label_value, + "label_type": "numeric", + "label_value": label_value, } }, ] @@ -172,41 +172,41 @@ def _test(): { "users": { "inference_id": None, - "model_name": "PandasTestModel", + "modelName": "PandasTestModel", "model_version": "1.5.0b1", "feature_name": "feature1", - "type": "categorical", - "value": "0", + "feature_type": "categorical", + "feature_value": "0", } }, { "users": { "inference_id": None, - "model_name": "PandasTestModel", + "modelName": "PandasTestModel", "model_version": "1.5.0b1", "feature_name": "feature2", - "type": "categorical", - "value": "0", + "feature_type": "categorical", + "feature_value": "0", } }, { "users": { "inference_id": None, - "model_name": "PandasTestModel", + "modelName": "PandasTestModel", "model_version": "1.5.0b1", "feature_name": "feature3", - "type": "categorical", - "value": "1", + "feature_type": "categorical", + "feature_value": "1", } }, { "users": { "inference_id": None, - "model_name": "PandasTestModel", + "modelName": "PandasTestModel", "model_version": "1.5.0b1", "label_name": "label1", - "type": "numerical", - "value": "0.5" if six.PY3 else "0.0", + "label_type": "numeric", + "label_value": "0.5" if six.PY3 else "0.0", } }, ] @@ -240,31 +240,31 @@ def _test(): { "users": { "inference_id": None, - "model_name": "MyDecisionTreeClassifier", + "modelName": "MyDecisionTreeClassifier", "model_version": "1.5.0b1", "feature_name": "feature1", - "type": "numerical", - "value": "12", + "feature_type": "numeric", + "feature_value": "12", } }, { "users": { "inference_id": None, - "model_name": "MyDecisionTreeClassifier", + "modelName": "MyDecisionTreeClassifier", "model_version": "1.5.0b1", "feature_name": "feature2", - "type": "numerical", - "value": "14", + "feature_type": "numeric", + "feature_value": "14", } }, { "users": { "inference_id": None, - "model_name": "MyDecisionTreeClassifier", + "modelName": "MyDecisionTreeClassifier", "model_version": "1.5.0b1", "label_name": "label1", - "type": "numerical", - "value": "0", + "label_type": "numeric", + "label_value": "0", } }, ] @@ -303,41 +303,41 @@ def _test(): { "users": { "inference_id": None, - "model_name": "MyDecisionTreeClassifier", + "modelName": "MyDecisionTreeClassifier", "model_version": "1.5.0b1", "feature_name": "col1", - "type": "numerical", - "value": "12", + "feature_type": "numeric", + "feature_value": "12", } }, { "users": { "inference_id": None, - "model_name": "MyDecisionTreeClassifier", + "modelName": "MyDecisionTreeClassifier", "model_version": "1.5.0b1", "feature_name": "col2", - "type": "numerical", - "value": "14", + "feature_type": "numeric", + "feature_value": "14", } }, { "users": { "inference_id": None, - "model_name": "MyDecisionTreeClassifier", + "modelName": "MyDecisionTreeClassifier", "model_version": "1.5.0b1", "feature_name": "col3", - "type": "numerical", - "value": "16", + "feature_type": "numeric", + "feature_value": "16", } }, { "users": { "inference_id": None, - "model_name": "MyDecisionTreeClassifier", + "modelName": "MyDecisionTreeClassifier", "model_version": "1.5.0b1", "label_name": "0", - "type": "numerical", - "value": "1", + "label_type": "numeric", + "label_value": "1", } }, ] @@ -375,31 +375,31 @@ def _test(): { "users": { "inference_id": None, - "model_name": "MyDecisionTreeClassifier", + "modelName": "MyDecisionTreeClassifier", "model_version": "0.0.1", "feature_name": "0", - "type": "str", - "value": "20", + "feature_type": "str", + "feature_value": "20", } }, { "users": { "inference_id": None, - "model_name": "MyDecisionTreeClassifier", + "modelName": "MyDecisionTreeClassifier", "model_version": "0.0.1", "feature_name": "1", - "type": "str", - "value": "21", + "feature_type": "str", + "feature_value": "21", } }, { "users": { "inference_id": None, - "model_name": "MyDecisionTreeClassifier", + "modelName": "MyDecisionTreeClassifier", "model_version": "0.0.1", "label_name": "0", - "type": "str", - "value": "21", + "label_type": "str", + "label_value": "21", } }, ] From e94d3d35e06732efe4b6d79363fb6f6e00918f3f Mon Sep 17 00:00:00 2001 From: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Mon, 26 Jun 2023 10:45:57 -0700 Subject: [PATCH 39/54] Backport main into develop-scikitlearn (#847) * Containerized CI Pipeline (#836) * Revert "Remove Python 2.7 and pypy2 testing (#835)" This reverts commit abb6405d2bfd629ed83f48e8a17b4a28e3a3c352. * Containerize CI process * Publish new docker container for CI images * Rename github actions job * Copyright tag scripts * Drop debug line * Swap to new CI image * Move pip install to just main python * Remove libcurl special case from tox * Install special case packages into main image * Remove unused packages * Remove all other triggers besides manual * Add make run command * Cleanup small bugs * Fix CI Image Tagging (#838) * Correct templated CI image name * Pin pypy2.7 in image * Fix up scripting * Temporarily Restore Old CI Pipeline (#841) * Restore old pipelines * Remove python 2 from setup-python * Rework CI Pipeline (#839) Change pypy to pypy27 in tox. Fix checkout logic Pin tox requires * Fix Tests on New CI (#843) * Remove non-root user * Test new CI image * Change pypy to pypy27 in tox. * Fix checkout logic * Fetch git tags properly * Pin tox requires * Adjust default db settings for github actions * Rename elasticsearch services * Reset to new pipelines * [Mega-Linter] Apply linters fixes * Fix timezone * Fix docker networking * Pin dev image to new sha * Standardize gearman DB settings * Fix elasticsearch settings bug * Fix gearman bug * Add missing odbc headers * Add more debug messages * Swap out dev ci image * Fix required virtualenv version * Swap out dev ci image * Swap out dev ci image * Remove aioredis v1 for EOL * Add coverage paths for docker container * Unpin ci container --------- Co-authored-by: TimPansino * Fix pypy27 dependency * Add skip for OTLP on py27 --------- Co-authored-by: TimPansino Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .github/containers/Dockerfile | 83 ++++ .github/containers/Makefile | 44 ++ .github/containers/install-python.sh | 56 +++ .github/containers/requirements.txt | 5 + .github/scripts/retry.sh | 14 + .github/workflows/build-ci-image.yml | 68 +++ .github/workflows/get-envs.py | 14 + .github/workflows/tests.yml | 403 ++++++++++-------- newrelic/config.py | 30 +- .../test_dimensional_metrics.py | 4 + tests/application_gearman/test_gearman.py | 6 +- .../test_connection.py | 2 +- tests/testing_support/db_settings.py | 152 +++---- .../validators/validate_tt_collector_json.py | 2 +- tox.ini | 138 +++--- 15 files changed, 674 insertions(+), 347 deletions(-) create mode 100644 .github/containers/Dockerfile create mode 100644 .github/containers/Makefile create mode 100755 .github/containers/install-python.sh create mode 100644 .github/containers/requirements.txt create mode 100644 .github/workflows/build-ci-image.yml diff --git a/.github/containers/Dockerfile b/.github/containers/Dockerfile new file mode 100644 index 000000000..3b4b0a7f8 --- /dev/null +++ b/.github/containers/Dockerfile @@ -0,0 +1,83 @@ + +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:20.04 + +# Install OS packages +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y \ + bash \ + build-essential \ + curl \ + expat \ + gcc \ + git \ + libbz2-dev \ + libcurl4-openssl-dev \ + libffi-dev \ + libgmp-dev \ + liblzma-dev \ + libmpfr-dev \ + libncurses-dev \ + libpq-dev \ + libreadline-dev \ + libsqlite3-dev \ + libssl-dev \ + locales \ + make \ + odbc-postgresql \ + openssl \ + python2-dev \ + python3-dev \ + python3-pip \ + tzdata \ + unixodbc-dev \ + unzip \ + wget \ + zip \ + zlib1g \ + zlib1g-dev && \ + rm -rf /var/lib/apt/lists/* + +# Setup ODBC config +RUN sed -i 's/Driver=psqlodbca.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psqlodbca.so/g' /etc/odbcinst.ini && \ + sed -i 's/Driver=psqlodbcw.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psqlodbcw.so/g' /etc/odbcinst.ini && \ + sed -i 's/Setup=libodbcpsqlS.so/Setup=\/usr\/lib\/x86_64-linux-gnu\/odbc\/libodbcpsqlS.so/g' /etc/odbcinst.ini + +# Set the locale +RUN locale-gen --no-purge en_US.UTF-8 +ENV LANG=en_US.UTF-8 \ LANGUAGE=en_US:en \ LC_ALL=en_US.UTF-8 +ENV TZ="Etc/UTC" +RUN ln -fs "/usr/share/zoneinfo/${TZ}" /etc/localtime && \ + dpkg-reconfigure -f noninteractive tzdata + +# Use root user +ENV HOME /root +WORKDIR "${HOME}" + +# Install pyenv +ENV PYENV_ROOT="${HOME}/.pyenv" +RUN curl https://pyenv.run/ | /bin/bash +ENV PATH="$PYENV_ROOT/bin:$PYENV_ROOT/shims:${PATH}" +RUN echo 'eval "$(pyenv init -)"' >>$HOME/.bashrc && \ + pyenv update + +# Install Python +ARG PYTHON_VERSIONS="3.10 3.9 3.8 3.7 3.11 2.7 pypy2.7-7.3.11 pypy3.7" +COPY --chown=1000:1000 --chmod=+x ./install-python.sh /tmp/install-python.sh +COPY ./requirements.txt /requirements.txt +RUN /tmp/install-python.sh && \ + rm /tmp/install-python.sh diff --git a/.github/containers/Makefile b/.github/containers/Makefile new file mode 100644 index 000000000..8a72f4c45 --- /dev/null +++ b/.github/containers/Makefile @@ -0,0 +1,44 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Repository root for mounting into container. +MAKEFILE_DIR:=$(dir $(realpath $(firstword $(MAKEFILE_LIST)))) +REPO_ROOT:=$(realpath $(MAKEFILE_DIR)../../) + +.PHONY: default +default: test + +.PHONY: build +build: + @# Perform a shortened build for testing + @docker build --build-arg='PYTHON_VERSIONS=3.10 2.7' $(MAKEFILE_DIR) -t ghcr.io/newrelic/newrelic-python-agent-ci:local + +.PHONY: test +test: build + @# Ensure python versions are usable + @docker run --rm ghcr.io/newrelic/python-agent-ci:local /bin/bash -c '\ + python3.10 --version && \ + python2.7 --version && \ + touch tox.ini && tox --version && \ + echo "Success! Python versions installed."' + +.PHONY: run +run: build + @docker run --rm -it \ + --mount type=bind,source="$(REPO_ROOT)",target=/home/github/python-agent \ + --workdir=/home/github/python-agent \ + -e NEW_RELIC_HOST="${NEW_RELIC_HOST}" \ + -e NEW_RELIC_LICENSE_KEY="${NEW_RELIC_LICENSE_KEY}" \ + -e NEW_RELIC_DEVELOPER_MODE="${NEW_RELIC_DEVELOPER_MODE}" \ + ghcr.io/newrelic/newrelic-python-agent-ci:local /bin/bash diff --git a/.github/containers/install-python.sh b/.github/containers/install-python.sh new file mode 100755 index 000000000..92184df3a --- /dev/null +++ b/.github/containers/install-python.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +SCRIPT_DIR=$(dirname "$0") +PIP_REQUIREMENTS=$(cat /requirements.txt) + +main() { + # Coerce space separated string to array + if [[ ${#PYTHON_VERSIONS[@]} -eq 1 ]]; then + PYTHON_VERSIONS=($PYTHON_VERSIONS) + fi + + if [[ -z "${PYTHON_VERSIONS[@]}" ]]; then + echo "No python versions specified. Make sure PYTHON_VERSIONS is set." 1>&2 + exit 1 + fi + + # Find all latest pyenv supported versions for requested python versions + PYENV_VERSIONS=() + for v in "${PYTHON_VERSIONS[@]}"; do + LATEST=$(pyenv latest -k "$v" || pyenv latest -k "$v-dev") + if [[ -z "$LATEST" ]]; then + echo "Latest version could not be found for ${v}." 1>&2 + exit 1 + fi + PYENV_VERSIONS+=($LATEST) + done + + # Install each specific version + for v in "${PYENV_VERSIONS[@]}"; do + pyenv install "$v" & + done + wait + + # Set all installed versions as globally accessible + pyenv global ${PYENV_VERSIONS[@]} + + # Install dependencies for main python installation + pyenv exec pip install --upgrade $PIP_REQUIREMENTS +} + +main diff --git a/.github/containers/requirements.txt b/.github/containers/requirements.txt new file mode 100644 index 000000000..27fa6624b --- /dev/null +++ b/.github/containers/requirements.txt @@ -0,0 +1,5 @@ +pip +setuptools +wheel +virtualenv<20.22.0 +tox \ No newline at end of file diff --git a/.github/scripts/retry.sh b/.github/scripts/retry.sh index f4aaca39b..079798a72 100755 --- a/.github/scripts/retry.sh +++ b/.github/scripts/retry.sh @@ -1,4 +1,18 @@ #!/bin/bash +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Time in seconds to backoff after the initial attempt. INITIAL_BACKOFF=10 diff --git a/.github/workflows/build-ci-image.yml b/.github/workflows/build-ci-image.yml new file mode 100644 index 000000000..5bd0e6f69 --- /dev/null +++ b/.github/workflows/build-ci-image.yml @@ -0,0 +1,68 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Build CI Image + +on: + workflow_dispatch: # Allow manual trigger + +concurrency: + group: ${{ github.ref || github.run_id }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + + - name: Generate Docker Metadata (Tags and Labels) + id: meta + uses: docker/metadata-action@v4 + with: + images: ghcr.io/${{ github.repository }}-ci + flavor: | + prefix= + suffix= + latest=false + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=schedule,pattern={{date 'YYYY-MM-DD'}} + type=sha,format=short,prefix=sha- + type=sha,format=long,prefix=sha- + + - name: Login to GitHub Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and Publish Image + uses: docker/build-push-action@v3 + with: + push: ${{ github.event_name != 'pull_request' }} + context: .github/containers + platforms: linux/amd64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/get-envs.py b/.github/workflows/get-envs.py index 576cbeb5c..4fcba6aa7 100755 --- a/.github/workflows/get-envs.py +++ b/.github/workflows/get-envs.py @@ -1,4 +1,18 @@ #!/usr/bin/env python3.8 +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import fileinput import os diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cbee2c947..d714c6013 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,10 +36,9 @@ jobs: - python - elasticsearchserver07 - elasticsearchserver08 - # - gearman + - gearman - grpc #- kafka - - libcurl - memcached - mongodb - mysql @@ -61,6 +60,7 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 with: python-version: "3.10" @@ -117,48 +117,19 @@ jobs: ] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 45 steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - - name: Get Environments - id: get-envs + - name: Fetch git tags run: | - echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT - env: - GROUP_NUMBER: ${{ matrix.group-number }} - - - name: Test - run: | - tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto - env: - TOX_PARALLEL_NO_SPINNER: 1 - PY_COLORS: 0 - - - name: Upload Coverage Artifacts - uses: actions/upload-artifact@v3 - with: - name: coverage-${{ github.job }}-${{ strategy.job-index }} - path: ./**/.coverage.* - retention-days: 1 - - grpc: - env: - TOTAL_GROUPS: 1 - - strategy: - fail-fast: false - matrix: - group-number: [1] - - runs-on: ubuntu-20.04 - timeout-minutes: 30 - - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -169,7 +140,7 @@ jobs: - name: Test run: | - tox -vv -e ${{ steps.get-envs.outputs.envs }} + tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto env: TOX_PARALLEL_NO_SPINNER: 1 PY_COLORS: 0 @@ -181,7 +152,7 @@ jobs: path: ./**/.coverage.* retention-days: 1 - libcurl: + grpc: env: TOTAL_GROUPS: 1 @@ -191,17 +162,19 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - # Special case packages - - name: Install libcurl-dev + - name: Fetch git tags run: | - sudo apt-get update - sudo apt-get install libcurl4-openssl-dev + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -212,7 +185,7 @@ jobs: - name: Test run: | - tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto + tox -vv -e ${{ steps.get-envs.outputs.envs }} env: TOX_PARALLEL_NO_SPINNER: 1 PY_COLORS: 0 @@ -234,6 +207,10 @@ jobs: group-number: [1, 2] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -253,15 +230,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - - name: Install odbc driver for postgresql + - name: Fetch git tags run: | - sudo apt-get update - sudo sudo apt-get install odbc-postgresql - sudo sed -i 's/Driver=psqlodbca.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psqlodbca.so/g' /etc/odbcinst.ini - sudo sed -i 's/Driver=psqlodbcw.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psqlodbcw.so/g' /etc/odbcinst.ini - sudo sed -i 's/Setup=libodbcpsqlS.so/Setup=\/usr\/lib\/x86_64-linux-gnu\/odbc\/libodbcpsqlS.so/g' /etc/odbcinst.ini + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -294,6 +267,10 @@ jobs: group-number: [1, 2] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -316,7 +293,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -349,6 +330,10 @@ jobs: group-number: [1, 2] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -366,7 +351,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -399,6 +388,10 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -418,7 +411,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -451,6 +448,10 @@ jobs: group-number: [1, 2] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -468,7 +469,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -501,6 +506,10 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -519,7 +528,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -542,77 +555,85 @@ jobs: path: ./**/.coverage.* retention-days: 1 - #kafka: - # env: - # TOTAL_GROUPS: 4 - - # strategy: - # fail-fast: false - # matrix: - # group-number: [1, 2, 3, 4] - - # runs-on: ubuntu-20.04 - # timeout-minutes: 30 - - # services: - # zookeeper: - # image: bitnami/zookeeper:3.7 - # env: - # ALLOW_ANONYMOUS_LOGIN: yes - - # ports: - # - 2181:2181 - - # kafka: - # image: bitnami/kafka:3.2 - # ports: - # - 8080:8080 - # - 8081:8081 - # env: - # ALLOW_PLAINTEXT_LISTENER: yes - # KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - # KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: true - # KAFKA_CFG_LISTENERS: L1://:8080,L2://:8081 - # KAFKA_CFG_ADVERTISED_LISTENERS: L1://127.0.0.1:8080,L2://kafka:8081, - # KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: L1:PLAINTEXT,L2:PLAINTEXT - # KAFKA_CFG_INTER_BROKER_LISTENER_NAME: L2 - - # steps: - # - uses: actions/checkout@v3 - # - uses: ./.github/actions/setup-python-matrix - - # # Special case packages - # - name: Install librdkafka-dev - # run: | - # # Use lsb-release to find the codename of Ubuntu to use to install the correct library name - # sudo apt-get update - # sudo ln -fs /usr/share/zoneinfo/America/Los_Angeles /etc/localtime - # sudo apt-get install -y wget gnupg2 software-properties-common - # sudo wget -qO - https://packages.confluent.io/deb/7.2/archive.key | sudo apt-key add - - # sudo add-apt-repository "deb https://packages.confluent.io/clients/deb $(lsb_release -cs) main" - # sudo apt-get update - # sudo apt-get install -y librdkafka-dev/$(lsb_release -c | cut -f 2) - - # - name: Get Environments - # id: get-envs - # run: | - # echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT - # env: - # GROUP_NUMBER: ${{ matrix.group-number }} - - # - name: Test - # run: | - # tox -vv -e ${{ steps.get-envs.outputs.envs }} - # env: - # TOX_PARALLEL_NO_SPINNER: 1 - # PY_COLORS: 0 - - # - name: Upload Coverage Artifacts - # uses: actions/upload-artifact@v3 - # with: - # name: coverage-${{ github.job }}-${{ strategy.job-index }} - # path: ./**/.coverage.* - # retention-days: 1 + # kafka: + # env: + # TOTAL_GROUPS: 4 + + # strategy: + # fail-fast: false + # matrix: + # group-number: [1, 2, 3, 4] + + # runs-on: ubuntu-20.04 + # container: + # image: ghcr.io/${{ github.repository }}-ci:latest + # options: >- + # --add-host=host.docker.internal:host-gateway + # timeout-minutes: 30 + + # services: + # zookeeper: + # image: bitnami/zookeeper:3.7 + # env: + # ALLOW_ANONYMOUS_LOGIN: yes + + # ports: + # - 2181:2181 + + # kafka: + # image: bitnami/kafka:3.2 + # ports: + # - 8080:8080 + # - 8081:8081 + # env: + # ALLOW_PLAINTEXT_LISTENER: yes + # KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + # KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: true + # KAFKA_CFG_LISTENERS: L1://:8080,L2://:8081 + # KAFKA_CFG_ADVERTISED_LISTENERS: L1://127.0.0.1:8080,L2://kafka:8081, + # KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: L1:PLAINTEXT,L2:PLAINTEXT + # KAFKA_CFG_INTER_BROKER_LISTENER_NAME: L2 + + # steps: + # - uses: actions/checkout@v3 + + # - name: Fetch git tags + # run: | + # git config --global --add safe.directory "$GITHUB_WORKSPACE" + # git fetch --tags origin + + # # Special case packages + # - name: Install librdkafka-dev + # run: | + # # Use lsb-release to find the codename of Ubuntu to use to install the correct library name + # sudo apt-get update + # sudo ln -fs /usr/share/zoneinfo/America/Los_Angeles /etc/localtime + # sudo apt-get install -y wget gnupg2 software-properties-common + # sudo wget -qO - https://packages.confluent.io/deb/7.2/archive.key | sudo apt-key add - + # sudo add-apt-repository "deb https://packages.confluent.io/clients/deb $(lsb_release -cs) main" + # sudo apt-get update + # sudo apt-get install -y librdkafka-dev/$(lsb_release -c | cut -f 2) + + # - name: Get Environments + # id: get-envs + # run: | + # echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT + # env: + # GROUP_NUMBER: ${{ matrix.group-number }} + + # - name: Test + # run: | + # tox -vv -e ${{ steps.get-envs.outputs.envs }} + # env: + # TOX_PARALLEL_NO_SPINNER: 1 + # PY_COLORS: 0 + + # - name: Upload Coverage Artifacts + # uses: actions/upload-artifact@v3 + # with: + # name: coverage-${{ github.job }}-${{ strategy.job-index }} + # path: ./**/.coverage.* + # retention-days: 1 mongodb: env: @@ -624,6 +645,10 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -641,7 +666,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -674,10 +703,14 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: - es07: + elasticsearch: image: elasticsearch:7.17.8 env: "discovery.type": "single-node" @@ -693,7 +726,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -726,10 +763,14 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: - es08: + elasticsearch: image: elasticsearch:8.6.0 env: "xpack.security.enabled": "false" @@ -746,7 +787,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -769,51 +814,59 @@ jobs: path: ./**/.coverage.* retention-days: 1 - # gearman: - # env: - # TOTAL_GROUPS: 1 + gearman: + env: + TOTAL_GROUPS: 1 - # strategy: - # fail-fast: false - # matrix: - # group-number: [1] + strategy: + fail-fast: false + matrix: + group-number: [1] - # runs-on: ubuntu-20.04 - # timeout-minutes: 30 + runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway + timeout-minutes: 30 - # services: - # gearman: - # image: artefactual/gearmand - # ports: - # - 4730:4730 - # # Set health checks to wait until gearman has started - # options: >- - # --health-cmd "(echo status ; sleep 0.1) | nc 127.0.0.1 4730 -w 1" - # --health-interval 10s - # --health-timeout 5s - # --health-retries 5 + services: + gearman: + image: artefactual/gearmand + ports: + - 8080:4730 + # Set health checks to wait until gearman has started + options: >- + --health-cmd "(echo status ; sleep 0.1) | nc 127.0.0.1 4730 -w 1" + --health-interval 10s + --health-timeout 5s + --health-retries 5 - # steps: - # - uses: actions/checkout@v3 - # - uses: ./.github/actions/setup-python-matrix + steps: + - uses: actions/checkout@v3 - # - name: Get Environments - # id: get-envs - # run: | - # echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT - # env: - # GROUP_NUMBER: ${{ matrix.group-number }} + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - # - name: Test - # run: | - # tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto - # env: - # TOX_PARALLEL_NO_SPINNER: 1 - # PY_COLORS: 0 + - name: Get Environments + id: get-envs + run: | + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT + env: + GROUP_NUMBER: ${{ matrix.group-number }} - # - name: Upload Coverage Artifacts - # uses: actions/upload-artifact@v3 - # with: - # name: coverage-${{ github.job }}-${{ strategy.job-index }} - # path: ./**/.coverage.* - # retention-days: 1 + - name: Test + run: | + tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto + env: + TOX_PARALLEL_NO_SPINNER: 1 + PY_COLORS: 0 + + - name: Upload Coverage Artifacts + uses: actions/upload-artifact@v3 + with: + name: coverage-${{ github.job }}-${{ strategy.job-index }} + path: ./**/.coverage.* + retention-days: 1 diff --git a/newrelic/config.py b/newrelic/config.py index 7b5da4dd4..842487306 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3793,21 +3793,21 @@ def _process_module_builtin_defaults(): _process_module_definition("thrift.transport.TSocket", "newrelic.hooks.external_thrift") - # _process_module_definition( - # "gearman.client", - # "newrelic.hooks.application_gearman", - # "instrument_gearman_client", - # ) - # _process_module_definition( - # "gearman.connection_manager", - # "newrelic.hooks.application_gearman", - # "instrument_gearman_connection_manager", - # ) - # _process_module_definition( - # "gearman.worker", - # "newrelic.hooks.application_gearman", - # "instrument_gearman_worker", - # ) + _process_module_definition( + "gearman.client", + "newrelic.hooks.application_gearman", + "instrument_gearman_client", + ) + _process_module_definition( + "gearman.connection_manager", + "newrelic.hooks.application_gearman", + "instrument_gearman_connection_manager", + ) + _process_module_definition( + "gearman.worker", + "newrelic.hooks.application_gearman", + "instrument_gearman_worker", + ) _process_module_definition( "botocore.endpoint", diff --git a/tests/agent_features/test_dimensional_metrics.py b/tests/agent_features/test_dimensional_metrics.py index b1f746a81..6d38fe05f 100644 --- a/tests/agent_features/test_dimensional_metrics.py +++ b/tests/agent_features/test_dimensional_metrics.py @@ -34,6 +34,7 @@ import newrelic.core.otlp_utils from newrelic.core.config import global_settings +from newrelic.packages import six try: @@ -46,6 +47,9 @@ @pytest.fixture(scope="module", autouse=True, params=["protobuf", "json"]) def otlp_content_encoding(request): + if six.PY2 and request.param == "protobuf": + pytest.skip("OTLP protos are not compatible with Python 2.") + _settings = global_settings() prev = _settings.debug.otlp_content_encoding _settings.debug.otlp_content_encoding = request.param diff --git a/tests/application_gearman/test_gearman.py b/tests/application_gearman/test_gearman.py index 7ddc13fdc..5dda4ef47 100644 --- a/tests/application_gearman/test_gearman.py +++ b/tests/application_gearman/test_gearman.py @@ -20,14 +20,16 @@ import gearman from newrelic.api.background_task import background_task +from testing_support.db_settings import gearman_settings worker_thread = None worker_event = threading.Event() gm_client = None -GEARMAND_HOST = os.environ.get("GEARMAND_PORT_4730_TCP_ADDR", "localhost") -GEARMAND_PORT = os.environ.get("GEARMAND_PORT_4730_TCP_PORT", "4730") +GEARMAND_SETTINGS = gearman_settings()[0] +GEARMAND_HOST = GEARMAND_SETTINGS["host"] +GEARMAND_PORT = GEARMAND_SETTINGS["port"] GEARMAND_ADDR = "%s:%s" % (GEARMAND_HOST, GEARMAND_PORT) diff --git a/tests/datastore_elasticsearch/test_connection.py b/tests/datastore_elasticsearch/test_connection.py index 2e888af9b..9e8f17b4c 100644 --- a/tests/datastore_elasticsearch/test_connection.py +++ b/tests/datastore_elasticsearch/test_connection.py @@ -36,7 +36,7 @@ def test_connection_default(): else: conn = Connection(**HOST) - assert conn._nr_host_port == ("localhost", ES_SETTINGS["port"]) + assert conn._nr_host_port == (ES_SETTINGS["host"], ES_SETTINGS["port"]) @SKIP_IF_V7 diff --git a/tests/testing_support/db_settings.py b/tests/testing_support/db_settings.py index c7c35935f..bda318062 100644 --- a/tests/testing_support/db_settings.py +++ b/tests/testing_support/db_settings.py @@ -29,25 +29,15 @@ def postgresql_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - - user = password = db = "postgres" - base_port = 8080 - else: - instances = 1 - - user = db = USER - password = "" - base_port = 5432 - + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "localhost" + instances = 2 settings = [ { - "user": user, - "password": password, - "name": db, - "host": "localhost", - "port": base_port + instance_num, + "user": "postgres", + "password": "postgres", + "name": "postgres", + "host": host, + "port": 8080 + instance_num, "table_name": "postgres_table_" + str(os.getpid()), } for instance_num in range(instances) @@ -66,25 +56,15 @@ def mysql_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - - user = password = db = "python_agent" - base_port = 8080 - else: - instances = 1 - - user = db = USER - password = "" - base_port = 3306 - + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "127.0.0.1" + instances = 1 settings = [ { - "user": user, - "password": password, - "name": db, - "host": "127.0.0.1", - "port": base_port + instance_num, + "user": "python_agent", + "password": "python_agent", + "name": "python_agent", + "host": host, + "port": 8080 + instance_num, "namespace": str(os.getpid()), } for instance_num in range(instances) @@ -103,17 +83,12 @@ def redis_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - base_port = 8080 - else: - instances = 1 - base_port = 6379 - + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "localhost" + instances = 2 settings = [ { - "host": "localhost", - "port": base_port + instance_num, + "host": host, + "port": 8080 + instance_num, } for instance_num in range(instances) ] @@ -131,17 +106,12 @@ def memcached_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - base_port = 8080 - else: - instances = 1 - base_port = 11211 - + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "127.0.0.1" + instances = 2 settings = [ { - "host": "127.0.0.1", - "port": base_port + instance_num, + "host": host, + "port": 8080 + instance_num, "namespace": str(os.getpid()), } for instance_num in range(instances) @@ -160,15 +130,10 @@ def mongodb_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - base_port = 8080 - else: - instances = 1 - base_port = 27017 - + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "127.0.0.1" + instances = 2 settings = [ - {"host": "127.0.0.1", "port": base_port + instance_num, "collection": "mongodb_collection_" + str(os.getpid())} + {"host": host, "port": 8080 + instance_num, "collection": "mongodb_collection_" + str(os.getpid())} for instance_num in range(instances) ] return settings @@ -185,17 +150,12 @@ def elasticsearch_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - base_port = 8080 - else: - instances = 1 - base_port = 9200 - + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "localhost" + instances = 2 settings = [ { - "host": "localhost", - "port": str(base_port + instance_num), + "host": host, + "port": str(8080 + instance_num), "namespace": str(os.getpid()), } for instance_num in range(instances) @@ -214,17 +174,12 @@ def solr_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - base_port = 8080 - else: - instances = 1 - base_port = 8983 - + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "127.0.0.1" + instances = 2 settings = [ { - "host": "127.0.0.1", - "port": base_port + instance_num, + "host": host, + "port": 8080 + instance_num, "namespace": str(os.getpid()), } for instance_num in range(instances) @@ -243,13 +198,12 @@ def rabbitmq_settings(): 2. Github Actions """ + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "localhost" instances = 1 - base_port = 5672 - settings = [ { - "host": "localhost", - "port": base_port + instance_num, + "host": host, + "port": 5672 + instance_num, } for instance_num in range(instances) ] @@ -267,17 +221,35 @@ def kafka_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - base_port = 8080 - else: - instances = 1 - base_port = 9092 + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "localhost" + instances = 2 + settings = [ + { + "host": host, + "port": 8080 + instance_num, + } + for instance_num in range(instances) + ] + return settings + +def gearman_settings(): + """Return a list of dict of settings for connecting to kafka. + + Will return the correct settings, depending on which of the environments it + is running in. It attempts to set variables in the following order, where + later environments override earlier ones. + + 1. Local + 2. Github Actions + """ + + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "localhost" + instances = 1 settings = [ { - "host": "localhost", - "port": base_port + instance_num, + "host": host, + "port": 8080 + instance_num, } for instance_num in range(instances) ] diff --git a/tests/testing_support/validators/validate_tt_collector_json.py b/tests/testing_support/validators/validate_tt_collector_json.py index 85e393280..28c9e93a3 100644 --- a/tests/testing_support/validators/validate_tt_collector_json.py +++ b/tests/testing_support/validators/validate_tt_collector_json.py @@ -135,7 +135,7 @@ def _check_params_and_start_time(node): if segment_name.startswith("Datastore"): for key in datastore_params: assert key in params, key - assert params[key] == datastore_params[key] + assert params[key] == datastore_params[key], "Expected %s. Got %s." % (datastore_params[key], params[key]) for key in datastore_forgone_params: assert key not in params, key diff --git a/tox.ini b/tox.ini index f4c354deb..3e2ac0fbd 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ ; framework_aiohttp-aiohttp01: aiohttp<2 ; framework_aiohttp-aiohttp0202: aiohttp<2.3 ; 3. Python version required. Uses the standard tox definitions. (https://tox.readthedocs.io/en/latest/config.html#tox-environments) -; Examples: py27,py37,py38,py39,pypy,pypy37 +; Examples: py27,py37,py38,py39,pypy27,pypy37 ; 4. Library and version (Optional). Used when testing multiple versions of the library, and may be omitted when only testing a single version. ; Versions should be specified with 2 digits per version number, so <3 becomes 02 and <3.5 becomes 0304. latest and master are also acceptable versions. ; Examples: uvicorn03, CherryPy0302, uvicornlatest @@ -43,10 +43,10 @@ requires = virtualenv<20.22.0 setupdir = {toxinidir} envlist = - python-adapter_cheroot-{py37,py38,py39,py310,py311}, + python-adapter_cheroot-{py27,py37,py38,py39,py310,py311}, python-adapter_daphne-{py37,py38,py39,py310,py311}-daphnelatest, python-adapter_daphne-py38-daphne{0204,0205}, - python-adapter_gevent-{py37,py38,py310,py311}, + python-adapter_gevent-{py27,py37,py38,py310,py311}, python-adapter_gunicorn-{py37,py38,py39,py310,py311}-aiohttp3-gunicornlatest, python-adapter_hypercorn-{py37,py38,py39,py310,py311}-hypercornlatest, python-adapter_hypercorn-py38-hypercorn{0010,0011,0012,0013}, @@ -55,83 +55,94 @@ envlist = python-adapter_waitress-{py37,py38,py39}-waitress010404, python-adapter_waitress-{py37,py38,py39,py310}-waitress02, python-adapter_waitress-{py37,py38,py39,py310,py311}-waitresslatest, - python-agent_features-{py37,py38,py39,py310,py311}-{with,without}_extensions, - python-agent_features-{pypy37}-without_extensions, + python-agent_features-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, + python-agent_features-{pypy27,pypy37}-without_extensions, + python-agent_streaming-py27-grpc0125-{with,without}_extensions, python-agent_streaming-{py37,py38,py39,py310,py311}-protobuf04-{with,without}_extensions, python-agent_streaming-py39-protobuf{03,0319}-{with,without}_extensions, - python-agent_unittests-{py37,py38,py39,py310,py311}-{with,without}_extensions, - python-agent_unittests-{pypy37}-without_extensions, - python-application_celery-{py37,py38,py39,py310,py311,pypy37}, - python-mlmodel_sklearn-{py38,py39,py310,py311}-scikitlearnlatest, - python-mlmodel_sklearn-{py37,pypy37}-scikitlearn0101, + python-agent_unittests-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, + python-agent_unittests-{pypy27,pypy37}-without_extensions, + python-application_celery-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + gearman-application_gearman-{py27,pypy27}, + python-component_djangorestframework-py27-djangorestframework0300, python-component_djangorestframework-{py37,py38,py39,py310,py311}-djangorestframeworklatest, python-component_flask_rest-{py37,py38,py39,pypy37}-flaskrestxlatest, + python-component_flask_rest-{py27,pypy27}-flaskrestx051, python-component_graphqlserver-{py37,py38,py39,py310,py311}, + python-component_tastypie-{py27,pypy27}-tastypie0143, python-component_tastypie-{py37,py38,py39,pypy37}-tastypie{0143,latest}, python-coroutines_asyncio-{py37,py38,py39,py310,py311,pypy37}, - python-cross_agent-{py37,py38,py39,py310,py311}-{with,without}_extensions, + python-cross_agent-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, + python-cross_agent-pypy27-without_extensions, postgres-datastore_asyncpg-{py37,py38,py39,py310,py311}, - memcached-datastore_bmemcached-{py37,py38,py39,py310,py311}-memcached030, - elasticsearchserver07-datastore_elasticsearch-{py37,py38,py39,py310,py311,pypy37}-elasticsearch07, + memcached-datastore_bmemcached-{pypy27,py27,py37,py38,py39,py310,py311}-memcached030, + elasticsearchserver07-datastore_elasticsearch-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}-elasticsearch07, elasticsearchserver08-datastore_elasticsearch-{py37,py38,py39,py310,py311,pypy37}-elasticsearch08, - memcached-datastore_memcache-{py37,py38,py39,py310,py311,pypy37}-memcached01, + memcached-datastore_memcache-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}-memcached01, + mysql-datastore_mysql-mysql080023-py27, mysql-datastore_mysql-mysqllatest-{py37,py38,py39,py310,py311}, postgres-datastore_postgresql-{py37,py38,py39}, - postgres-datastore_psycopg2-{py37,py38,py39,py310,py311}-psycopg2latest, - postgres-datastore_psycopg2cffi-{py37,py38,py39,py310,py311}-psycopg2cffilatest, - postgres-datastore_pyodbc-{py37,py311}-pyodbclatest, - memcached-datastore_pylibmc-{py37}, - memcached-datastore_pymemcache-{py37,py38,py39,py310,py311,pypy37}, - mongodb-datastore_pymongo-{py37,py38,py39,py310,py311}-pymongo{03}, - mongodb-datastore_pymongo-{py37,py38,py39,py310,py311,pypy37}-pymongo04, - mysql-datastore_pymysql-{py37,py38,py39,py310,py311,pypy37}, - solr-datastore_pysolr-{py37,py38,py39,py310,py311,pypy37}, - redis-datastore_redis-{py37,py38,pypy37}-redis03, + postgres-datastore_psycopg2-{py27,py37,py38,py39,py310,py311}-psycopg2latest + postgres-datastore_psycopg2cffi-{py27,pypy27,py37,py38,py39,py310,py311}-psycopg2cffilatest, + postgres-datastore_pyodbc-{py27,py37,py311}-pyodbclatest + memcached-datastore_pylibmc-{py27,py37}, + memcached-datastore_pymemcache-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + mongodb-datastore_pymongo-{py27,py37,py38,py39,py310,py311,pypy27}-pymongo{03}, + mongodb-datastore_pymongo-{py37,py38,py39,py310,py311,pypy27,pypy37}-pymongo04, + mysql-datastore_pymysql-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + solr-datastore_pysolr-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + redis-datastore_redis-{py27,py37,py38,pypy27,pypy37}-redis03, redis-datastore_redis-{py37,py38,py39,py310,py311,pypy37}-redis{0400,latest}, redis-datastore_aioredis-{py37,py38,py39,py310,pypy37}-aioredislatest, - redis-datastore_aioredis-{py37,py310}-aioredis01, redis-datastore_aioredis-{py37,py38,py39,py310,py311,pypy37}-redislatest, redis-datastore_aredis-{py37,py38,py39,pypy37}-aredislatest, - python-datastore_sqlite-{py37,py38,py39,py310,py311,pypy37}, - python-external_boto3-{py37,py38,py39,py310,py311}-boto01, + solr-datastore_solrpy-{py27,pypy27}-solrpy{00,01}, + python-datastore_sqlite-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + python-external_boto3-{py27,py37,py38,py39,py310,py311}-boto01, python-external_botocore-{py37,py38,py39,py310,py311}-botocorelatest, python-external_botocore-{py311}-botocore128, python-external_botocore-py310-botocore0125, - python-external_http-{py37,py38,py39,py310,py311}, - python-external_httplib-{py37,py38,py39,py310,py311,pypy37}, - python-external_httplib2-{py37,py38,py39,py310,py311,pypy37}, + python-external_feedparser-py27-feedparser{05,06}, + python-external_http-{py27,py37,py38,py39,py310,py311,pypy27}, + python-external_httplib-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + python-external_httplib2-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, python-external_httpx-{py37,py38,py39,py310,py311}, - python-external_requests-{py37,py38,py39,py310,py311,pypy37}, - python-external_urllib3-{py37}-urllib3{0109}, - python-external_urllib3-{py37,py38,py39,py310,py311,pypy37}-urllib3latest, + python-external_requests-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + python-external_urllib3-{py27,py37,pypy27}-urllib3{0109}, + python-external_urllib3-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}-urllib3latest, python-framework_aiohttp-{py37,py38,py39,py310,py311,pypy37}-aiohttp03, python-framework_ariadne-{py37,py38,py39,py310,py311}-ariadnelatest, python-framework_ariadne-py37-ariadne{0011,0012,0013}, - python-framework_bottle-{py37,py38,py39,pypy37}-bottle{0011,0012}, + python-framework_bottle-py27-bottle{0008,0009,0010}, + python-framework_bottle-{py27,py37,py38,py39,pypy37}-bottle{0011,0012}, python-framework_bottle-{py310,py311}-bottle0012, + python-framework_bottle-pypy27-bottle{0008,0009,0010,0011,0012}, ; CherryPy still uses inspect.getargspec, deprecated in favor of inspect.getfullargspec. Not supported in 3.11 python-framework_cherrypy-{py37,py38,py39,py310,py311,pypy37}-CherryPylatest, - python-framework_django-{py37}-Django0108, + python-framework_django-{pypy27,py27}-Django0103, + python-framework_django-{pypy27,py27,py37}-Django0108, python-framework_django-{py39}-Django{0200,0201,0202,0300,0301,latest}, python-framework_django-{py37,py38,py39,py310,py311}-Django0302, - python-framework_falcon-{py37,py38,py39,pypy37}-falcon0103, + python-framework_falcon-{py27,py37,py38,py39,pypy27,pypy37}-falcon0103, python-framework_falcon-{py37,py38,py39,py310,pypy37}-falcon{0200,master}, # Falcon master branch failing on 3.11 currently. python-framework_falcon-py311-falcon0200, python-framework_fastapi-{py37,py38,py39,py310,py311}, - python-framework_flask-{py37,py38,py39,py310,py311,pypy37}-flask0101, + python-framework_flask-{pypy27,py27}-flask0012, + python-framework_flask-{pypy27,py27,py37,py38,py39,py310,py311,pypy37}-flask0101, ; temporarily disabling flaskmaster tests python-framework_flask-{py37,py38,py39,py310,py311,pypy37}-flask{latest}, python-framework_graphene-{py37,py38,py39,py310,py311}-graphenelatest, - python-framework_graphene-{py37,py38,py39,pypy37}-graphene{0200,0201}, + python-framework_graphene-{py27,py37,py38,py39,pypy27,pypy37}-graphene{0200,0201}, python-framework_graphene-{py310,py311}-graphene0201, - python-framework_graphql-{py37,py38,py39,py310,py311,pypy37}-graphql02, + python-framework_graphql-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}-graphql02, python-framework_graphql-{py37,py38,py39,py310,py311,pypy37}-graphql03, ; temporarily disabling graphqlmaster tests python-framework_graphql-py37-graphql{0202,0203,0300,0301,0302}, + grpc-framework_grpc-py27-grpc0125, grpc-framework_grpc-{py37,py38,py39,py310,py311}-grpclatest, - python-framework_pyramid-{py38}-Pyramid0104, - python-framework_pyramid-{pypy37,py37,py38,py39,py310,py311}-Pyramid0110-cornice, + python-framework_pyramid-{pypy27,py27,py38}-Pyramid0104, + python-framework_pyramid-{pypy27,py27,pypy37,py37,py38,py39,py310,py311}-Pyramid0110-cornice, python-framework_pyramid-{py37,py38,py39,py310,py311,pypy37}-Pyramidlatest, python-framework_sanic-{py38,pypy37}-sanic{190301,1906,1912,200904,210300,2109,2112,2203,2290}, python-framework_sanic-{py37,py38,py39,py310,py311,pypy37}-saniclatest, @@ -139,26 +150,27 @@ envlist = python-framework_starlette-{py37,py38}-starlette{002001}, python-framework_starlette-{py37,py38,py39,py310,py311,pypy37}-starlettelatest, python-framework_strawberry-{py37,py38,py39,py310,py311}-strawberrylatest, - python-logger_logging-{py37,py38,py39,py310,py311,pypy37}, + python-logger_logging-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, python-logger_loguru-{py37,py38,py39,py310,py311,pypy37}-logurulatest, python-logger_loguru-py39-loguru{06,05,04,03}, - libcurl-framework_tornado-{py37,py38,py39,py310,py311,pypy37}-tornado0600, - libcurl-framework_tornado-{py38,py39,py310,py311}-tornadomaster, - rabbitmq-messagebroker_pika-{py37,py38,py39,pypy37}-pika0.13, + python-framework_tornado-{py37,py38,py39,py310,py311,pypy37}-tornado0600, + python-framework_tornado-{py38,py39,py310,py311}-tornadomaster, + rabbitmq-messagebroker_pika-{py27,py37,py38,py39,pypy27,pypy37}-pika0.13, rabbitmq-messagebroker_pika-{py37,py38,py39,py310,py311,pypy37}-pikalatest, - kafka-messagebroker_confluentkafka-{py37,py38,py39,py310,py311}-confluentkafkalatest, - kafka-messagebroker_confluentkafka-{py39}-confluentkafka{0107,0106}, + kafka-messagebroker_confluentkafka-{py27,py37,py38,py39,py310,py311}-confluentkafkalatest, + kafka-messagebroker_confluentkafka-{py27,py39}-confluentkafka{0107,0106}, ; confluent-kafka had a bug in 1.8.2's setup.py file which was incompatible with 2.7. kafka-messagebroker_confluentkafka-{py39}-confluentkafka{0108}, - kafka-messagebroker_kafkapython-{py37,py38,pypy37}-kafkapythonlatest, - kafka-messagebroker_kafkapython-{py38}-kafkapython{020001,020000,0104}, - python-template_genshi-{py37,py311}-genshilatest, - python-template_mako-{py37,py310,py311}, + kafka-messagebroker_kafkapython-{pypy27,py27,py37,py38,pypy37}-kafkapythonlatest, + kafka-messagebroker_kafkapython-{py27,py38}-kafkapython{020001,020000,0104}, + python-template_genshi-{py27,py37,py311}-genshilatest + python-template_mako-{py27,py37,py310,py311} [testenv] deps = # Base Dependencies {py37,py38,py39,py310,py311,pypy37}: pytest==7.2.2 + {py27,pypy27}: pytest==4.6.11 iniconfig coverage WebTest==2.0.35 @@ -189,10 +201,10 @@ deps = adapter_waitress-waitresslatest: waitress agent_features: beautifulsoup4 agent_features-{py37,py38,py39,py310,py311,pypy37}: protobuf - agent_features-{py27,pypy}: protobuf<3.18.0 + agent_features-{py27,pypy27}: protobuf<3.18.0 application_celery: celery<6.0 - application_celery-py{py37,37}: importlib-metadata<5.0 - ; application_gearman: gearman<3.0.0 + application_celery-{py37,pypy37}: importlib-metadata<5.0 + application_gearman: gearman<3.0.0 mlmodel_sklearn: pandas mlmodel_sklearn: protobuf mlmodel_sklearn: numpy @@ -214,7 +226,7 @@ deps = component_graphqlserver: markupsafe<2.1 component_graphqlserver: jinja2<3.1 component_tastypie-tastypie0143: django-tastypie<0.14.4 - component_tastypie-{py27,pypy}-tastypie0143: django<1.12 + component_tastypie-{py27,pypy27}-tastypie0143: django<1.12 component_tastypie-{py37,py38,py39,py310,py311,pypy37}-tastypie0143: django<3.0.1 component_tastypie-{py37,py38,py39,py310,py311,pypy37}-tastypie0143: asgiref<3.7.1 # asgiref==3.7.1 only suppport Python 3.10+ component_tastypie-tastypielatest: django-tastypie @@ -246,10 +258,9 @@ deps = datastore_redis-redislatest: redis datastore_redis-redis0400: redis<4.1 datastore_redis-redis03: redis<4.0 - datastore_redis-{py27,pypy}: rb + datastore_redis-{py27,pypy27}: rb datastore_aioredis-redislatest: redis datastore_aioredis-aioredislatest: aioredis - datastore_aioredis-aioredis01: aioredis<2 datastore_aredis-aredislatest: aredis datastore_solrpy-solrpy00: solrpy<1.0 datastore_solrpy-solrpy01: solrpy<2.0 @@ -364,7 +375,7 @@ deps = messagebroker_pika-pika0.13: pika<0.14 messagebroker_pika-pikalatest: pika messagebroker_pika: tornado<5 - messagebroker_pika-{py27,pypy}: enum34 + messagebroker_pika-{py27,pypy27}: enum34 messagebroker_confluentkafka-confluentkafkalatest: confluent-kafka messagebroker_confluentkafka-confluentkafka0108: confluent-kafka<1.9 messagebroker_confluentkafka-confluentkafka0107: confluent-kafka<1.8 @@ -385,9 +396,9 @@ setenv = without_extensions: NEW_RELIC_EXTENSIONS = false agent_features: NEW_RELIC_APDEX_T = 1000 framework_grpc: PYTHONPATH={toxinidir}/tests/:{toxinidir}/tests/framework_grpc/sample_application - libcurl: PYCURL_SSL_LIBRARY=openssl - libcurl: LDFLAGS=-L/usr/local/opt/openssl/lib - libcurl: CPPFLAGS=-I/usr/local/opt/openssl/include + framework_tornado: PYCURL_SSL_LIBRARY=openssl + framework_tornado: LDFLAGS=-L/usr/local/opt/openssl/lib + framework_tornado: CPPFLAGS=-I/usr/local/opt/openssl/include passenv = NEW_RELIC_DEVELOPER_MODE @@ -402,7 +413,7 @@ commands = framework_grpc: --grpc_python_out={toxinidir}/tests/framework_grpc/sample_application \ framework_grpc: /{toxinidir}/tests/framework_grpc/sample_application/sample_application.proto - libcurl: pip install --ignore-installed --config-settings="--build-option=--with-openssl" pycurl + framework_tornado: pip install --ignore-installed --config-settings="--build-option=--with-openssl" pycurl coverage run -m pytest -v [] allowlist_externals={toxinidir}/.github/scripts/* @@ -425,7 +436,7 @@ changedir = agent_streaming: tests/agent_streaming agent_unittests: tests/agent_unittests application_celery: tests/application_celery - ; application_gearman: tests/application_gearman + application_gearman: tests/application_gearman mlmodel_sklearn: tests/mlmodel_sklearn component_djangorestframework: tests/component_djangorestframework component_flask_rest: tests/component_flask_rest @@ -499,6 +510,7 @@ source = newrelic source = newrelic/ .tox/**/site-packages/newrelic/ + /__w/**/site-packages/newrelic/ [coverage:html] directory = ${TOX_ENV_DIR-.}/htmlcov From f2ac7292a7448f04fb586204f84dc6098db717b2 Mon Sep 17 00:00:00 2001 From: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Tue, 27 Jun 2023 09:35:28 -0700 Subject: [PATCH 40/54] Use Dimensional Metrics in SKLearn (#850) * Convert ML custom metrics to dimensional with tags * Rename _class to class_ * Remove typo * Adjust ML metric tests for dimensional metrics * Pin sklearn to <1.11.0 for testing * [Mega-Linter] Apply linters fixes --------- Co-authored-by: TimPansino --- newrelic/hooks/mlmodel_sklearn.py | 73 ++- .../mlmodel_sklearn/test_prediction_stats.py | 543 +++++++++++++----- tox.ini | 2 +- 3 files changed, 441 insertions(+), 177 deletions(-) diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index 358605788..c7c1d30bb 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -54,7 +54,7 @@ def __init__(self, wrapped, model_name, training_step): self._nr_training_step = training_step -def _wrap_method_trace(module, _class, method, name=None, group=None): +def _wrap_method_trace(module, class_, method, name=None, group=None): def _nr_wrapper_method(wrapped, instance, args, kwargs): transaction = current_transaction() trace = current_trace() @@ -96,15 +96,15 @@ def _nr_wrapper_method(wrapped, instance, args, kwargs): if method in ("predict", "fit_predict"): training_step = getattr(instance, "_nr_wrapped_training_step", "Unknown") inference_id = uuid.uuid4() - create_feature_event(transaction, _class, inference_id, instance, args, kwargs) - create_label_event(transaction, _class, inference_id, instance, return_val) - return PredictReturnTypeProxy(return_val, model_name=_class, training_step=training_step) + create_feature_event(transaction, class_, inference_id, instance, args, kwargs) + create_label_event(transaction, class_, inference_id, instance, return_val) + return PredictReturnTypeProxy(return_val, model_name=class_, training_step=training_step) return return_val - wrap_function_wrapper(module, "%s.%s" % (_class, method), _nr_wrapper_method) + wrap_function_wrapper(module, "%s.%s" % (class_, method), _nr_wrapper_method) -def _calc_prediction_feature_stats(prediction_input, _class, feature_column_names): +def _calc_prediction_feature_stats(prediction_input, class_, feature_column_names, tags): import numpy as np # Drop any feature columns that are not numeric since we can't compute stats @@ -126,10 +126,10 @@ def _calc_prediction_feature_stats(prediction_input, _class, feature_column_name features = np.reshape(numeric_features, (len(numeric_features) // num_cols, num_cols)) features = features.astype(dtype=np.float64) - _record_stats(features, feature_column_names, _class, "Feature") + _record_stats(features, feature_column_names, class_, "Feature", tags) -def _record_stats(data, column_names, _class, column_type): +def _record_stats(data, column_names, class_, column_type, tags): import numpy as np mean = np.mean(data, axis=0) @@ -147,30 +147,31 @@ def _record_stats(data, column_names, _class, column_type): # to upload them one at a time instead of as a dictionary of stats per # feature column. for index, col_name in enumerate(column_names): - metric_name = "MLModel/Sklearn/Named/%s/Predict/%s/%s" % (_class, column_type, col_name) - transaction.record_custom_metrics( + metric_name = "MLModel/Sklearn/Named/%s/Predict/%s/%s" % (class_, column_type, col_name) + + transaction.record_dimensional_metrics( [ - ("%s/%s" % (metric_name, "Mean"), float(mean[index])), - ("%s/%s" % (metric_name, "Percentile25"), float(percentile25[index])), - ("%s/%s" % (metric_name, "Percentile50"), float(percentile50[index])), - ("%s/%s" % (metric_name, "Percentile75"), float(percentile75[index])), - ("%s/%s" % (metric_name, "StandardDeviation"), float(standard_deviation[index])), - ("%s/%s" % (metric_name, "Min"), float(_min[index])), - ("%s/%s" % (metric_name, "Max"), float(_max[index])), - ("%s/%s" % (metric_name, "Count"), _count), + ("%s/%s" % (metric_name, "Mean"), float(mean[index]), tags), + ("%s/%s" % (metric_name, "Percentile25"), float(percentile25[index]), tags), + ("%s/%s" % (metric_name, "Percentile50"), float(percentile50[index]), tags), + ("%s/%s" % (metric_name, "Percentile75"), float(percentile75[index]), tags), + ("%s/%s" % (metric_name, "StandardDeviation"), float(standard_deviation[index]), tags), + ("%s/%s" % (metric_name, "Min"), float(_min[index]), tags), + ("%s/%s" % (metric_name, "Max"), float(_max[index]), tags), + ("%s/%s" % (metric_name, "Count"), _count, tags), ] ) -def _calc_prediction_label_stats(labels, _class, label_column_names): +def _calc_prediction_label_stats(labels, class_, label_column_names, tags): import numpy as np labels = np.array(labels, dtype=np.float64) - _record_stats(labels, label_column_names, _class, "Label") + _record_stats(labels, label_column_names, class_, "Label", tags) -def create_label_event(transaction, _class, inference_id, instance, return_val): - model_name = getattr(instance, "_nr_wrapped_name", _class) +def create_label_event(transaction, class_, inference_id, instance, return_val): + model_name = getattr(instance, "_nr_wrapped_name", class_) model_version = getattr(instance, "_nr_wrapped_version", "0.0.0") label_names = getattr(instance, "_nr_wrapped_label_names", None) @@ -186,7 +187,17 @@ def create_label_event(transaction, _class, inference_id, instance, return_val): labels = np.reshape(labels, (len(labels) // 1, 1)) label_names_list = _get_label_names(label_names, labels) - _calc_prediction_label_stats(labels, _class, label_names_list) + _calc_prediction_label_stats( + labels, + class_, + label_names_list, + tags={ + "inference_id": inference_id, + "model_version": model_version, + # The following are used for entity synthesis. + "modelName": model_name, + }, + ) for prediction in labels: for index, value in enumerate(prediction): python_value_type = str(type(value)) @@ -272,18 +283,28 @@ def bind_predict(X, *args, **kwargs): return X -def create_feature_event(transaction, _class, inference_id, instance, args, kwargs): +def create_feature_event(transaction, class_, inference_id, instance, args, kwargs): import numpy as np data_set = bind_predict(*args, **kwargs) - model_name = getattr(instance, "_nr_wrapped_name", _class) + model_name = getattr(instance, "_nr_wrapped_name", class_) model_version = getattr(instance, "_nr_wrapped_version", "0.0.0") user_provided_feature_names = getattr(instance, "_nr_wrapped_feature_names", None) settings = transaction.settings if transaction.settings is not None else global_settings() final_feature_names = _get_feature_column_names(user_provided_feature_names, data_set) np_casted_data_set = np.array(data_set) - _calc_prediction_feature_stats(data_set, _class, final_feature_names) + _calc_prediction_feature_stats( + data_set, + class_, + final_feature_names, + tags={ + "inference_id": inference_id, + "model_version": model_version, + # The following are used for entity synthesis. + "modelName": model_name, + }, + ) for col_index, feature in enumerate(np_casted_data_set): for row_index, value in enumerate(feature): value_type = find_type_category(data_set, row_index, col_index) diff --git a/tests/mlmodel_sklearn/test_prediction_stats.py b/tests/mlmodel_sklearn/test_prediction_stats.py index e38595039..d3d5f122b 100644 --- a/tests/mlmodel_sklearn/test_prediction_stats.py +++ b/tests/mlmodel_sklearn/test_prediction_stats.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import uuid + import numpy as np import pandas as pd import pytest @@ -22,6 +24,18 @@ from newrelic.api.background_task import background_task from newrelic.packages import six +ML_METRIC_FORCED_UUID = "0b59992f-2349-4a46-8de1-696d3fe1088b" + + +@pytest.fixture(scope="function") +def force_uuid(monkeypatch): + monkeypatch.setattr(uuid, "uuid4", lambda *a, **k: ML_METRIC_FORCED_UUID) + + +_test_prediction_stats_tags = frozenset( + {("modelName", "DummyClassifier"), ("inference_id", ML_METRIC_FORCED_UUID), ("model_version", "0.0.0")} +) + @pytest.mark.parametrize( "x_train,y_train,x_test,metrics", @@ -31,30 +45,66 @@ [0, 1], [[2.0, 2.0], [0, 0.5]], [ - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Mean", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Percentile25", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Percentile50", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Percentile75", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/StandardDeviation", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Min", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Max", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Count", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Mean", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Percentile25", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Percentile50", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Percentile75", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/StandardDeviation", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Min", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Max", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Count", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Mean", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Percentile25", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Percentile50", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Percentile75", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/StandardDeviation", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Min", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Max", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Count", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Mean", _test_prediction_stats_tags, 1), + ( + "MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Percentile25", + _test_prediction_stats_tags, + 1, + ), + ( + "MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Percentile50", + _test_prediction_stats_tags, + 1, + ), + ( + "MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Percentile75", + _test_prediction_stats_tags, + 1, + ), + ( + "MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/StandardDeviation", + _test_prediction_stats_tags, + 1, + ), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Min", _test_prediction_stats_tags, 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Max", _test_prediction_stats_tags, 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Count", _test_prediction_stats_tags, 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Mean", _test_prediction_stats_tags, 1), + ( + "MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Percentile25", + _test_prediction_stats_tags, + 1, + ), + ( + "MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Percentile50", + _test_prediction_stats_tags, + 1, + ), + ( + "MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Percentile75", + _test_prediction_stats_tags, + 1, + ), + ( + "MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/StandardDeviation", + _test_prediction_stats_tags, + 1, + ), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Min", _test_prediction_stats_tags, 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Max", _test_prediction_stats_tags, 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Count", _test_prediction_stats_tags, 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Mean", _test_prediction_stats_tags, 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Percentile25", _test_prediction_stats_tags, 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Percentile50", _test_prediction_stats_tags, 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Percentile75", _test_prediction_stats_tags, 1), + ( + "MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/StandardDeviation", + _test_prediction_stats_tags, + 1, + ), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Min", _test_prediction_stats_tags, 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Max", _test_prediction_stats_tags, 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Count", _test_prediction_stats_tags, 1), ], ), ( @@ -62,30 +112,66 @@ [0, 1], np.array([[2.0, 2.0], [0, 0.5]]), [ - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Mean", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Percentile25", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Percentile50", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Percentile75", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/StandardDeviation", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Min", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Max", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Count", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Mean", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Percentile25", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Percentile50", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Percentile75", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/StandardDeviation", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Min", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Max", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Count", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Mean", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Percentile25", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Percentile50", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Percentile75", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/StandardDeviation", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Min", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Max", 1), - ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Count", 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Mean", _test_prediction_stats_tags, 1), + ( + "MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Percentile25", + _test_prediction_stats_tags, + 1, + ), + ( + "MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Percentile50", + _test_prediction_stats_tags, + 1, + ), + ( + "MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Percentile75", + _test_prediction_stats_tags, + 1, + ), + ( + "MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/StandardDeviation", + _test_prediction_stats_tags, + 1, + ), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Min", _test_prediction_stats_tags, 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Max", _test_prediction_stats_tags, 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/0/Count", _test_prediction_stats_tags, 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Mean", _test_prediction_stats_tags, 1), + ( + "MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Percentile25", + _test_prediction_stats_tags, + 1, + ), + ( + "MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Percentile50", + _test_prediction_stats_tags, + 1, + ), + ( + "MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Percentile75", + _test_prediction_stats_tags, + 1, + ), + ( + "MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/StandardDeviation", + _test_prediction_stats_tags, + 1, + ), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Min", _test_prediction_stats_tags, 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Max", _test_prediction_stats_tags, 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Feature/1/Count", _test_prediction_stats_tags, 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Mean", _test_prediction_stats_tags, 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Percentile25", _test_prediction_stats_tags, 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Percentile50", _test_prediction_stats_tags, 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Percentile75", _test_prediction_stats_tags, 1), + ( + "MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/StandardDeviation", + _test_prediction_stats_tags, + 1, + ), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Min", _test_prediction_stats_tags, 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Max", _test_prediction_stats_tags, 1), + ("MLModel/Sklearn/Named/DummyClassifier/Predict/Label/0/Count", _test_prediction_stats_tags, 1), ], ), ( @@ -93,30 +179,66 @@ [0, 1], np.array([["a", 2.0, 3], ["b", 0.5, 4]], dtype="._test" if six.PY3 else "test_prediction_stats:_test" ) @validate_transaction_metrics( expected_transaction_name, - custom_metrics=metrics, + dimensional_metrics=metrics, background_task=True, ) @background_task() @@ -231,7 +461,12 @@ def _test(): _test() -def test_prediction_stats_multilabel_output(): +_test_prediction_stats_multilabel_output_tags = frozenset( + {("modelName", "MultiOutputClassifier"), ("inference_id", ML_METRIC_FORCED_UUID), ("model_version", "0.0.0")} +) + + +def test_prediction_stats_multilabel_output(force_uuid): expected_transaction_name = ( "test_prediction_stats:test_prediction_stats_multilabel_output.._test" if six.PY3 @@ -239,13 +474,21 @@ def test_prediction_stats_multilabel_output(): ) stats = ["Mean", "Percentile25", "Percentile50", "Percentile75", "StandardDeviation", "Min", "Max", "Count"] metrics = [ - ("MLModel/Sklearn/Named/MultiOutputClassifier/Predict/Feature/%s/%s" % (feature_col, stat_name), 1) + ( + "MLModel/Sklearn/Named/MultiOutputClassifier/Predict/Feature/%s/%s" % (feature_col, stat_name), + _test_prediction_stats_multilabel_output_tags, + 1, + ) for feature_col in range(20) for stat_name in stats ] metrics.extend( [ - ("MLModel/Sklearn/Named/MultiOutputClassifier/Predict/Label/%s/%s" % (label_col, stat_name), 1) + ( + "MLModel/Sklearn/Named/MultiOutputClassifier/Predict/Label/%s/%s" % (label_col, stat_name), + _test_prediction_stats_multilabel_output_tags, + 1, + ) for label_col in range(3) for stat_name in stats ] @@ -253,7 +496,7 @@ def test_prediction_stats_multilabel_output(): @validate_transaction_metrics( expected_transaction_name, - custom_metrics=metrics, + dimensional_metrics=metrics, background_task=True, ) @background_task() diff --git a/tox.ini b/tox.ini index 3e2ac0fbd..f0c62b311 100644 --- a/tox.ini +++ b/tox.ini @@ -208,7 +208,7 @@ deps = mlmodel_sklearn: pandas mlmodel_sklearn: protobuf mlmodel_sklearn: numpy - mlmodel_sklearn-scikitlearnlatest: scikit-learn + mlmodel_sklearn-scikitlearnlatest: scikit-learn<1.11.0 mlmodel_sklearn-scikitlearn0101: scikit-learn<1.1 component_djangorestframework-djangorestframework0300: Django<1.9 component_djangorestframework-djangorestframework0300: djangorestframework<3.1 From f6ec42e3af86a2f5f6bfb812446e80efc1b15164 Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Tue, 27 Jun 2023 10:12:25 -0700 Subject: [PATCH 41/54] Fixup dependency pinning --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index b9d6bba01..f38872898 100644 --- a/tox.ini +++ b/tox.ini @@ -209,7 +209,8 @@ deps = mlmodel_sklearn: pandas mlmodel_sklearn: protobuf mlmodel_sklearn: numpy - mlmodel_sklearn-scikitlearnlatest: scikit-learn<1.11.0 + mlmodel_sklearn: scipy<1.11.0 + mlmodel_sklearn-scikitlearnlatest: scikit-learn mlmodel_sklearn-scikitlearn0101: scikit-learn<1.1 component_djangorestframework-djangorestframework0300: Django<1.9 component_djangorestframework-djangorestframework0300: djangorestframework<3.1 From 8e45f4dba00df2168adf0cc6106aa0d062d5c8f7 Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Tue, 27 Jun 2023 14:14:49 -0700 Subject: [PATCH 42/54] Hook up ml event to OTLP (#822) * Use protos and otlp protocol class for ml_events * inferenceData -> InferenceData * Add LogsData import * Add utf-8 encoding for json otlp payload * Cast timestamp to int * Use ml_event validator in tests * Fixup payload tests * Change str_value -> string_value * Move event payload gen into otlp_utils * Fixup: put back print * Fixup: cast as str for py27 * Fixup lint errors * Skip py2 protobuf --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- newrelic/core/agent_protocol.py | 6 +- newrelic/core/data_collector.py | 10 +- newrelic/core/otlp_utils.py | 39 ++- newrelic/hooks/mlmodel_sklearn.py | 4 +- tests/agent_features/test_ml_events.py | 66 +++++ .../_validate_custom_events.py | 96 ------- .../mlmodel_sklearn/test_inference_events.py | 269 ++++++++++-------- tests/mlmodel_sklearn/test_ml_model.py | 181 ++++++------ .../validate_dimensional_metric_payload.py | 2 +- .../validators/validate_ml_event_payload.py | 104 +++++++ .../validators/validate_ml_events.py | 6 +- 11 files changed, 456 insertions(+), 327 deletions(-) delete mode 100644 tests/mlmodel_sklearn/_validate_custom_events.py create mode 100644 tests/testing_support/validators/validate_ml_event_payload.py diff --git a/newrelic/core/agent_protocol.py b/newrelic/core/agent_protocol.py index b661fd0ca..dd4dc264f 100644 --- a/newrelic/core/agent_protocol.py +++ b/newrelic/core/agent_protocol.py @@ -595,11 +595,7 @@ def connect( return protocol def _to_http(self, method, payload=()): - params = dict(self._params) - params["method"] = method - if self._run_token: - params["run_id"] = self._run_token - return params, self._headers, otlp_encode(payload) + return {}, self._headers, otlp_encode(payload) def decode_response(self, response): return response.decode("utf-8") diff --git a/newrelic/core/data_collector.py b/newrelic/core/data_collector.py index 0df6fc3fe..269139664 100644 --- a/newrelic/core/data_collector.py +++ b/newrelic/core/data_collector.py @@ -32,12 +32,10 @@ ) from newrelic.core.agent_streaming import StreamingRpc from newrelic.core.config import global_settings -from newrelic.core.otlp_utils import encode_metric_data +from newrelic.core.otlp_utils import encode_metric_data, encode_ml_event_data _logger = logging.getLogger(__name__) -DIMENSIONAL_METRIC_DATA_TEMP = [] # TODO: REMOVE THIS - class Session(object): PROTOCOL = AgentProtocol @@ -125,10 +123,8 @@ def send_custom_events(self, sampling_info, custom_event_data): def send_ml_events(self, sampling_info, custom_event_data): """Called to submit sample set for machine learning events.""" - - # TODO Make this send to MELT/OTLP endpoint instead of agent listener - payload = (self.agent_run_id, sampling_info, custom_event_data) # TODO this payload will be different - return self._protocol.send("custom_event_data", payload) + payload = encode_ml_event_data(custom_event_data, str(self.agent_run_id)) + return self._otlp_protocol.send("ml_event_data", payload, path="/v1/logs") def send_span_events(self, sampling_info, span_event_data): """Called to submit sample set for span events.""" diff --git a/newrelic/core/otlp_utils.py b/newrelic/core/otlp_utils.py index cd0328bcc..1e0864bd8 100644 --- a/newrelic/core/otlp_utils.py +++ b/newrelic/core/otlp_utils.py @@ -22,8 +22,8 @@ import logging from newrelic.common.encoding_utils import json_encode -from newrelic.core.stats_engine import CountStats, TimeStats from newrelic.core.config import global_settings +from newrelic.core.stats_engine import CountStats, TimeStats _logger = logging.getLogger(__name__) @@ -33,7 +33,7 @@ try: from newrelic.packages.opentelemetry_proto.common_pb2 import AnyValue, KeyValue from newrelic.packages.opentelemetry_proto.logs_pb2 import ( - LogRecord, + LogsData, ResourceLogs, ScopeLogs, ) @@ -58,8 +58,8 @@ except Exception: if otlp_content_setting == "protobuf": raise # Reraise exception if content type explicitly set - else: # Fallback to JSON - otlp_content_setting = "json" + # Fallback to JSON + otlp_content_setting = "json" if otlp_content_setting == "json": @@ -77,7 +77,7 @@ ValueAtQuantile = dict ResourceLogs = dict ScopeLogs = dict - LogRecord = dict + LogsData = dict AGGREGATION_TEMPORALITY_DELTA = 1 OTLP_CONTENT_TYPE = "application/json" @@ -88,9 +88,8 @@ def otlp_encode(payload): _logger.warning( "Using OTLP integration while protobuf is not installed. This may result in larger payload sizes and data loss." ) - return json_encode(payload) - else: - return payload.SerializeToString() + return json_encode(payload).encode("utf-8") + return payload.SerializeToString() def create_key_value(key, value): @@ -216,3 +215,27 @@ def encode_metric_data(metric_data, start_time, end_time, resource=None, scope=N ) ] ) + + +def encode_ml_event_data(custom_event_data, agent_run_id): + resource = create_resource() + ml_events = [] + for event in custom_event_data: + event_info, event_attrs = event + event_attrs.update( + { + "real_agent_id": agent_run_id, + "event.domain": "newrelic.ml_events", + "event.name": event_info["type"], + } + ) + ml_attrs = create_key_values_from_iterable(event_attrs) + unix_nano_timestamp = event_info["timestamp"] * 1e6 + ml_events.append( + { + "time_unix_nano": int(unix_nano_timestamp), + "attributes": ml_attrs, + } + ) + + return LogsData(resource_logs=[ResourceLogs(resource=resource, scope_logs=[ScopeLogs(log_records=ml_events)])]) diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index c7c1d30bb..a0b28e494 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -214,7 +214,7 @@ def create_label_event(transaction, class_, inference_id, instance, return_val): # Don't include the raw value when inference_event_value is disabled. if settings and settings.machine_learning.inference_events_value.enabled: event["label_value"] = str(value) - transaction.record_custom_event("inferenceData", event) + transaction.record_ml_event("InferenceData", event) def _get_label_names(user_defined_label_names, prediction_array): @@ -319,7 +319,7 @@ def create_feature_event(transaction, class_, inference_id, instance, args, kwar # Don't include the raw value when inference_event_value is disabled. if settings and settings.machine_learning and settings.machine_learning.inference_events_value.enabled: event["feature_value"] = str(value) - transaction.record_custom_event("inferenceData", event) + transaction.record_ml_event("InferenceData", event) def _nr_instrument_model(module, model_class): diff --git a/tests/agent_features/test_ml_events.py b/tests/agent_features/test_ml_events.py index f0dcf33c6..1c275a4d6 100644 --- a/tests/agent_features/test_ml_events.py +++ b/tests/agent_features/test_ml_events.py @@ -21,14 +21,27 @@ reset_core_stats_engine, ) from testing_support.validators.validate_ml_event_count import validate_ml_event_count +from testing_support.validators.validate_ml_event_payload import ( + validate_ml_event_payload, +) from testing_support.validators.validate_ml_events import validate_ml_events from testing_support.validators.validate_ml_events_outside_transaction import ( validate_ml_events_outside_transaction, ) +import newrelic.core.otlp_utils from newrelic.api.application import application_instance as application from newrelic.api.background_task import background_task from newrelic.api.transaction import record_ml_event +from newrelic.core.config import global_settings +from newrelic.packages import six + +try: + # python 2.x + reload +except NameError: + # python 3.x + from importlib import reload _now = time.time() @@ -38,6 +51,38 @@ } +@pytest.fixture(scope="session") +def core_app(collector_agent_registration): + app = collector_agent_registration + return app._agent.application(app.name) + + +@validate_ml_event_payload( + [{"foo": "bar", "real_agent_id": "1234567", "event.domain": "newrelic.ml_events", "event.name": "InferenceEvent"}] +) +@reset_core_stats_engine() +def test_ml_event_payload_inside_transaction(core_app): + @background_task(name="test_ml_event_payload_inside_transaction") + def _test(): + record_ml_event("InferenceEvent", {"foo": "bar"}) + + _test() + core_app.harvest() + + +@validate_ml_event_payload( + [{"foo": "bar", "real_agent_id": "1234567", "event.domain": "newrelic.ml_events", "event.name": "InferenceEvent"}] +) +@reset_core_stats_engine() +def test_ml_event_payload_outside_transaction(core_app): + def _test(): + app = application() + record_ml_event("InferenceEvent", {"foo": "bar"}, application=app) + + _test() + core_app.harvest() + + @pytest.mark.parametrize( "params,expected", [ @@ -47,6 +92,7 @@ ], ids=["Valid key/value", "Bad key", "Value too long"], ) +@reset_core_stats_engine() def test_record_ml_event_inside_transaction(params, expected): @validate_ml_events(expected) @background_task() @@ -75,6 +121,7 @@ def _test(): _test() +@reset_core_stats_engine() @validate_ml_event_count(count=0) @background_task() def test_record_ml_event_inside_transaction_bad_event_type(): @@ -88,6 +135,7 @@ def test_record_ml_event_outside_transaction_bad_event_type(): record_ml_event("!@#$%^&*()", {"foo": "bar"}, application=app) +@reset_core_stats_engine() @validate_ml_event_count(count=0) @background_task() def test_record_ml_event_inside_transaction_params_not_a_dict(): @@ -120,6 +168,7 @@ def test_ml_event_settings_check_ml_insights_enabled(): @override_application_settings({"ml_insights_events.enabled": False}) +@reset_core_stats_engine() @function_not_called("newrelic.api.transaction", "create_custom_event") @background_task() def test_transaction_create_ml_event_not_called(): @@ -127,8 +176,25 @@ def test_transaction_create_ml_event_not_called(): @override_application_settings({"ml_insights_events.enabled": False}) +@reset_core_stats_engine() @function_not_called("newrelic.core.application", "create_custom_event") @background_task() def test_application_create_ml_event_not_called(): app = application() record_ml_event("FooEvent", {"foo": "bar"}, application=app) + + +@pytest.fixture(scope="module", autouse=True, params=["protobuf", "json"]) +def otlp_content_encoding(request): + if six.PY2 and request.param == "protobuf": + pytest.skip("OTLP protos are not compatible with Python 2.") + + _settings = global_settings() + prev = _settings.debug.otlp_content_encoding + _settings.debug.otlp_content_encoding = request.param + reload(newrelic.core.otlp_utils) + assert newrelic.core.otlp_utils.otlp_content_setting == request.param, "Content encoding mismatch." + + yield + + _settings.debug.otlp_content_encoding = prev diff --git a/tests/mlmodel_sklearn/_validate_custom_events.py b/tests/mlmodel_sklearn/_validate_custom_events.py deleted file mode 100644 index 77a63497f..000000000 --- a/tests/mlmodel_sklearn/_validate_custom_events.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import copy - -from testing_support.fixtures import catch_background_exceptions - -from newrelic.common.object_wrapper import function_wrapper, transient_function_wrapper -from newrelic.packages import six - - -def validate_custom_events(events): - @function_wrapper - def _validate_wrapper(wrapped, instance, args, kwargs): - - record_called = [] - recorded_events = [] - - @transient_function_wrapper("newrelic.core.stats_engine", "StatsEngine.record_transaction") - @catch_background_exceptions - def _validate_custom_events(wrapped, instance, args, kwargs): - record_called.append(True) - try: - result = wrapped(*args, **kwargs) - except: - raise - else: - txn = args[0] - recorded_events[:] = [] - recorded_events.extend(list(txn.custom_events)) - - return result - - _new_wrapper = _validate_custom_events(wrapped) - val = _new_wrapper(*args, **kwargs) - assert record_called - custom_events = copy.copy(recorded_events) - - record_called[:] = [] - recorded_events[:] = [] - for expected in events: - matching_custom_events = 0 - mismatches = [] - for captured in custom_events: - if _check_custom_event_attributes(expected, captured, mismatches): - matching_custom_events += 1 - assert matching_custom_events == 1, _custom_event_details(matching_custom_events, custom_events, mismatches) - - return val - - def _check_custom_event_attributes(expected, captured, mismatches): - assert len(captured) == 2 # [intrinsic, user attributes] - expected_intrinsics = expected.get("intrinsics", {}) - expected_users = expected.get("users", {}) - intrinsics = captured[0] - users = captured[1] - - def _validate(expected, captured): - for key, value in six.iteritems(expected): - if key in captured: - - captured_value = captured[key] - else: - mismatches.append("key: %s, value:<%s><%s>" % (key, value, getattr(captured, key, None))) - return False - - if value is not None: - if value != captured_value: - mismatches.append("key: %s, value:<%s><%s>" % (key, value, captured_value)) - return False - - return True - - return _validate(expected_intrinsics, intrinsics) and _validate(expected_users, users) - - def _custom_event_details(matching_custom_events, captured, mismatches): - details = [ - "matching_custom_events=%d" % matching_custom_events, - "mismatches=%s" % mismatches, - "captured_events=%s" % captured, - ] - - return "\n".join(details) - - return _validate_wrapper diff --git a/tests/mlmodel_sklearn/test_inference_events.py b/tests/mlmodel_sklearn/test_inference_events.py index 2cf71ce94..5b9974cd3 100644 --- a/tests/mlmodel_sklearn/test_inference_events.py +++ b/tests/mlmodel_sklearn/test_inference_events.py @@ -16,53 +16,56 @@ import numpy as np import pandas -from _validate_custom_events import validate_custom_events from testing_support.fixtures import ( override_application_settings, reset_core_stats_engine, - validate_custom_event_count, ) +from testing_support.validators.validate_ml_event_count import validate_ml_event_count +from testing_support.validators.validate_ml_events import validate_ml_events from newrelic.api.background_task import background_task pandas_df_category_recorded_custom_events = [ - { - "users": { + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "DecisionTreeClassifier", "model_version": "0.0.0", "feature_name": "col1", "feature_type": "categorical", "feature_value": "2.0", - } - }, - { - "users": { + }, + ), + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "DecisionTreeClassifier", "model_version": "0.0.0", "feature_name": "col2", "feature_type": "categorical", "feature_value": "4.0", - } - }, - { - "users": { + }, + ), + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "DecisionTreeClassifier", "model_version": "0.0.0", "label_name": "0", "label_type": "numeric", "label_value": "27.0", - } - }, + }, + ), ] @reset_core_stats_engine() def test_pandas_df_categorical_feature_event(): - @validate_custom_events(pandas_df_category_recorded_custom_events) - @validate_custom_event_count(count=3) + @validate_ml_events(pandas_df_category_recorded_custom_events) + @validate_ml_event_count(count=3) @background_task() def _test(): import sklearn.tree @@ -83,43 +86,46 @@ def _test(): true_label_value = "True" if sys.version_info < (3, 8) else "1.0" false_label_value = "False" if sys.version_info < (3, 8) else "0.0" pandas_df_bool_recorded_custom_events = [ - { - "users": { + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "DecisionTreeClassifier", "model_version": "0.0.0", "feature_name": "col1", "feature_type": "bool", "feature_value": "True", - } - }, - { - "users": { + }, + ), + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "DecisionTreeClassifier", "model_version": "0.0.0", "feature_name": "col2", "feature_type": "bool", "feature_value": "True", - } - }, - { - "users": { + }, + ), + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "DecisionTreeClassifier", "model_version": "0.0.0", "label_name": "0", "label_type": label_type, "label_value": true_label_value, - } - }, + }, + ), ] @reset_core_stats_engine() def test_pandas_df_bool_feature_event(): - @validate_custom_events(pandas_df_bool_recorded_custom_events) - @validate_custom_event_count(count=3) + @validate_ml_events(pandas_df_bool_recorded_custom_events) + @validate_ml_event_count(count=3) @background_task() def _test(): import sklearn.tree @@ -139,43 +145,46 @@ def _test(): pandas_df_float_recorded_custom_events = [ - { - "users": { + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "DecisionTreeRegressor", "model_version": "0.0.0", "feature_name": "col1", "feature_type": "numeric", "feature_value": "100.0", - } - }, - { - "users": { + }, + ), + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "DecisionTreeRegressor", "model_version": "0.0.0", "feature_name": "col2", "feature_type": "numeric", "feature_value": "300.0", - } - }, - { - "users": { + }, + ), + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "DecisionTreeRegressor", "model_version": "0.0.0", "label_name": "0", "label_type": "numeric", "label_value": "345.6", - } - }, + }, + ), ] @reset_core_stats_engine() def test_pandas_df_float_feature_event(): - @validate_custom_events(pandas_df_float_recorded_custom_events) - @validate_custom_event_count(count=3) + @validate_ml_events(pandas_df_float_recorded_custom_events) + @validate_ml_event_count(count=3) @background_task() def _test(): import sklearn.tree @@ -195,43 +204,46 @@ def _test(): int_list_recorded_custom_events = [ - { - "users": { + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "ExtraTreeRegressor", "model_version": "0.0.0", "feature_name": "0", "feature_type": "numeric", "feature_value": "1", - } - }, - { - "users": { + }, + ), + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "ExtraTreeRegressor", "model_version": "0.0.0", "feature_name": "1", "feature_type": "numeric", "feature_value": "2", - } - }, - { - "users": { + }, + ), + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "ExtraTreeRegressor", "model_version": "0.0.0", "label_name": "0", "label_type": "numeric", "label_value": "1.0", - } - }, + }, + ), ] @reset_core_stats_engine() def test_int_list(): - @validate_custom_events(int_list_recorded_custom_events) - @validate_custom_event_count(count=3) + @validate_ml_events(int_list_recorded_custom_events) + @validate_ml_event_count(count=3) @background_task() def _test(): import sklearn.tree @@ -250,43 +262,46 @@ def _test(): numpy_int_recorded_custom_events = [ - { - "users": { + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "ExtraTreeRegressor", "model_version": "0.0.0", "feature_name": "0", "feature_type": "numeric", "feature_value": "12", - } - }, - { - "users": { + }, + ), + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "ExtraTreeRegressor", "model_version": "0.0.0", "feature_name": "1", "feature_type": "numeric", "feature_value": "13", - } - }, - { - "users": { + }, + ), + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "ExtraTreeRegressor", "model_version": "0.0.0", "label_name": "0", "label_type": "numeric", "label_value": "11.0", - } - }, + }, + ), ] @reset_core_stats_engine() def test_numpy_int_array(): - @validate_custom_events(numpy_int_recorded_custom_events) - @validate_custom_event_count(count=3) + @validate_ml_events(numpy_int_recorded_custom_events) + @validate_ml_event_count(count=3) @background_task() def _test(): import sklearn.tree @@ -305,53 +320,57 @@ def _test(): numpy_str_recorded_custom_events = [ - { - "users": { + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "DecisionTreeClassifier", "model_version": "0.0.0", "feature_name": "0", "feature_type": "str", "feature_value": "20", - } - }, - { - "users": { + }, + ), + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "DecisionTreeClassifier", "model_version": "0.0.0", "feature_name": "1", "feature_type": "str", "feature_value": "21", - } - }, - { - "users": { + }, + ), + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "DecisionTreeClassifier", "model_version": "0.0.0", "feature_name": "0", "feature_type": "str", "feature_value": "22", - } - }, - { - "users": { + }, + ), + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "DecisionTreeClassifier", "model_version": "0.0.0", "feature_name": "1", "feature_type": "str", "feature_value": "23", - } - }, + }, + ), ] @reset_core_stats_engine() def test_numpy_str_array_multiple_features(): - @validate_custom_events(numpy_str_recorded_custom_events) - @validate_custom_event_count(count=6) + @validate_ml_events(numpy_str_recorded_custom_events) + @validate_ml_event_count(count=6) @background_task() def _test(): import sklearn.tree @@ -370,41 +389,44 @@ def _test(): numpy_str_recorded_custom_events_no_value = [ - { - "users": { + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "DecisionTreeClassifier", "model_version": "0.0.0", "feature_name": "0", "feature_type": "str", - } - }, - { - "users": { + }, + ), + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "DecisionTreeClassifier", "model_version": "0.0.0", "feature_name": "1", "feature_type": "str", - } - }, - { - "users": { + }, + ), + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "DecisionTreeClassifier", "model_version": "0.0.0", "label_name": "0", "label_type": "str", - } - }, + }, + ), ] @reset_core_stats_engine() @override_application_settings({"machine_learning.inference_events_value.enabled": False}) def test_does_not_include_value_when_inference_event_value_enabled_is_false(): - @validate_custom_events(numpy_str_recorded_custom_events_no_value) - @validate_custom_event_count(count=3) + @validate_ml_events(numpy_str_recorded_custom_events_no_value) + @validate_ml_event_count(count=3) @background_task() def _test(): import sklearn.tree @@ -423,14 +445,14 @@ def _test(): @reset_core_stats_engine() -@override_application_settings({"custom_insights_events.enabled": False}) -def test_does_not_include_events_when_custom_insights_events_enabled_is_false(): +@override_application_settings({"ml_insights_events.enabled": False}) +def test_does_not_include_events_when_ml_insights_events_enabled_is_false(): """ Verifies that all ml events can be disabled by setting custom_insights_events.enabled. """ - @validate_custom_event_count(count=0) + @validate_ml_event_count(count=0) @background_task() def _test(): import sklearn.tree @@ -451,7 +473,7 @@ def _test(): @reset_core_stats_engine() @override_application_settings({"machine_learning.enabled": False}) def test_does_not_include_events_when_machine_learning_enabled_is_false(): - @validate_custom_event_count(count=0) + @validate_ml_event_count(count=0) @background_task() def _test(): import sklearn.tree @@ -470,44 +492,47 @@ def _test(): multilabel_output_label_events = [ - { - "users": { + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "MultiOutputClassifier", "model_version": "0.0.0", "label_name": "0", "label_type": "numeric", "label_value": "1", - } - }, - { - "users": { + }, + ), + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "MultiOutputClassifier", "model_version": "0.0.0", "label_name": "1", "label_type": "numeric", "label_value": "0", - } - }, - { - "users": { + }, + ), + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "MultiOutputClassifier", "model_version": "0.0.0", "label_name": "2", "label_type": "numeric", "label_value": "1", - } - }, + }, + ), ] @reset_core_stats_engine() def test_custom_event_count_multilabel_output(): - @validate_custom_events(multilabel_output_label_events) + @validate_ml_events(multilabel_output_label_events) # The expected count of 23 comes from 20 feature events + 3 label events to be generated - @validate_custom_event_count(count=23) + @validate_ml_event_count(count=23) @background_task() def _test(): from sklearn.datasets import make_multilabel_classification diff --git a/tests/mlmodel_sklearn/test_ml_model.py b/tests/mlmodel_sklearn/test_ml_model.py index 571126e6c..65fb1eabb 100644 --- a/tests/mlmodel_sklearn/test_ml_model.py +++ b/tests/mlmodel_sklearn/test_ml_model.py @@ -16,11 +16,9 @@ import pandas import six -from _validate_custom_events import validate_custom_events -from testing_support.fixtures import ( - reset_core_stats_engine, - validate_custom_event_count, -) +from testing_support.fixtures import reset_core_stats_engine +from testing_support.validators.validate_ml_event_count import validate_ml_event_count +from testing_support.validators.validate_ml_events import validate_ml_events from newrelic.api.background_task import background_task from newrelic.api.ml_model import wrap_mlmodel @@ -115,43 +113,46 @@ def predict(self, X, check_input=True): label_value = "1.0" if six.PY2 else "0.5" int_list_recorded_custom_events = [ - { - "users": { + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "MyCustomModel", "model_version": "1.2.3", "feature_name": "0", "feature_type": "numeric", "feature_value": "1.0", - } - }, - { - "users": { + }, + ), + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "MyCustomModel", "model_version": "1.2.3", "feature_name": "1", "feature_type": "numeric", "feature_value": "2.0", - } - }, - { - "users": { + }, + ), + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "MyCustomModel", "model_version": "1.2.3", "label_name": "0", "label_type": "numeric", "label_value": label_value, - } - }, + }, + ), ] @reset_core_stats_engine() def test_custom_model_int_list_no_features_and_labels(): - @validate_custom_event_count(count=3) - @validate_custom_events(int_list_recorded_custom_events) + @validate_ml_event_count(count=3) + @validate_ml_events(int_list_recorded_custom_events) @background_task() def _test(): x_train = [[0, 0], [1, 1]] @@ -169,53 +170,57 @@ def _test(): pandas_df_recorded_custom_events = [ - { - "users": { + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "PandasTestModel", "model_version": "1.5.0b1", "feature_name": "feature1", "feature_type": "categorical", "feature_value": "0", - } - }, - { - "users": { + }, + ), + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "PandasTestModel", "model_version": "1.5.0b1", "feature_name": "feature2", "feature_type": "categorical", "feature_value": "0", - } - }, - { - "users": { + }, + ), + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "PandasTestModel", "model_version": "1.5.0b1", "feature_name": "feature3", "feature_type": "categorical", "feature_value": "1", - } - }, - { - "users": { + }, + ), + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "PandasTestModel", "model_version": "1.5.0b1", "label_name": "label1", "label_type": "numeric", "label_value": "0.5" if six.PY3 else "0.0", - } - }, + }, + ), ] @reset_core_stats_engine() def test_wrapper_attrs_custom_model_pandas_df(): - @validate_custom_event_count(count=4) - @validate_custom_events(pandas_df_recorded_custom_events) + @validate_ml_event_count(count=4) + @validate_ml_events(pandas_df_recorded_custom_events) @background_task() def _test(): x_train = pandas.DataFrame({"col1": [0, 1], "col2": [0, 1], "col3": [1, 2]}, dtype="category") @@ -237,43 +242,46 @@ def _test(): pandas_df_recorded_builtin_events = [ - { - "users": { + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "MyDecisionTreeClassifier", "model_version": "1.5.0b1", "feature_name": "feature1", "feature_type": "numeric", "feature_value": "12", - } - }, - { - "users": { + }, + ), + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "MyDecisionTreeClassifier", "model_version": "1.5.0b1", "feature_name": "feature2", "feature_type": "numeric", "feature_value": "14", - } - }, - { - "users": { + }, + ), + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "MyDecisionTreeClassifier", "model_version": "1.5.0b1", "label_name": "label1", "label_type": "numeric", "label_value": "0", - } - }, + }, + ), ] @reset_core_stats_engine() def test_wrapper_attrs_builtin_model(): - @validate_custom_event_count(count=3) - @validate_custom_events(pandas_df_recorded_builtin_events) + @validate_ml_event_count(count=3) + @validate_ml_events(pandas_df_recorded_builtin_events) @background_task() def _test(): import sklearn.tree @@ -300,53 +308,57 @@ def _test(): pandas_df_mismatched_custom_events = [ - { - "users": { + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "MyDecisionTreeClassifier", "model_version": "1.5.0b1", "feature_name": "col1", "feature_type": "numeric", "feature_value": "12", - } - }, - { - "users": { + }, + ), + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "MyDecisionTreeClassifier", "model_version": "1.5.0b1", "feature_name": "col2", "feature_type": "numeric", "feature_value": "14", - } - }, - { - "users": { + }, + ), + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "MyDecisionTreeClassifier", "model_version": "1.5.0b1", "feature_name": "col3", "feature_type": "numeric", "feature_value": "16", - } - }, - { - "users": { + }, + ), + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "MyDecisionTreeClassifier", "model_version": "1.5.0b1", "label_name": "0", "label_type": "numeric", "label_value": "1", - } - }, + }, + ), ] @reset_core_stats_engine() def test_wrapper_mismatched_features_and_labels_df(): - @validate_custom_event_count(count=4) - @validate_custom_events(pandas_df_mismatched_custom_events) + @validate_ml_event_count(count=4) + @validate_ml_events(pandas_df_mismatched_custom_events) @background_task() def _test(): import sklearn.tree @@ -372,43 +384,46 @@ def _test(): numpy_str_mismatched_custom_events = [ - { - "users": { + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "MyDecisionTreeClassifier", "model_version": "0.0.1", "feature_name": "0", "feature_type": "str", "feature_value": "20", - } - }, - { - "users": { + }, + ), + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "MyDecisionTreeClassifier", "model_version": "0.0.1", "feature_name": "1", "feature_type": "str", "feature_value": "21", - } - }, - { - "users": { + }, + ), + ( + {"type": "InferenceData"}, + { "inference_id": None, "modelName": "MyDecisionTreeClassifier", "model_version": "0.0.1", "label_name": "0", "label_type": "str", "label_value": "21", - } - }, + }, + ), ] @reset_core_stats_engine() def test_wrapper_mismatched_features_and_labels_np_array(): - @validate_custom_events(numpy_str_mismatched_custom_events) - @validate_custom_event_count(count=3) + @validate_ml_events(numpy_str_mismatched_custom_events) + @validate_ml_event_count(count=3) @background_task() def _test(): import numpy as np diff --git a/tests/testing_support/validators/validate_dimensional_metric_payload.py b/tests/testing_support/validators/validate_dimensional_metric_payload.py index 58523e2fd..d587e02b7 100644 --- a/tests/testing_support/validators/validate_dimensional_metric_payload.py +++ b/tests/testing_support/validators/validate_dimensional_metric_payload.py @@ -39,7 +39,7 @@ def attribute_to_value(attribute): return float(attribute_value) elif attribute_type == "bool_value": return bool(attribute_value) - elif attribute_type == "str_value": + elif attribute_type == "string_value": return str(attribute_value) else: raise TypeError("Invalid attribute type: %s" % attribute_type) diff --git a/tests/testing_support/validators/validate_ml_event_payload.py b/tests/testing_support/validators/validate_ml_event_payload.py new file mode 100644 index 000000000..4d43cbb22 --- /dev/null +++ b/tests/testing_support/validators/validate_ml_event_payload.py @@ -0,0 +1,104 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from newrelic.common.object_wrapper import function_wrapper, transient_function_wrapper +from newrelic.core.otlp_utils import otlp_content_setting + +if otlp_content_setting == "protobuf": + from google.protobuf.json_format import MessageToDict +else: + MessageToDict = None + + +def attribute_to_value(attribute): + attribute_type, attribute_value = next(iter(attribute.items())) + if attribute_type == "int_value": + return int(attribute_value) + elif attribute_type == "double_value": + return float(attribute_value) + elif attribute_type == "bool_value": + return bool(attribute_value) + elif attribute_type == "string_value": + return str(attribute_value) + else: + raise TypeError("Invalid attribute type: %s" % attribute_type) + + +def payload_to_ml_events(payload): + if type(payload) is not dict: + message = MessageToDict(payload, use_integers_for_enums=True, preserving_proto_field_name=True) + else: + message = payload + + resource_logs = message.get("resource_logs") + assert len(resource_logs) == 1 + resource_logs = resource_logs[0] + resource = resource_logs.get("resource") + assert resource and resource.get("attributes")[0] == { + "key": "instrumentation.provider", + "value": {"string_value": "newrelic-opentelemetry-python-ml"}, + } + scope_logs = resource_logs.get("scope_logs") + assert len(scope_logs) == 1 + scope_logs = scope_logs[0] + + scope = scope_logs.get("scope") + assert scope is None + logs = scope_logs.get("log_records") + + return logs + + +def validate_ml_event_payload(ml_events=None): + # Validates OTLP events as they are sent to the collector. + + ml_events = ml_events or [] + + @function_wrapper + def _validate_wrapper(wrapped, instance, args, kwargs): + recorded_ml_events = [] + + @transient_function_wrapper("newrelic.core.agent_protocol", "OtlpProtocol.send") + def send_request_wrapper(wrapped, instance, args, kwargs): + def _bind_params(method, payload=(), *args, **kwargs): + return method, payload + + method, payload = _bind_params(*args, **kwargs) + + if method == "ml_event_data" and payload: + recorded_ml_events.append(payload) + + return wrapped(*args, **kwargs) + + wrapped = send_request_wrapper(wrapped) + val = wrapped(*args, **kwargs) + assert recorded_ml_events + + decoded_payloads = [payload_to_ml_events(payload) for payload in recorded_ml_events] + all_logs = [] + for sent_logs in decoded_payloads: + for data_point in sent_logs: + for key in ("time_unix_nano",): + assert key in data_point, "Invalid log format. Missing key: %s" % key + + all_logs.append( + {attr["key"]: attribute_to_value(attr["value"]) for attr in (data_point.get("attributes") or [])} + ) + + for expected_event in ml_events: + assert expected_event in all_logs, "%s Not Found. Got: %s" % (expected_event, all_logs) + + return val + + return _validate_wrapper diff --git a/tests/testing_support/validators/validate_ml_events.py b/tests/testing_support/validators/validate_ml_events.py index 8f3150225..251e8dbe7 100644 --- a/tests/testing_support/validators/validate_ml_events.py +++ b/tests/testing_support/validators/validate_ml_events.py @@ -44,7 +44,7 @@ def _validate_ml_events(wrapped, instance, args, kwargs): _new_wrapper = _validate_ml_events(wrapped) val = _new_wrapper(*args, **kwargs) assert record_called - events = copy.copy(recorded_events) + found_events = copy.copy(recorded_events) record_called[:] = [] recorded_events[:] = [] @@ -52,7 +52,7 @@ def _validate_ml_events(wrapped, instance, args, kwargs): for expected in events: matching_ml_events = 0 mismatches = [] - for captured in events: + for captured in found_events: if _check_event_attributes(expected, captured, mismatches): matching_ml_events += 1 assert matching_ml_events == 1, _event_details(matching_ml_events, events, mismatches) @@ -82,7 +82,7 @@ def _check_event_attributes(expected, captured, mismatches): extra_keys = captured_keys - expected_keys if extra_keys: - mismatches.append("extra_keys: %s" % tuple(extra_keys)) + mismatches.append("extra_keys: %s" % str(tuple(extra_keys))) return False for key, value in six.iteritems(expected[1]): From db915d074a9cbaf5c5ed1ac3bc5876fbcd9c7107 Mon Sep 17 00:00:00 2001 From: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Wed, 28 Jun 2023 12:59:13 -0700 Subject: [PATCH 43/54] Fix OTLP Count Metric Serialization (#856) * Fix metric filtering in OTLP encoding * Add regression test for duplicate metrics * Make error message more clear * Add pylint ignore C0123 * Add explanation comment * Linting fixups --- newrelic/core/otlp_utils.py | 12 ++++--- .../test_dimensional_metrics.py | 35 ++++++++++++++++--- .../validate_dimensional_metric_payload.py | 4 +-- 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/newrelic/core/otlp_utils.py b/newrelic/core/otlp_utils.py index 1e0864bd8..e78a63603 100644 --- a/newrelic/core/otlp_utils.py +++ b/newrelic/core/otlp_utils.py @@ -84,7 +84,7 @@ def otlp_encode(payload): - if type(payload) is dict: + if type(payload) is dict: # pylint: disable=C0123 _logger.warning( "Using OTLP integration while protobuf is not installed. This may result in larger payload sizes and data loss." ) @@ -162,7 +162,9 @@ def stats_to_otlp_metrics(metric_data, start_time, end_time): separate the types and report multiple metrics, one for each type. """ for name, metric_container in metric_data: - if any(isinstance(metric, CountStats) for metric in metric_container.values()): + # Types are checked here using type() instead of isinstance, as CountStats is a subclass of TimeStats. + # Imporperly checking with isinstance will lead to count metrics being encoded and reported twice. + if any(type(metric) is CountStats for metric in metric_container.values()): # pylint: disable=C0123 # Metric contains Sum metric data points. yield Metric( name=name, @@ -177,11 +179,11 @@ def stats_to_otlp_metrics(metric_data, start_time, end_time): attributes=create_key_values_from_iterable(tags), ) for tags, value in metric_container.items() - if isinstance(value, CountStats) + if type(value) is CountStats # pylint: disable=C0123 ], ), ) - if any(isinstance(metric, TimeStats) for metric in metric_container.values()): + if any(type(metric) is TimeStats for metric in metric_container.values()): # pylint: disable=C0123 # Metric contains Summary metric data points. yield Metric( name=name, @@ -194,7 +196,7 @@ def stats_to_otlp_metrics(metric_data, start_time, end_time): attributes=create_key_values_from_iterable(tags), ) for tags, value in metric_container.items() - if isinstance(value, TimeStats) + if type(value) is TimeStats # pylint: disable=C0123 ] ), ) diff --git a/tests/agent_features/test_dimensional_metrics.py b/tests/agent_features/test_dimensional_metrics.py index 6d38fe05f..ef9e98418 100644 --- a/tests/agent_features/test_dimensional_metrics.py +++ b/tests/agent_features/test_dimensional_metrics.py @@ -24,6 +24,7 @@ validate_transaction_metrics, ) +import newrelic.core.otlp_utils from newrelic.api.application import application_instance from newrelic.api.background_task import background_task from newrelic.api.transaction import ( @@ -31,12 +32,9 @@ record_dimensional_metrics, ) from newrelic.common.metric_utils import create_metric_identity - -import newrelic.core.otlp_utils from newrelic.core.config import global_settings from newrelic.packages import six - try: # python 2.x reload @@ -55,7 +53,7 @@ def otlp_content_encoding(request): _settings.debug.otlp_content_encoding = request.param reload(newrelic.core.otlp_utils) assert newrelic.core.otlp_utils.otlp_content_setting == request.param, "Content encoding mismatch." - + yield _settings.debug.otlp_content_encoding = prev @@ -177,7 +175,7 @@ def _test(): ("Metric.NotPresent", None, None), ], ) -def test_dimensional_metric_payload(): +def test_dimensional_metrics_payload(): @background_task(name="test_dimensional_metric_payload") def _test(): record_dimensional_metrics( @@ -197,3 +195,30 @@ def _test(): app = application_instance() core_app = app._agent.application(app.name) core_app.harvest() + + +@reset_core_stats_engine() +@validate_dimensional_metric_payload( + summary_metrics=[ + ("Metric.Summary", None, 1), + ("Metric.Count", None, None), # Should NOT be present + ], + count_metrics=[ + ("Metric.Count", None, 1), + ("Metric.Summary", None, None), # Should NOT be present + ], +) +def test_dimensional_metrics_no_duplicate_encodings(): + @background_task(name="test_dimensional_metric_payload") + def _test(): + record_dimensional_metrics( + [ + ("Metric.Summary", 1), + ("Metric.Count", {"count": 1}), + ] + ) + + _test() + app = application_instance() + core_app = app._agent.application(app.name) + core_app.harvest() diff --git a/tests/testing_support/validators/validate_dimensional_metric_payload.py b/tests/testing_support/validators/validate_dimensional_metric_payload.py index d587e02b7..2f4f48c07 100644 --- a/tests/testing_support/validators/validate_dimensional_metric_payload.py +++ b/tests/testing_support/validators/validate_dimensional_metric_payload.py @@ -117,7 +117,7 @@ def _bind_params(method, payload=(), *args, **kwargs): if not count: if metric in sent_summary_metrics: data_points = data_points_to_dict(sent_summary_metrics[metric]["summary"]["data_points"]) - assert tags not in data_points, "(%s, %s) Found." % (metric, tags and dict(tags)) + assert tags not in data_points, "(%s, %s) Unexpected but found." % (metric, tags and dict(tags)) else: assert metric in sent_summary_metrics, "%s Not Found. Got: %s" % ( metric, @@ -153,7 +153,7 @@ def _bind_params(method, payload=(), *args, **kwargs): if not count: if metric in sent_count_metrics: data_points = data_points_to_dict(sent_count_metrics[metric]["sum"]["data_points"]) - assert tags not in data_points, "(%s, %s) Found." % (metric, tags and dict(tags)) + assert tags not in data_points, "(%s, %s) Unexpected but found." % (metric, tags and dict(tags)) else: assert metric in sent_count_metrics, "%s Not Found. Got: %s" % ( metric, From c77adce25ae5356b5abbd8faf0e4a271080d617b Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Mon, 31 Jul 2023 12:06:33 -0700 Subject: [PATCH 44/54] Merge main (#874) * Exclude command line functionality from test coverage (#855) * FIX: resilient environment settings (#825) if the application uses generalimport to manage optional depedencies, it's possible that generalimport.MissingOptionalDependency is raised. In this case, we should not report the module as it is not actually loaded and is not a runtime dependency of the application. Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Hannah Stepanek * Replace drop_transaction logic by using transaction context manager (#832) * Replace drop_transaction call * [Mega-Linter] Apply linters fixes * Empty commit to start tests * Change logic in BG Wrappers --------- Co-authored-by: lrafeei Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * Upgrade to Pypy38 for TypedDict (#861) * Fix base branch * Revert tox dependencies * Replace all pypy37 with pypy38 * Remove action.yml file * Push Empty Commit * Fix skip_missing_interpreters behavior * Fix skip_missing_interpreters behavior * Pin dev CI image sha * Remove unsupported Tornado tests * Add latest tests to Tornado * Remove pypy38 (for now) --------- Co-authored-by: Tim Pansino * Add profile_trace testing (#858) * Include isort stdlibs for determining stdlib modules * Use isort & sys to eliminate std & builtin modules Previously, the logic would fail to identify third party modules installed within the local user socpe. This fixes that issue by skipping builtin and stdlib modules by name, instead of attempting to identify third party modules based on file paths. * Handle importlib_metadata.version being a callable * Add isort into third party notices * [Mega-Linter] Apply linters fixes * Remove Python 2.7 and pypy2 testing (#835) * Change setup-python to @v2 for py2.7 * Remove py27 and pypy testing * Fix syntax errors * Fix comma related syntax errors * Fix more issues in tox * Remove gearman test * Containerized CI Pipeline (#836) * Revert "Remove Python 2.7 and pypy2 testing (#835)" This reverts commit abb6405d2bfd629ed83f48e8a17b4a28e3a3c352. * Containerize CI process * Publish new docker container for CI images * Rename github actions job * Copyright tag scripts * Drop debug line * Swap to new CI image * Move pip install to just main python * Remove libcurl special case from tox * Install special case packages into main image * Remove unused packages * Remove all other triggers besides manual * Add make run command * Cleanup small bugs * Fix CI Image Tagging (#838) * Correct templated CI image name * Pin pypy2.7 in image * Fix up scripting * Temporarily Restore Old CI Pipeline (#841) * Restore old pipelines * Remove python 2 from setup-python * Rework CI Pipeline (#839) Change pypy to pypy27 in tox. Fix checkout logic Pin tox requires * Fix Tests on New CI (#843) * Remove non-root user * Test new CI image * Change pypy to pypy27 in tox. * Fix checkout logic * Fetch git tags properly * Pin tox requires * Adjust default db settings for github actions * Rename elasticsearch services * Reset to new pipelines * [Mega-Linter] Apply linters fixes * Fix timezone * Fix docker networking * Pin dev image to new sha * Standardize gearman DB settings * Fix elasticsearch settings bug * Fix gearman bug * Add missing odbc headers * Add more debug messages * Swap out dev ci image * Fix required virtualenv version * Swap out dev ci image * Swap out dev ci image * Remove aioredis v1 for EOL * Add coverage paths for docker container * Unpin ci container --------- Co-authored-by: TimPansino * Trigger tests * Add testing for profile trace. * [Mega-Linter] Apply linters fixes * Ignore __call__ from coverage on profile_trace. * [Mega-Linter] Apply linters fixes --------- Co-authored-by: Hannah Stepanek Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: hmstepanek Co-authored-by: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Co-authored-by: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Co-authored-by: TimPansino Co-authored-by: umaannamalai * Add Transaction API Tests (#857) * Test for suppress_apdex_metric * Add custom_metrics tests * Add distributed_trace_headers testing in existing tests * [Mega-Linter] Apply linters fixes * Remove redundant if-statement * Ignore deprecated transaction function from coverage * [Mega-Linter] Apply linters fixes * Push empty commit * Update newrelic/api/transaction.py --------- Co-authored-by: lrafeei Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Co-authored-by: Uma Annamalai * Add tests for jinja2. (#842) * Add tests for jinja2. * [Mega-Linter] Apply linters fixes * Update tox.ini Co-authored-by: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> --------- Co-authored-by: umaannamalai Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> * Add tests for newrelic/config.py (#860) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * Fix starlette testing matrix for updated behavior. (#869) Co-authored-by: Lalleh Rafeei Co-authored-by: Hannah Stepanek Co-authored-by: Uma Annamalai * Correct Serverless Distributed Tracing Logic (#870) * Fix serverless logic for distributed tracing * Test stubs * Collapse testing changes * Add negative testing to regular DT test suite * Apply linter fixes * [Mega-Linter] Apply linters fixes --------- Co-authored-by: TimPansino * Fix Kafka CI (#863) * Reenable kafka testing * Add kafka dev lib * Sync install python with devcontainer * Fix kafka local host setting * Drop set -u flag * Pin CI image dev sha * Add parallel flag to kafka * Fix proper exit status * Build librdkafka from source * Updated dev image sha * Remove coverage exclusions * Add new options to better emulate GHA * Reconfigure kafka networking Co-authored-by: Hannah Stepanek * Fix kafka ports on GHA * Run kafka tests serially * Separate kafka consumer groups * Put CI container makefile back * Remove confluent kafka Py27 for latest * Roll back ubuntu version update * Update dev ci sha --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Hannah Stepanek * Change image tag to latest (#871) * Change image tag to latest * Use built sha * Fixup * Replace w/ latest * Add full version for pypy3.8 to tox (#872) * Add full version for pypy3.8 * Remove solrpy from tests * Fix merge conflict * Fix tests for scikit-learn >= 1.3.0 In 1.3.0 sklearn renamed fit to _fit in BaseDecisionTree. * Add gfortran to container * Use ci image sha * Add pkg-config * New CI build --------- Co-authored-by: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Co-authored-by: Ahmed Helil Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: lrafeei Co-authored-by: Tim Pansino Co-authored-by: Uma Annamalai Co-authored-by: hmstepanek Co-authored-by: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Co-authored-by: TimPansino Co-authored-by: umaannamalai --- .../actions/setup-python-matrix/action.yml | 50 --- .github/containers/Dockerfile | 22 +- .github/containers/Makefile | 6 +- .github/containers/install-python.sh | 7 +- .github/workflows/tests.yml | 178 ++++---- codecov.yml | 5 +- newrelic/api/background_task.py | 38 +- newrelic/api/message_transaction.py | 26 +- newrelic/api/profile_trace.py | 50 +-- newrelic/api/transaction.py | 65 ++- newrelic/config.py | 19 +- newrelic/core/environment.py | 10 +- tests/agent_features/test_apdex_metrics.py | 31 +- tests/agent_features/test_configuration.py | 395 +++++++++++++++++- tests/agent_features/test_custom_metrics.py | 62 +++ .../test_distributed_tracing.py | 74 +++- tests/agent_features/test_profile_trace.py | 88 ++++ tests/agent_features/test_serverless_mode.py | 144 +++---- tests/cross_agent/test_w3c_trace_context.py | 253 +++++------ tests/framework_starlette/test_bg_tasks.py | 17 +- .../messagebroker_confluentkafka/conftest.py | 13 +- tests/messagebroker_kafkapython/conftest.py | 15 +- tests/mlmodel_sklearn/test_ml_model.py | 20 +- tests/template_jinja2/conftest.py | 30 ++ tests/template_jinja2/test_jinja2.py | 41 ++ tests/testing_support/db_settings.py | 5 +- tox.ini | 129 +++--- 27 files changed, 1245 insertions(+), 548 deletions(-) delete mode 100644 .github/actions/setup-python-matrix/action.yml create mode 100644 tests/agent_features/test_custom_metrics.py create mode 100644 tests/agent_features/test_profile_trace.py create mode 100644 tests/template_jinja2/conftest.py create mode 100644 tests/template_jinja2/test_jinja2.py diff --git a/.github/actions/setup-python-matrix/action.yml b/.github/actions/setup-python-matrix/action.yml deleted file mode 100644 index a11e2197c..000000000 --- a/.github/actions/setup-python-matrix/action.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: "setup-python-matrix" -description: "Sets up all versions of python required for matrix testing in this repo." -runs: - using: "composite" - steps: - - uses: actions/setup-python@v4 - with: - python-version: "pypy-3.7" - architecture: x64 - - # - uses: actions/setup-python@v4 - # with: - # python-version: "pypy-2.7" - # architecture: x64 - - - uses: actions/setup-python@v4 - with: - python-version: "3.7" - architecture: x64 - - - uses: actions/setup-python@v4 - with: - python-version: "3.8" - architecture: x64 - - - uses: actions/setup-python@v4 - with: - python-version: "3.9" - architecture: x64 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - architecture: x64 - - - uses: actions/setup-python@v4 - with: - python-version: "3.11" - architecture: x64 - - # - uses: actions/setup-python@v4 - # with: - # python-version: "2.7" - # architecture: x64 - - - name: Install Dependencies - shell: bash - run: | - python3.10 -m pip install -U pip - python3.10 -m pip install -U wheel setuptools tox 'virtualenv<20.22.0' diff --git a/.github/containers/Dockerfile b/.github/containers/Dockerfile index 260c01d89..483492f35 100644 --- a/.github/containers/Dockerfile +++ b/.github/containers/Dockerfile @@ -26,6 +26,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ freetds-common \ freetds-dev \ gcc \ + gfortran \ git \ libbz2-dev \ libcurl4-openssl-dev \ @@ -43,6 +44,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ make \ odbc-postgresql \ openssl \ + pkg-config \ python2-dev \ python3-dev \ python3-pip \ @@ -55,10 +57,22 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ zlib1g-dev && \ rm -rf /var/lib/apt/lists/* +# Build librdkafka from source +ARG LIBRDKAFKA_VERSION=2.1.1 +RUN cd /tmp && \ + wget https://github.com/confluentinc/librdkafka/archive/refs/tags/v${LIBRDKAFKA_VERSION}.zip -O ./librdkafka.zip && \ + unzip ./librdkafka.zip && \ + rm ./librdkafka.zip && \ + cd ./librdkafka-${LIBRDKAFKA_VERSION} && \ + ./configure && \ + make all install && \ + cd /tmp && \ + rm -rf ./librdkafka-${LIBRDKAFKA_VERSION} + # Setup ODBC config -RUN sed -i 's/Driver=psqlodbca.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psqlodbca.so/g' /etc/odbcinst.ini && \ - sed -i 's/Driver=psqlodbcw.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psqlodbcw.so/g' /etc/odbcinst.ini && \ - sed -i 's/Setup=libodbcpsqlS.so/Setup=\/usr\/lib\/x86_64-linux-gnu\/odbc\/libodbcpsqlS.so/g' /etc/odbcinst.ini +RUN sed -i 's|Driver=psqlodbca.so|Driver=/usr/lib/x86_64-linux-gnu/odbc/psqlodbca.so|g' /etc/odbcinst.ini && \ + sed -i 's|Driver=psqlodbcw.so|Driver=/usr/lib/x86_64-linux-gnu/odbc/psqlodbcw.so|g' /etc/odbcinst.ini && \ + sed -i 's|Setup=libodbcpsqlS.so|Setup=/usr/lib/x86_64-linux-gnu/odbc/libodbcpsqlS.so|g' /etc/odbcinst.ini # Set the locale RUN locale-gen --no-purge en_US.UTF-8 @@ -79,7 +93,7 @@ RUN echo 'eval "$(pyenv init -)"' >>$HOME/.bashrc && \ pyenv update # Install Python -ARG PYTHON_VERSIONS="3.10 3.9 3.8 3.7 3.11 2.7 pypy2.7-7.3.11 pypy3.7" +ARG PYTHON_VERSIONS="3.10 3.9 3.8 3.7 3.11 2.7 pypy2.7-7.3.12 pypy3.8-7.3.11" COPY --chown=1000:1000 --chmod=+x ./install-python.sh /tmp/install-python.sh COPY ./requirements.txt /requirements.txt RUN /tmp/install-python.sh && \ diff --git a/.github/containers/Makefile b/.github/containers/Makefile index 8a72f4c45..35081f738 100644 --- a/.github/containers/Makefile +++ b/.github/containers/Makefile @@ -22,7 +22,9 @@ default: test .PHONY: build build: @# Perform a shortened build for testing - @docker build --build-arg='PYTHON_VERSIONS=3.10 2.7' $(MAKEFILE_DIR) -t ghcr.io/newrelic/newrelic-python-agent-ci:local + @docker build $(MAKEFILE_DIR) \ + -t ghcr.io/newrelic/newrelic-python-agent-ci:local \ + --build-arg='PYTHON_VERSIONS=3.10 2.7' .PHONY: test test: build @@ -38,7 +40,9 @@ run: build @docker run --rm -it \ --mount type=bind,source="$(REPO_ROOT)",target=/home/github/python-agent \ --workdir=/home/github/python-agent \ + --add-host=host.docker.internal:host-gateway \ -e NEW_RELIC_HOST="${NEW_RELIC_HOST}" \ -e NEW_RELIC_LICENSE_KEY="${NEW_RELIC_LICENSE_KEY}" \ -e NEW_RELIC_DEVELOPER_MODE="${NEW_RELIC_DEVELOPER_MODE}" \ + -e GITHUB_ACTIONS="true" \ ghcr.io/newrelic/newrelic-python-agent-ci:local /bin/bash diff --git a/.github/containers/install-python.sh b/.github/containers/install-python.sh index 92184df3a..2031e2d92 100755 --- a/.github/containers/install-python.sh +++ b/.github/containers/install-python.sh @@ -13,10 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -e - -SCRIPT_DIR=$(dirname "$0") -PIP_REQUIREMENTS=$(cat /requirements.txt) +set -eo pipefail main() { # Coerce space separated string to array @@ -50,7 +47,7 @@ main() { pyenv global ${PYENV_VERSIONS[@]} # Install dependencies for main python installation - pyenv exec pip install --upgrade $PIP_REQUIREMENTS + pyenv exec pip install --upgrade -r /requirements.txt } main diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ce2747911..a410d5ffd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -38,7 +38,7 @@ jobs: - elasticsearchserver08 - gearman - grpc - #- kafka + - kafka - memcached - mongodb - mssql @@ -119,7 +119,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-7f0fb125e22d5350dc5c775316e513af17f0e693@sha256:e4d5d68559e7637e090305d07498f6fa37f75eac2b61c2558fac7e54e4af8a7c options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -164,7 +164,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-7f0fb125e22d5350dc5c775316e513af17f0e693@sha256:e4d5d68559e7637e090305d07498f6fa37f75eac2b61c2558fac7e54e4af8a7c options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -209,7 +209,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-7f0fb125e22d5350dc5c775316e513af17f0e693@sha256:e4d5d68559e7637e090305d07498f6fa37f75eac2b61c2558fac7e54e4af8a7c options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -269,7 +269,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-7f0fb125e22d5350dc5c775316e513af17f0e693@sha256:e4d5d68559e7637e090305d07498f6fa37f75eac2b61c2558fac7e54e4af8a7c options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -332,7 +332,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-7f0fb125e22d5350dc5c775316e513af17f0e693@sha256:e4d5d68559e7637e090305d07498f6fa37f75eac2b61c2558fac7e54e4af8a7c options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -395,7 +395,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-7f0fb125e22d5350dc5c775316e513af17f0e693@sha256:e4d5d68559e7637e090305d07498f6fa37f75eac2b61c2558fac7e54e4af8a7c options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -453,7 +453,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-7f0fb125e22d5350dc5c775316e513af17f0e693@sha256:e4d5d68559e7637e090305d07498f6fa37f75eac2b61c2558fac7e54e4af8a7c options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -513,7 +513,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-7f0fb125e22d5350dc5c775316e513af17f0e693@sha256:e4d5d68559e7637e090305d07498f6fa37f75eac2b61c2558fac7e54e4af8a7c options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -571,7 +571,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-7f0fb125e22d5350dc5c775316e513af17f0e693@sha256:e4d5d68559e7637e090305d07498f6fa37f75eac2b61c2558fac7e54e4af8a7c options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -619,85 +619,75 @@ jobs: path: ./**/.coverage.* retention-days: 1 - # kafka: - # env: - # TOTAL_GROUPS: 4 - - # strategy: - # fail-fast: false - # matrix: - # group-number: [1, 2, 3, 4] - - # runs-on: ubuntu-20.04 - # container: - # image: ghcr.io/newrelic/newrelic-python-agent-ci:latest - # options: >- - # --add-host=host.docker.internal:host-gateway - # timeout-minutes: 30 - - # services: - # zookeeper: - # image: bitnami/zookeeper:3.7 - # env: - # ALLOW_ANONYMOUS_LOGIN: yes - - # ports: - # - 2181:2181 - - # kafka: - # image: bitnami/kafka:3.2 - # ports: - # - 8080:8080 - # - 8081:8081 - # env: - # ALLOW_PLAINTEXT_LISTENER: yes - # KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - # KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: true - # KAFKA_CFG_LISTENERS: L1://:8080,L2://:8081 - # KAFKA_CFG_ADVERTISED_LISTENERS: L1://127.0.0.1:8080,L2://kafka:8081, - # KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: L1:PLAINTEXT,L2:PLAINTEXT - # KAFKA_CFG_INTER_BROKER_LISTENER_NAME: L2 - - # steps: - # - uses: actions/checkout@v3 - - # - name: Fetch git tags - # run: | - # git config --global --add safe.directory "$GITHUB_WORKSPACE" - # git fetch --tags origin - - # # Special case packages - # - name: Install librdkafka-dev - # run: | - # # Use lsb-release to find the codename of Ubuntu to use to install the correct library name - # sudo apt-get update - # sudo ln -fs /usr/share/zoneinfo/America/Los_Angeles /etc/localtime - # sudo apt-get install -y wget gnupg2 software-properties-common - # sudo wget -qO - https://packages.confluent.io/deb/7.2/archive.key | sudo apt-key add - - # sudo add-apt-repository "deb https://packages.confluent.io/clients/deb $(lsb_release -cs) main" - # sudo apt-get update - # sudo apt-get install -y librdkafka-dev/$(lsb_release -c | cut -f 2) - - # - name: Get Environments - # id: get-envs - # run: | - # echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT - # env: - # GROUP_NUMBER: ${{ matrix.group-number }} - - # - name: Test - # run: | - # tox -vv -e ${{ steps.get-envs.outputs.envs }} - # env: - # TOX_PARALLEL_NO_SPINNER: 1 - # PY_COLORS: 0 - - # - name: Upload Coverage Artifacts - # uses: actions/upload-artifact@v3 - # with: - # name: coverage-${{ github.job }}-${{ strategy.job-index }} - # path: ./**/.coverage.* - # retention-days: 1 + kafka: + env: + TOTAL_GROUPS: 4 + + strategy: + fail-fast: false + matrix: + group-number: [1, 2, 3, 4] + + runs-on: ubuntu-20.04 + container: + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-7f0fb125e22d5350dc5c775316e513af17f0e693@sha256:e4d5d68559e7637e090305d07498f6fa37f75eac2b61c2558fac7e54e4af8a7c + options: >- + --add-host=host.docker.internal:host-gateway + timeout-minutes: 30 + + services: + zookeeper: + image: bitnami/zookeeper:3.7 + env: + ALLOW_ANONYMOUS_LOGIN: yes + + ports: + - 2181:2181 + + kafka: + image: bitnami/kafka:3.2 + ports: + - 8080:8080 + - 8082:8082 + - 8083:8083 + env: + KAFKA_ENABLE_KRAFT: no + ALLOW_PLAINTEXT_LISTENER: yes + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: true + KAFKA_CFG_LISTENERS: L1://:8082,L2://:8083,L3://:8080 + KAFKA_CFG_ADVERTISED_LISTENERS: L1://host.docker.internal:8082,L2://host.docker.internal:8083,L3://kafka:8080 + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: L1:PLAINTEXT,L2:PLAINTEXT,L3:PLAINTEXT + KAFKA_CFG_INTER_BROKER_LISTENER_NAME: L3 + + steps: + - uses: actions/checkout@v3 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin + + - name: Get Environments + id: get-envs + run: | + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT + env: + GROUP_NUMBER: ${{ matrix.group-number }} + + - name: Test + run: | + tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto + env: + TOX_PARALLEL_NO_SPINNER: 1 + PY_COLORS: 0 + + - name: Upload Coverage Artifacts + uses: actions/upload-artifact@v3 + with: + name: coverage-${{ github.job }}-${{ strategy.job-index }} + path: ./**/.coverage.* + retention-days: 1 mongodb: env: @@ -710,7 +700,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-7f0fb125e22d5350dc5c775316e513af17f0e693@sha256:e4d5d68559e7637e090305d07498f6fa37f75eac2b61c2558fac7e54e4af8a7c options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -768,7 +758,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-7f0fb125e22d5350dc5c775316e513af17f0e693@sha256:e4d5d68559e7637e090305d07498f6fa37f75eac2b61c2558fac7e54e4af8a7c options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -828,7 +818,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-7f0fb125e22d5350dc5c775316e513af17f0e693@sha256:e4d5d68559e7637e090305d07498f6fa37f75eac2b61c2558fac7e54e4af8a7c options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -889,7 +879,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-7f0fb125e22d5350dc5c775316e513af17f0e693@sha256:e4d5d68559e7637e090305d07498f6fa37f75eac2b61c2558fac7e54e4af8a7c options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -933,4 +923,4 @@ jobs: with: name: coverage-${{ github.job }}-${{ strategy.job-index }} path: ./**/.coverage.* - retention-days: 1 \ No newline at end of file + retention-days: 1 diff --git a/codecov.yml b/codecov.yml index 61c135aba..c2441c970 100644 --- a/codecov.yml +++ b/codecov.yml @@ -19,6 +19,5 @@ ignore: - "newrelic/hooks/database_oursql.py" - "newrelic/hooks/database_psycopg2ct.py" - "newrelic/hooks/datastore_umemcache.py" - # Temporarily disable kafka - - "newrelic/hooks/messagebroker_kafkapython.py" - - "newrelic/hooks/messagebroker_confluentkafka.py" + - "newrelic/admin/*" + - "newrelic/console.py" diff --git a/newrelic/api/background_task.py b/newrelic/api/background_task.py index a4a9e8e6a..4cdcd8a0d 100644 --- a/newrelic/api/background_task.py +++ b/newrelic/api/background_task.py @@ -13,19 +13,16 @@ # limitations under the License. import functools -import sys from newrelic.api.application import Application, application_instance from newrelic.api.transaction import Transaction, current_transaction -from newrelic.common.async_proxy import async_proxy, TransactionContext +from newrelic.common.async_proxy import TransactionContext, async_proxy from newrelic.common.object_names import callable_name from newrelic.common.object_wrapper import FunctionWrapper, wrap_object class BackgroundTask(Transaction): - def __init__(self, application, name, group=None, source=None): - # Initialise the common transaction base class. super(BackgroundTask, self).__init__(application, source=source) @@ -53,7 +50,6 @@ def __init__(self, application, name, group=None, source=None): def BackgroundTaskWrapper(wrapped, application=None, name=None, group=None): - def wrapper(wrapped, instance, args, kwargs): if callable(name): if instance is not None: @@ -107,39 +103,19 @@ def create_transaction(transaction): manager = create_transaction(current_transaction(active_only=False)) + # This means that a transaction already exists, so we want to return if not manager: return wrapped(*args, **kwargs) - success = True - - try: - manager.__enter__() - try: - return wrapped(*args, **kwargs) - except: - success = False - if not manager.__exit__(*sys.exc_info()): - raise - finally: - if success and manager._ref_count == 0: - manager._is_finalized = True - manager.__exit__(None, None, None) - else: - manager._request_handler_finalize = True - manager._server_adapter_finalize = True - old_transaction = current_transaction() - if old_transaction is not None: - old_transaction.drop_transaction() + with manager: + return wrapped(*args, **kwargs) return FunctionWrapper(wrapped, wrapper) def background_task(application=None, name=None, group=None): - return functools.partial(BackgroundTaskWrapper, - application=application, name=name, group=group) + return functools.partial(BackgroundTaskWrapper, application=application, name=name, group=group) -def wrap_background_task(module, object_path, application=None, - name=None, group=None): - wrap_object(module, object_path, BackgroundTaskWrapper, - (application, name, group)) +def wrap_background_task(module, object_path, application=None, name=None, group=None): + wrap_object(module, object_path, BackgroundTaskWrapper, (application, name, group)) diff --git a/newrelic/api/message_transaction.py b/newrelic/api/message_transaction.py index 291a3897e..54a71f6ef 100644 --- a/newrelic/api/message_transaction.py +++ b/newrelic/api/message_transaction.py @@ -13,7 +13,6 @@ # limitations under the License. import functools -import sys from newrelic.api.application import Application, application_instance from newrelic.api.background_task import BackgroundTask @@ -39,7 +38,6 @@ def __init__( transport_type="AMQP", source=None, ): - name, group = self.get_transaction_name(library, destination_type, destination_name) super(MessageTransaction, self).__init__(application, name, group=group, source=source) @@ -218,30 +216,12 @@ def create_transaction(transaction): manager = create_transaction(current_transaction(active_only=False)) + # This means that transaction already exists and we want to return if not manager: return wrapped(*args, **kwargs) - success = True - - try: - manager.__enter__() - try: - return wrapped(*args, **kwargs) - except: # Catch all - success = False - if not manager.__exit__(*sys.exc_info()): - raise - finally: - if success and manager._ref_count == 0: - manager._is_finalized = True - manager.__exit__(None, None, None) - else: - manager._request_handler_finalize = True - manager._server_adapter_finalize = True - - old_transaction = current_transaction() - if old_transaction is not None: - old_transaction.drop_transaction() + with manager: + return wrapped(*args, **kwargs) return FunctionWrapper(wrapped, wrapper) diff --git a/newrelic/api/profile_trace.py b/newrelic/api/profile_trace.py index 28113b1d8..93aa191a4 100644 --- a/newrelic/api/profile_trace.py +++ b/newrelic/api/profile_trace.py @@ -13,31 +13,27 @@ # limitations under the License. import functools -import sys import os +import sys -from newrelic.packages import six - -from newrelic.api.time_trace import current_trace +from newrelic import __file__ as AGENT_PACKAGE_FILE from newrelic.api.function_trace import FunctionTrace -from newrelic.common.object_wrapper import FunctionWrapper, wrap_object +from newrelic.api.time_trace import current_trace from newrelic.common.object_names import callable_name +from newrelic.common.object_wrapper import FunctionWrapper, wrap_object +from newrelic.packages import six -from newrelic import __file__ as AGENT_PACKAGE_FILE -AGENT_PACKAGE_DIRECTORY = os.path.dirname(AGENT_PACKAGE_FILE) + '/' +AGENT_PACKAGE_DIRECTORY = os.path.dirname(AGENT_PACKAGE_FILE) + "/" class ProfileTrace(object): - def __init__(self, depth): self.function_traces = [] self.maximum_depth = depth self.current_depth = 0 - def __call__(self, frame, event, arg): - - if event not in ['call', 'c_call', 'return', 'c_return', - 'exception', 'c_exception']: + def __call__(self, frame, event, arg): # pragma: no cover + if event not in ["call", "c_call", "return", "c_return", "exception", "c_exception"]: return parent = current_trace() @@ -49,8 +45,7 @@ def __call__(self, frame, event, arg): # coroutine systems based on greenlets so don't run # if we detect may be using greenlets. - if (hasattr(sys, '_current_frames') and - parent.thread_id not in sys._current_frames()): + if hasattr(sys, "_current_frames") and parent.thread_id not in sys._current_frames(): return co = frame.f_code @@ -84,7 +79,7 @@ def _callable(): except Exception: pass - if event in ['call', 'c_call']: + if event in ["call", "c_call"]: # Skip the outermost as we catch that with the root # function traces for the profile trace. @@ -100,19 +95,17 @@ def _callable(): self.function_traces.append(None) return - if event == 'call': + if event == "call": func = _callable() if func: name = callable_name(func) else: - name = '%s:%s#%s' % (func_filename, func_name, - func_line_no) + name = "%s:%s#%s" % (func_filename, func_name, func_line_no) else: func = arg name = callable_name(arg) if not name: - name = '%s:@%s#%s' % (func_filename, func_name, - func_line_no) + name = "%s:@%s#%s" % (func_filename, func_name, func_line_no) function_trace = FunctionTrace(name=name, parent=parent) function_trace.__enter__() @@ -127,7 +120,7 @@ def _callable(): self.function_traces.append(function_trace) self.current_depth += 1 - elif event in ['return', 'c_return', 'c_exception']: + elif event in ["return", "c_return", "c_exception"]: if not self.function_traces: return @@ -143,9 +136,7 @@ def _callable(): self.current_depth -= 1 -def ProfileTraceWrapper(wrapped, name=None, group=None, label=None, - params=None, depth=3): - +def ProfileTraceWrapper(wrapped, name=None, group=None, label=None, params=None, depth=3): def wrapper(wrapped, instance, args, kwargs): parent = current_trace() @@ -192,7 +183,7 @@ def wrapper(wrapped, instance, args, kwargs): _params = params with FunctionTrace(_name, _group, _label, _params, parent=parent, source=wrapped): - if not hasattr(sys, 'getprofile'): + if not hasattr(sys, "getprofile"): return wrapped(*args, **kwargs) profiler = sys.getprofile() @@ -212,11 +203,8 @@ def wrapper(wrapped, instance, args, kwargs): def profile_trace(name=None, group=None, label=None, params=None, depth=3): - return functools.partial(ProfileTraceWrapper, name=name, - group=group, label=label, params=params, depth=depth) + return functools.partial(ProfileTraceWrapper, name=name, group=group, label=label, params=params, depth=depth) -def wrap_profile_trace(module, object_path, name=None, - group=None, label=None, params=None, depth=3): - return wrap_object(module, object_path, ProfileTraceWrapper, - (name, group, label, params, depth)) +def wrap_profile_trace(module, object_path, name=None, group=None, label=None, params=None, depth=3): + return wrap_object(module, object_path, ProfileTraceWrapper, (name, group, label, params, depth)) diff --git a/newrelic/api/transaction.py b/newrelic/api/transaction.py index 9afd49da1..988b56be6 100644 --- a/newrelic/api/transaction.py +++ b/newrelic/api/transaction.py @@ -1039,7 +1039,9 @@ def _create_distributed_trace_data(self): settings = self._settings account_id = settings.account_id - trusted_account_key = settings.trusted_account_key + trusted_account_key = settings.trusted_account_key or ( + self._settings.serverless_mode.enabled and self._settings.account_id + ) application_id = settings.primary_application_id if not (account_id and application_id and trusted_account_key and settings.distributed_tracing.enabled): @@ -1130,7 +1132,10 @@ def _can_accept_distributed_trace_headers(self): return False settings = self._settings - if not (settings.distributed_tracing.enabled and settings.trusted_account_key): + trusted_account_key = settings.trusted_account_key or ( + self._settings.serverless_mode.enabled and self._settings.account_id + ) + if not (settings.distributed_tracing.enabled and trusted_account_key): return False if self._distributed_trace_state: @@ -1176,10 +1181,13 @@ def _accept_distributed_trace_payload(self, payload, transport_type="HTTP"): settings = self._settings account_id = data.get("ac") + trusted_account_key = settings.trusted_account_key or ( + self._settings.serverless_mode.enabled and self._settings.account_id + ) # If trust key doesn't exist in the payload, use account_id received_trust_key = data.get("tk", account_id) - if settings.trusted_account_key != received_trust_key: + if trusted_account_key != received_trust_key: self._record_supportability("Supportability/DistributedTrace/AcceptPayload/Ignored/UntrustedAccount") if settings.debug.log_untrusted_distributed_trace_keys: _logger.debug( @@ -1193,11 +1201,10 @@ def _accept_distributed_trace_payload(self, payload, transport_type="HTTP"): except: return False - if "pr" in data: - try: - data["pr"] = float(data["pr"]) - except: - data["pr"] = None + try: + data["pr"] = float(data["pr"]) + except Exception: + data["pr"] = None self._accept_distributed_trace_data(data, transport_type) self._record_supportability("Supportability/DistributedTrace/AcceptPayload/Success") @@ -1289,8 +1296,10 @@ def accept_distributed_trace_headers(self, headers, transport_type="HTTP"): tracestate = ensure_str(tracestate) try: vendors = W3CTraceState.decode(tracestate) - tk = self._settings.trusted_account_key - payload = vendors.pop(tk + "@nr", "") + trusted_account_key = self._settings.trusted_account_key or ( + self._settings.serverless_mode.enabled and self._settings.account_id + ) + payload = vendors.pop(trusted_account_key + "@nr", "") self.tracing_vendors = ",".join(vendors.keys()) self.tracestate = vendors.text(limit=31) except: @@ -1299,7 +1308,7 @@ def accept_distributed_trace_headers(self, headers, transport_type="HTTP"): # Remove trusted new relic header if available and parse if payload: try: - tracestate_data = NrTraceState.decode(payload, tk) + tracestate_data = NrTraceState.decode(payload, trusted_account_key) except: tracestate_data = None if tracestate_data: @@ -1426,11 +1435,17 @@ def _generate_response_headers(self, read_length=None): return nr_headers - def get_response_metadata(self): + # This function is CAT related and has been deprecated. + # Eventually, this will be removed. Until then, coverage + # does not need to factor this function into its analysis. + def get_response_metadata(self): # pragma: no cover nr_headers = dict(self._generate_response_headers()) return convert_to_cat_metadata_value(nr_headers) - def process_request_metadata(self, cat_linking_value): + # This function is CAT related and has been deprecated. + # Eventually, this will be removed. Until then, coverage + # does not need to factor this function into its analysis. + def process_request_metadata(self, cat_linking_value): # pragma: no cover try: payload = base64_decode(cat_linking_value) except: @@ -1516,7 +1531,9 @@ def record_log_event(self, message, level=None, timestamp=None, priority=None): self._log_events.add(event, priority=priority) - def record_exception(self, exc=None, value=None, tb=None, params=None, ignore_errors=None): + # This function has been deprecated (and will be removed eventually) + # and therefore does not need to be included in coverage analysis + def record_exception(self, exc=None, value=None, tb=None, params=None, ignore_errors=None): # pragma: no cover # Deprecation Warning warnings.warn( ("The record_exception function is deprecated. Please use the new api named notice_error instead."), @@ -1706,7 +1723,9 @@ def add_custom_attributes(self, items): return result - def add_custom_parameter(self, name, value): + # This function has been deprecated (and will be removed eventually) + # and therefore does not need to be included in coverage analysis + def add_custom_parameter(self, name, value): # pragma: no cover # Deprecation warning warnings.warn( ("The add_custom_parameter API has been deprecated. " "Please use the add_custom_attribute API."), @@ -1714,7 +1733,9 @@ def add_custom_parameter(self, name, value): ) return self.add_custom_attribute(name, value) - def add_custom_parameters(self, items): + # This function has been deprecated (and will be removed eventually) + # and therefore does not need to be included in coverage analysis + def add_custom_parameters(self, items): # pragma: no cover # Deprecation warning warnings.warn( ("The add_custom_parameters API has been deprecated. " "Please use the add_custom_attributes API."), @@ -1818,19 +1839,23 @@ def add_custom_attributes(items): return False -def add_custom_parameter(key, value): +# This function has been deprecated (and will be removed eventually) +# and therefore does not need to be included in coverage analysis +def add_custom_parameter(key, value): # pragma: no cover # Deprecation warning warnings.warn( - ("The add_custom_parameter API has been deprecated. " "Please use the add_custom_attribute API."), + ("The add_custom_parameter API has been deprecated. Please use the add_custom_attribute API."), DeprecationWarning, ) return add_custom_attribute(key, value) -def add_custom_parameters(items): +# This function has been deprecated (and will be removed eventually) +# and therefore does not need to be included in coverage analysis +def add_custom_parameters(items): # pragma: no cover # Deprecation warning warnings.warn( - ("The add_custom_parameters API has been deprecated. " "Please use the add_custom_attributes API."), + ("The add_custom_parameters API has been deprecated. Please use the add_custom_attributes API."), DeprecationWarning, ) return add_custom_attributes(items) diff --git a/newrelic/config.py b/newrelic/config.py index 842487306..c72b7fdf9 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -102,6 +102,14 @@ _cache_object = [] + +def _reset_config_parser(): + global _config_object + global _cache_object + _config_object = ConfigParser.RawConfigParser() + _cache_object = [] + + # Mechanism for extracting settings from the configuration for use in # instrumentation modules and extensions. @@ -558,6 +566,11 @@ def _process_configuration(section): _configuration_done = False +def _reset_configuration_done(): + global _configuration_done + _configuration_done = False + + def _process_app_name_setting(): # Do special processing to handle the case where the application # name was actually a semicolon separated list of names. In this @@ -1254,7 +1267,6 @@ def _process_wsgi_application_configuration(): for section in _config_object.sections(): if not section.startswith("wsgi-application:"): continue - enabled = False try: @@ -3870,6 +3882,11 @@ def _process_module_entry_points(): _instrumentation_done = False +def _reset_instrumentation_done(): + global _instrumentation_done + _instrumentation_done = False + + def _setup_instrumentation(): global _instrumentation_done diff --git a/newrelic/core/environment.py b/newrelic/core/environment.py index 66efe6112..9bca085a3 100644 --- a/newrelic/core/environment.py +++ b/newrelic/core/environment.py @@ -216,7 +216,15 @@ def environment_settings(): # If the module isn't actually loaded (such as failed relative imports # in Python 2.7), the module will be None and should not be reported. - if not module: + try: + if not module: + continue + except Exception: + # if the application uses generalimport to manage optional depedencies, + # it's possible that generalimport.MissingOptionalDependency is raised. + # In this case, we should not report the module as it is not actually loaded and + # is not a runtime dependency of the application. + # continue # Exclude standard library/built-in modules. diff --git a/tests/agent_features/test_apdex_metrics.py b/tests/agent_features/test_apdex_metrics.py index e32a96e31..c150fcf7e 100644 --- a/tests/agent_features/test_apdex_metrics.py +++ b/tests/agent_features/test_apdex_metrics.py @@ -13,24 +13,41 @@ # limitations under the License. import webtest - -from testing_support.validators.validate_apdex_metrics import ( - validate_apdex_metrics) from testing_support.sample_applications import simple_app +from testing_support.validators.validate_apdex_metrics import validate_apdex_metrics +from newrelic.api.transaction import current_transaction, suppress_apdex_metric +from newrelic.api.wsgi_application import wsgi_application normal_application = webtest.TestApp(simple_app) - # NOTE: This test validates that the server-side apdex_t is set to 0.5 # If the server-side configuration changes, this test will start to fail. @validate_apdex_metrics( - name='', - group='Uri', + name="", + group="Uri", apdex_t_min=0.5, apdex_t_max=0.5, ) def test_apdex(): - normal_application.get('/') + normal_application.get("/") + + +# This has to be a Web Transaction. +# The apdex measurement only applies to Web Transactions +def test_apdex_suppression(): + @wsgi_application() + def simple_apdex_supression_app(environ, start_response): + suppress_apdex_metric() + + start_response(status="200 OK", response_headers=[]) + transaction = current_transaction() + + assert transaction.suppress_apdex + assert transaction.apdex == 0 + return [] + + apdex_suppression_app = webtest.TestApp(simple_apdex_supression_app) + apdex_suppression_app.get("/") diff --git a/tests/agent_features/test_configuration.py b/tests/agent_features/test_configuration.py index 547a0eeb6..1a311e693 100644 --- a/tests/agent_features/test_configuration.py +++ b/tests/agent_features/test_configuration.py @@ -13,6 +13,7 @@ # limitations under the License. import collections +import tempfile import pytest @@ -21,8 +22,18 @@ except ImportError: import urllib.parse as urlparse +import logging + +from newrelic.api.exceptions import ConfigurationError from newrelic.common.object_names import callable_name -from newrelic.config import delete_setting, translate_deprecated_settings +from newrelic.config import ( + _reset_config_parser, + _reset_configuration_done, + _reset_instrumentation_done, + delete_setting, + initialize, + translate_deprecated_settings, +) from newrelic.core.config import ( Settings, apply_config_setting, @@ -34,6 +45,10 @@ ) +def function_to_trace(): + pass + + def parameterize_local_config(settings_list): settings_object_list = [] @@ -262,7 +277,6 @@ def parameterize_local_config(settings_list): @parameterize_local_config(_test_dictionary_local_config) def test_dict_parse(settings): - assert "NR-SESSION" in settings.request_headers_map config = settings.event_harvest_config @@ -585,3 +599,380 @@ def test_default_values(name, expected_value): settings = global_settings() value = fetch_config_setting(settings, name) assert value == expected_value + + +def test_initialize(): + initialize() + + +newrelic_ini_contents = b""" +[newrelic] +app_name = Python Agent Test (agent_features) +""" + + +def test_initialize_raises_if_config_does_not_match_previous(): + error_message = "Configuration has already been done against " "differing configuration file or environment.*" + with pytest.raises(ConfigurationError, match=error_message): + with tempfile.NamedTemporaryFile() as f: + f.write(newrelic_ini_contents) + f.seek(0) + + initialize(config_file=f.name) + + +def test_initialize_via_config_file(): + _reset_configuration_done() + with tempfile.NamedTemporaryFile() as f: + f.write(newrelic_ini_contents) + f.seek(0) + + initialize(config_file=f.name) + + +def test_initialize_no_config_file(): + _reset_configuration_done() + initialize() + + +def test_initialize_config_file_does_not_exist(): + _reset_configuration_done() + error_message = "Unable to open configuration file does-not-exist." + with pytest.raises(ConfigurationError, match=error_message): + initialize(config_file="does-not-exist") + + +def test_initialize_environment(): + _reset_configuration_done() + with tempfile.NamedTemporaryFile() as f: + f.write(newrelic_ini_contents) + f.seek(0) + + initialize(config_file=f.name, environment="developement") + + +def test_initialize_log_level(): + _reset_configuration_done() + with tempfile.NamedTemporaryFile() as f: + f.write(newrelic_ini_contents) + f.seek(0) + + initialize(config_file=f.name, log_level="debug") + + +def test_initialize_log_file(): + _reset_configuration_done() + with tempfile.NamedTemporaryFile() as f: + f.write(newrelic_ini_contents) + f.seek(0) + + initialize(config_file=f.name, log_file="stdout") + + +@pytest.mark.parametrize( + "feature_flag,expect_warning", + ( + (["django.instrumentation.inclusion-tags.r1"], False), + (["noexist"], True), + ), +) +def test_initialize_config_file_feature_flag(feature_flag, expect_warning, logger): + settings = global_settings() + apply_config_setting(settings, "feature_flag", feature_flag) + _reset_configuration_done() + + with tempfile.NamedTemporaryFile() as f: + f.write(newrelic_ini_contents) + f.seek(0) + + initialize(config_file=f.name) + + message = ( + "Unknown agent feature flag 'noexist' provided. " + "Check agent documentation or release notes, or " + "contact New Relic support for clarification of " + "validity of the specific feature flag." + ) + if expect_warning: + assert message in logger.caplog.records + else: + assert message not in logger.caplog.records + + apply_config_setting(settings, "feature_flag", []) + + +@pytest.mark.parametrize( + "feature_flag,expect_warning", + ( + (["django.instrumentation.inclusion-tags.r1"], False), + (["noexist"], True), + ), +) +def test_initialize_no_config_file_feature_flag(feature_flag, expect_warning, logger): + settings = global_settings() + apply_config_setting(settings, "feature_flag", feature_flag) + _reset_configuration_done() + + initialize() + + message = ( + "Unknown agent feature flag 'noexist' provided. " + "Check agent documentation or release notes, or " + "contact New Relic support for clarification of " + "validity of the specific feature flag." + ) + + if expect_warning: + assert message in logger.caplog.records + else: + assert message not in logger.caplog.records + + apply_config_setting(settings, "feature_flag", []) + + +@pytest.mark.parametrize( + "setting_name,setting_value,expect_error", + ( + ("transaction_tracer.function_trace", [callable_name(function_to_trace)], False), + ("transaction_tracer.generator_trace", [callable_name(function_to_trace)], False), + ("transaction_tracer.function_trace", ["no_exist"], True), + ("transaction_tracer.generator_trace", ["no_exist"], True), + ), +) +def test_initialize_config_file_with_traces(setting_name, setting_value, expect_error, logger): + settings = global_settings() + apply_config_setting(settings, setting_name, setting_value) + _reset_configuration_done() + + with tempfile.NamedTemporaryFile() as f: + f.write(newrelic_ini_contents) + f.seek(0) + + initialize(config_file=f.name) + + if expect_error: + assert "CONFIGURATION ERROR" in logger.caplog.records + else: + assert "CONFIGURATION ERROR" not in logger.caplog.records + + apply_config_setting(settings, setting_name, []) + + +func_newrelic_ini = b""" +[function-trace:] +enabled = True +function = test_configuration:function_to_trace +name = function_to_trace +group = group +label = label +terminal = False +rollup = foo/all +""" + +bad_func_newrelic_ini = b""" +[function-trace:] +enabled = True +function = function_to_trace +""" + +func_missing_enabled_newrelic_ini = b""" +[function-trace:] +function = function_to_trace +""" + +external_newrelic_ini = b""" +[external-trace:] +enabled = True +function = test_configuration:function_to_trace +library = "foo" +url = localhost:80/foo +method = GET +""" + +bad_external_newrelic_ini = b""" +[external-trace:] +enabled = True +function = function_to_trace +""" + +external_missing_enabled_newrelic_ini = b""" +[external-trace:] +function = function_to_trace +""" + +generator_newrelic_ini = b""" +[generator-trace:] +enabled = True +function = test_configuration:function_to_trace +name = function_to_trace +group = group +""" + +bad_generator_newrelic_ini = b""" +[generator-trace:] +enabled = True +function = function_to_trace +""" + +generator_missing_enabled_newrelic_ini = b""" +[generator-trace:] +function = function_to_trace +""" + +bg_task_newrelic_ini = b""" +[background-task:] +enabled = True +function = test_configuration:function_to_trace +lambda = test_configuration:function_to_trace +""" + +bad_bg_task_newrelic_ini = b""" +[background-task:] +enabled = True +function = function_to_trace +""" + +bg_task_missing_enabled_newrelic_ini = b""" +[background-task:] +function = function_to_trace +""" + +db_trace_newrelic_ini = b""" +[database-trace:] +enabled = True +function = test_configuration:function_to_trace +sql = test_configuration:function_to_trace +""" + +bad_db_trace_newrelic_ini = b""" +[database-trace:] +enabled = True +function = function_to_trace +""" + +db_trace_missing_enabled_newrelic_ini = b""" +[database-trace:] +function = function_to_trace +""" + +wsgi_newrelic_ini = b""" +[wsgi-application:] +enabled = True +function = test_configuration:function_to_trace +application = app +""" + +bad_wsgi_newrelic_ini = b""" +[wsgi-application:] +enabled = True +function = function_to_trace +application = app +""" + +wsgi_missing_enabled_newrelic_ini = b""" +[wsgi-application:] +function = function_to_trace +application = app +""" + +wsgi_unparseable_enabled_newrelic_ini = b""" +[wsgi-application:] +enabled = not-a-bool +function = function_to_trace +application = app +""" + + +@pytest.mark.parametrize( + "section,expect_error", + ( + (func_newrelic_ini, False), + (bad_func_newrelic_ini, True), + (func_missing_enabled_newrelic_ini, False), + (external_newrelic_ini, False), + (bad_external_newrelic_ini, True), + (external_missing_enabled_newrelic_ini, False), + (generator_newrelic_ini, False), + (bad_generator_newrelic_ini, True), + (generator_missing_enabled_newrelic_ini, False), + (bg_task_newrelic_ini, False), + (bad_bg_task_newrelic_ini, True), + (bg_task_missing_enabled_newrelic_ini, False), + (db_trace_newrelic_ini, False), + (bad_db_trace_newrelic_ini, True), + (db_trace_missing_enabled_newrelic_ini, False), + (wsgi_newrelic_ini, False), + (bad_wsgi_newrelic_ini, True), + (wsgi_missing_enabled_newrelic_ini, False), + (wsgi_unparseable_enabled_newrelic_ini, True), + ), + ids=( + "func_newrelic_ini", + "bad_func_newrelic_ini", + "func_missing_enabled_newrelic_ini", + "external_newrelic_ini", + "bad_external_newrelic_ini", + "external_missing_enabled_newrelic_ini", + "generator_newrelic_ini", + "bad_generator_newrelic_ini", + "generator_missing_enabled_newrelic_ini", + "bg_task_newrelic_ini", + "bad_bg_task_newrelic_ini", + "bg_task_missing_enabled_newrelic_ini", + "db_trace_newrelic_ini", + "bad_db_trace_newrelic_ini", + "db_trace_missing_enabled_newrelic_ini", + "wsgi_newrelic_ini", + "bad_wsgi_newrelic_ini", + "wsgi_missing_enabled_newrelic_ini", + "wsgi_unparseable_enabled_newrelic_ini", + ), +) +def test_initialize_developer_mode(section, expect_error, logger): + settings = global_settings() + apply_config_setting(settings, "monitor_mode", False) + apply_config_setting(settings, "developer_mode", True) + _reset_configuration_done() + _reset_instrumentation_done() + _reset_config_parser() + + with tempfile.NamedTemporaryFile() as f: + f.write(newrelic_ini_contents) + f.write(section) + f.seek(0) + + initialize(config_file=f.name) + + if expect_error: + assert "CONFIGURATION ERROR" in logger.caplog.records + else: + assert "CONFIGURATION ERROR" not in logger.caplog.records + + +@pytest.fixture +def caplog_handler(): + class CaplogHandler(logging.StreamHandler): + """ + To prevent possible issues with pytest's monkey patching + use a custom Caplog handler to capture all records + """ + + def __init__(self, *args, **kwargs): + self.records = [] + super(CaplogHandler, self).__init__(*args, **kwargs) + + def emit(self, record): + self.records.append(self.format(record)) + + return CaplogHandler() + + +@pytest.fixture +def logger(caplog_handler): + _logger = logging.getLogger("newrelic.config") + _logger.addHandler(caplog_handler) + _logger.caplog = caplog_handler + _logger.setLevel(logging.WARNING) + yield _logger + del caplog_handler.records[:] + _logger.removeHandler(caplog_handler) diff --git a/tests/agent_features/test_custom_metrics.py b/tests/agent_features/test_custom_metrics.py new file mode 100644 index 000000000..21a67149a --- /dev/null +++ b/tests/agent_features/test_custom_metrics.py @@ -0,0 +1,62 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from testing_support.fixtures import reset_core_stats_engine +from testing_support.validators.validate_custom_metrics_outside_transaction import ( + validate_custom_metrics_outside_transaction, +) + +from newrelic.api.application import application_instance as application +from newrelic.api.background_task import background_task +from newrelic.api.transaction import ( + current_transaction, + record_custom_metric, + record_custom_metrics, +) + + +# Testing record_custom_metric +@reset_core_stats_engine() +@background_task() +def test_custom_metric_inside_transaction(): + transaction = current_transaction() + record_custom_metric("CustomMetric/InsideTransaction/Count", 1) + for metric in transaction._custom_metrics.metrics(): + assert metric == ("CustomMetric/InsideTransaction/Count", [1, 1, 1, 1, 1, 1]) + + +@reset_core_stats_engine() +@validate_custom_metrics_outside_transaction([("CustomMetric/OutsideTransaction/Count", 1)]) +@background_task() +def test_custom_metric_outside_transaction_with_app(): + app = application() + record_custom_metric("CustomMetric/OutsideTransaction/Count", 1, application=app) + + +# Testing record_custom_metricS +@reset_core_stats_engine() +@background_task() +def test_custom_metrics_inside_transaction(): + transaction = current_transaction() + record_custom_metrics([("CustomMetrics/InsideTransaction/Count", 1)]) + for metric in transaction._custom_metrics.metrics(): + assert metric == ("CustomMetrics/InsideTransaction/Count", [1, 1, 1, 1, 1, 1]) + + +@reset_core_stats_engine() +@validate_custom_metrics_outside_transaction([("CustomMetrics/OutsideTransaction/Count", 1)]) +@background_task() +def test_custom_metrics_outside_transaction_with_app(): + app = application() + record_custom_metrics([("CustomMetrics/OutsideTransaction/Count", 1)], application=app) diff --git a/tests/agent_features/test_distributed_tracing.py b/tests/agent_features/test_distributed_tracing.py index 4db6d2dab..263b1bdcf 100644 --- a/tests/agent_features/test_distributed_tracing.py +++ b/tests/agent_features/test_distributed_tracing.py @@ -30,8 +30,12 @@ from newrelic.api.application import application_instance from newrelic.api.background_task import BackgroundTask, background_task +from newrelic.api.external_trace import ExternalTrace from newrelic.api.time_trace import current_trace from newrelic.api.transaction import ( + accept_distributed_trace_headers, + accept_distributed_trace_payload, + create_distributed_trace_payload, current_span_id, current_trace_id, current_transaction, @@ -185,10 +189,10 @@ def _test(): payload["d"]["pa"] = "5e5733a911cfbc73" if accept_payload: - result = txn.accept_distributed_trace_payload(payload) + result = accept_distributed_trace_payload(payload) assert result else: - txn._create_distributed_trace_payload() + create_distributed_trace_payload() try: raise ValueError("cookies") @@ -319,7 +323,6 @@ def _test(): ) @override_application_settings(_override_settings) def test_distributed_tracing_backwards_compatibility(traceparent, tracestate, newrelic, metrics): - headers = [] if traceparent: headers.append(("traceparent", TRACEPARENT)) @@ -333,8 +336,7 @@ def test_distributed_tracing_backwards_compatibility(traceparent, tracestate, ne ) @background_task(name="test_distributed_tracing_backwards_compatibility") def _test(): - transaction = current_transaction() - transaction.accept_distributed_trace_headers(headers) + accept_distributed_trace_headers(headers) _test() @@ -360,3 +362,65 @@ def test_current_span_id_inside_transaction(): def test_current_span_id_outside_transaction(): span_id = current_span_id() assert span_id is None + + +@pytest.mark.parametrize("trusted_account_key", ("1", None), ids=("tk_set", "tk_unset")) +def test_outbound_dt_payload_generation(trusted_account_key): + @override_application_settings( + { + "distributed_tracing.enabled": True, + "account_id": "1", + "trusted_account_key": trusted_account_key, + "primary_application_id": "1", + } + ) + @background_task(name="test_outbound_dt_payload_generation") + def _test_outbound_dt_payload_generation(): + transaction = current_transaction() + payload = ExternalTrace.generate_request_headers(transaction) + if trusted_account_key: + assert payload + # Ensure trusted account key present as vendor + assert dict(payload)["tracestate"].startswith("1@nr=") + else: + assert not payload + + _test_outbound_dt_payload_generation() + + +@pytest.mark.parametrize("trusted_account_key", ("1", None), ids=("tk_set", "tk_unset")) +def test_inbound_dt_payload_acceptance(trusted_account_key): + @override_application_settings( + { + "distributed_tracing.enabled": True, + "account_id": "1", + "trusted_account_key": trusted_account_key, + "primary_application_id": "1", + } + ) + @background_task(name="_test_inbound_dt_payload_acceptance") + def _test_inbound_dt_payload_acceptance(): + transaction = current_transaction() + + payload = { + "v": [0, 1], + "d": { + "ty": "Mobile", + "ac": "1", + "tk": "1", + "ap": "2827902", + "pa": "5e5733a911cfbc73", + "id": "7d3efb1b173fecfa", + "tr": "d6b4ba0c3a712ca", + "ti": 1518469636035, + "tx": "8703ff3d88eefe9d", + }, + } + + result = transaction.accept_distributed_trace_payload(payload) + if trusted_account_key: + assert result + else: + assert not result + + _test_inbound_dt_payload_acceptance() diff --git a/tests/agent_features/test_profile_trace.py b/tests/agent_features/test_profile_trace.py new file mode 100644 index 000000000..f696b7480 --- /dev/null +++ b/tests/agent_features/test_profile_trace.py @@ -0,0 +1,88 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.api.profile_trace import ProfileTraceWrapper, profile_trace + + +def test_profile_trace_wrapper(): + def _test(): + def nested_fn(): + pass + + nested_fn() + + wrapped_test = ProfileTraceWrapper(_test) + wrapped_test() + + +@validate_transaction_metrics("test_profile_trace:test_profile_trace_empty_args", background_task=True) +@background_task() +def test_profile_trace_empty_args(): + @profile_trace() + def _test(): + pass + + _test() + + +_test_profile_trace_defined_args_scoped_metrics = [("Custom/TestTrace", 1)] + + +@validate_transaction_metrics( + "test_profile_trace:test_profile_trace_defined_args", + scoped_metrics=_test_profile_trace_defined_args_scoped_metrics, + background_task=True, +) +@background_task() +def test_profile_trace_defined_args(): + @profile_trace(name="TestTrace", group="Custom", label="Label", params={"key": "value"}, depth=7) + def _test(): + pass + + _test() + + +_test_profile_trace_callable_args_scoped_metrics = [("Function/TestProfileTrace", 1)] + + +@validate_transaction_metrics( + "test_profile_trace:test_profile_trace_callable_args", + scoped_metrics=_test_profile_trace_callable_args_scoped_metrics, + background_task=True, +) +@background_task() +def test_profile_trace_callable_args(): + def name_callable(): + return "TestProfileTrace" + + def group_callable(): + return "Function" + + def label_callable(): + return "HSM" + + def params_callable(): + return {"account_id": "12345"} + + @profile_trace(name=name_callable, group=group_callable, label=label_callable, params=params_callable, depth=0) + def _test(): + pass + + _test() diff --git a/tests/agent_features/test_serverless_mode.py b/tests/agent_features/test_serverless_mode.py index 75b5f0075..189481f70 100644 --- a/tests/agent_features/test_serverless_mode.py +++ b/tests/agent_features/test_serverless_mode.py @@ -13,7 +13,16 @@ # limitations under the License. import json + import pytest +from testing_support.fixtures import override_generic_settings +from testing_support.validators.validate_serverless_data import validate_serverless_data +from testing_support.validators.validate_serverless_metadata import ( + validate_serverless_metadata, +) +from testing_support.validators.validate_serverless_payload import ( + validate_serverless_payload, +) from newrelic.api.application import application_instance from newrelic.api.background_task import background_task @@ -22,23 +31,14 @@ from newrelic.api.transaction import current_transaction from newrelic.core.config import global_settings -from testing_support.fixtures import override_generic_settings -from testing_support.validators.validate_serverless_data import ( - validate_serverless_data) -from testing_support.validators.validate_serverless_payload import ( - validate_serverless_payload) -from testing_support.validators.validate_serverless_metadata import ( - validate_serverless_metadata) - -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def serverless_application(request): settings = global_settings() orig = settings.serverless_mode.enabled settings.serverless_mode.enabled = True - application_name = 'Python Agent Test (test_serverless_mode:%s)' % ( - request.node.name) + application_name = "Python Agent Test (test_serverless_mode:%s)" % (request.node.name) application = application_instance(application_name) application.activate() @@ -48,17 +48,18 @@ def serverless_application(request): def test_serverless_payload(capsys, serverless_application): - - @override_generic_settings(serverless_application.settings, { - 'distributed_tracing.enabled': True, - }) + @override_generic_settings( + serverless_application.settings, + { + "distributed_tracing.enabled": True, + }, + ) @validate_serverless_data( - expected_methods=('metric_data', 'analytic_event_data'), - forgone_methods=('preconnect', 'connect', 'get_agent_commands')) + expected_methods=("metric_data", "analytic_event_data"), + forgone_methods=("preconnect", "connect", "get_agent_commands"), + ) @validate_serverless_payload() - @background_task( - application=serverless_application, - name='test_serverless_payload') + @background_task(application=serverless_application, name="test_serverless_payload") def _test(): transaction = current_transaction() assert transaction.settings.serverless_mode.enabled @@ -75,17 +76,15 @@ def _test(): def test_no_cat_headers(serverless_application): - @background_task( - application=serverless_application, - name='test_cat_headers') + @background_task(application=serverless_application, name="test_cat_headers") def _test_cat_headers(): transaction = current_transaction() payload = ExternalTrace.generate_request_headers(transaction) assert not payload - trace = ExternalTrace('testlib', 'http://example.com') - response_headers = [('X-NewRelic-App-Data', 'Cookies')] + trace = ExternalTrace("testlib", "http://example.com") + response_headers = [("X-NewRelic-App-Data", "Cookies")] with trace: trace.process_response_headers(response_headers) @@ -94,61 +93,66 @@ def _test_cat_headers(): _test_cat_headers() -def test_dt_outbound(serverless_application): - @override_generic_settings(serverless_application.settings, { - 'distributed_tracing.enabled': True, - 'account_id': '1', - 'trusted_account_key': '1', - 'primary_application_id': '1', - }) - @background_task( - application=serverless_application, - name='test_dt_outbound') - def _test_dt_outbound(): +@pytest.mark.parametrize("trusted_account_key", ("1", None), ids=("tk_set", "tk_unset")) +def test_outbound_dt_payload_generation(serverless_application, trusted_account_key): + @override_generic_settings( + serverless_application.settings, + { + "distributed_tracing.enabled": True, + "account_id": "1", + "trusted_account_key": trusted_account_key, + "primary_application_id": "1", + }, + ) + @background_task(application=serverless_application, name="test_outbound_dt_payload_generation") + def _test_outbound_dt_payload_generation(): transaction = current_transaction() payload = ExternalTrace.generate_request_headers(transaction) assert payload - - _test_dt_outbound() - - -def test_dt_inbound(serverless_application): - @override_generic_settings(serverless_application.settings, { - 'distributed_tracing.enabled': True, - 'account_id': '1', - 'trusted_account_key': '1', - 'primary_application_id': '1', - }) - @background_task( - application=serverless_application, - name='test_dt_inbound') - def _test_dt_inbound(): + # Ensure trusted account key or account ID present as vendor + assert dict(payload)["tracestate"].startswith("1@nr=") + + _test_outbound_dt_payload_generation() + + +@pytest.mark.parametrize("trusted_account_key", ("1", None), ids=("tk_set", "tk_unset")) +def test_inbound_dt_payload_acceptance(serverless_application, trusted_account_key): + @override_generic_settings( + serverless_application.settings, + { + "distributed_tracing.enabled": True, + "account_id": "1", + "trusted_account_key": trusted_account_key, + "primary_application_id": "1", + }, + ) + @background_task(application=serverless_application, name="test_inbound_dt_payload_acceptance") + def _test_inbound_dt_payload_acceptance(): transaction = current_transaction() payload = { - 'v': [0, 1], - 'd': { - 'ty': 'Mobile', - 'ac': '1', - 'tk': '1', - 'ap': '2827902', - 'pa': '5e5733a911cfbc73', - 'id': '7d3efb1b173fecfa', - 'tr': 'd6b4ba0c3a712ca', - 'ti': 1518469636035, - 'tx': '8703ff3d88eefe9d', - } + "v": [0, 1], + "d": { + "ty": "Mobile", + "ac": "1", + "tk": "1", + "ap": "2827902", + "pa": "5e5733a911cfbc73", + "id": "7d3efb1b173fecfa", + "tr": "d6b4ba0c3a712ca", + "ti": 1518469636035, + "tx": "8703ff3d88eefe9d", + }, } result = transaction.accept_distributed_trace_payload(payload) assert result - _test_dt_inbound() + _test_inbound_dt_payload_acceptance() -@pytest.mark.parametrize('arn_set', (True, False)) +@pytest.mark.parametrize("arn_set", (True, False)) def test_payload_metadata_arn(serverless_application, arn_set): - # If the session object gathers the arn from the settings object before the # lambda handler records it there, then this test will fail. @@ -157,17 +161,17 @@ def test_payload_metadata_arn(serverless_application, arn_set): arn = None if arn_set: - arn = 'arrrrrrrrrrRrrrrrrrn' + arn = "arrrrrrrrrrRrrrrrrrn" - settings.aws_lambda_metadata.update({'arn': arn, 'function_version': '$LATEST'}) + settings.aws_lambda_metadata.update({"arn": arn, "function_version": "$LATEST"}) class Context(object): invoked_function_arn = arn - @validate_serverless_metadata(exact_metadata={'arn': arn}) + @validate_serverless_metadata(exact_metadata={"arn": arn}) @lambda_handler(application=serverless_application) def handler(event, context): - assert settings.aws_lambda_metadata['arn'] == arn + assert settings.aws_lambda_metadata["arn"] == arn return {} try: diff --git a/tests/cross_agent/test_w3c_trace_context.py b/tests/cross_agent/test_w3c_trace_context.py index 05f157f7b..893274ce4 100644 --- a/tests/cross_agent/test_w3c_trace_context.py +++ b/tests/cross_agent/test_w3c_trace_context.py @@ -14,88 +14,105 @@ import json import os + import pytest import webtest -from newrelic.packages import six - -from newrelic.api.transaction import current_transaction +from testing_support.fixtures import override_application_settings, validate_attributes +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_event_attributes import ( + validate_transaction_event_attributes, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.transaction import ( + accept_distributed_trace_headers, + current_transaction, + insert_distributed_trace_headers, +) from newrelic.api.wsgi_application import wsgi_application -from newrelic.common.object_wrapper import transient_function_wrapper -from testing_support.validators.validate_span_events import ( - validate_span_events) -from testing_support.fixtures import (override_application_settings, - validate_attributes) from newrelic.common.encoding_utils import W3CTraceState -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics -from testing_support.validators.validate_transaction_event_attributes import validate_transaction_event_attributes +from newrelic.common.object_wrapper import transient_function_wrapper +from newrelic.packages import six CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) -JSON_DIR = os.path.normpath(os.path.join(CURRENT_DIR, 'fixtures', - 'distributed_tracing')) - -_parameters_list = ('test_name', 'trusted_account_key', 'account_id', - 'web_transaction', 'raises_exception', 'force_sampled_true', - 'span_events_enabled', 'transport_type', 'inbound_headers', - 'outbound_payloads', 'intrinsics', 'expected_metrics') - -_parameters = ','.join(_parameters_list) +JSON_DIR = os.path.normpath(os.path.join(CURRENT_DIR, "fixtures", "distributed_tracing")) + +_parameters_list = ( + "test_name", + "trusted_account_key", + "account_id", + "web_transaction", + "raises_exception", + "force_sampled_true", + "span_events_enabled", + "transport_type", + "inbound_headers", + "outbound_payloads", + "intrinsics", + "expected_metrics", +) + +_parameters = ",".join(_parameters_list) XFAIL_TESTS = [ - 'spans_disabled_root', - 'missing_traceparent', - 'missing_traceparent_and_tracestate', - 'w3c_and_newrelc_headers_present_error_parsing_traceparent' + "spans_disabled_root", + "missing_traceparent", + "missing_traceparent_and_tracestate", + "w3c_and_newrelc_headers_present_error_parsing_traceparent", ] + def load_tests(): result = [] - path = os.path.join(JSON_DIR, 'trace_context.json') - with open(path, 'r') as fh: + path = os.path.join(JSON_DIR, "trace_context.json") + with open(path, "r") as fh: tests = json.load(fh) for test in tests: values = (test.get(param, None) for param in _parameters_list) - param = pytest.param(*values, id=test.get('test_name')) + param = pytest.param(*values, id=test.get("test_name")) result.append(param) return result ATTR_MAP = { - 'traceparent.version': 0, - 'traceparent.trace_id': 1, - 'traceparent.parent_id': 2, - 'traceparent.trace_flags': 3, - 'tracestate.version': 0, - 'tracestate.parent_type': 1, - 'tracestate.parent_account_id': 2, - 'tracestate.parent_application_id': 3, - 'tracestate.span_id': 4, - 'tracestate.transaction_id': 5, - 'tracestate.sampled': 6, - 'tracestate.priority': 7, - 'tracestate.timestamp': 8, - 'tracestate.tenant_id': None, + "traceparent.version": 0, + "traceparent.trace_id": 1, + "traceparent.parent_id": 2, + "traceparent.trace_flags": 3, + "tracestate.version": 0, + "tracestate.parent_type": 1, + "tracestate.parent_account_id": 2, + "tracestate.parent_application_id": 3, + "tracestate.span_id": 4, + "tracestate.transaction_id": 5, + "tracestate.sampled": 6, + "tracestate.priority": 7, + "tracestate.timestamp": 8, + "tracestate.tenant_id": None, } def validate_outbound_payload(actual, expected, trusted_account_key): - traceparent = '' - tracestate = '' + traceparent = "" + tracestate = "" for key, value in actual: - if key == 'traceparent': - traceparent = value.split('-') - elif key == 'tracestate': + if key == "traceparent": + traceparent = value.split("-") + elif key == "tracestate": vendors = W3CTraceState.decode(value) - nr_entry = vendors.pop(trusted_account_key + '@nr', '') - tracestate = nr_entry.split('-') - exact_values = expected.get('exact', {}) - expected_attrs = expected.get('expected', []) - unexpected_attrs = expected.get('unexpected', []) - expected_vendors = expected.get('vendors', []) + nr_entry = vendors.pop(trusted_account_key + "@nr", "") + tracestate = nr_entry.split("-") + exact_values = expected.get("exact", {}) + expected_attrs = expected.get("expected", []) + unexpected_attrs = expected.get("unexpected", []) + expected_vendors = expected.get("vendors", []) for key, value in exact_values.items(): - header = traceparent if key.startswith('traceparent.') else tracestate + header = traceparent if key.startswith("traceparent.") else tracestate attr = ATTR_MAP[key] if attr is not None: if isinstance(value, bool): @@ -106,13 +123,13 @@ def validate_outbound_payload(actual, expected, trusted_account_key): assert header[attr] == str(value) for key in expected_attrs: - header = traceparent if key.startswith('traceparent.') else tracestate + header = traceparent if key.startswith("traceparent.") else tracestate attr = ATTR_MAP[key] if attr is not None: assert header[attr], key for key in unexpected_attrs: - header = traceparent if key.startswith('traceparent.') else tracestate + header = traceparent if key.startswith("traceparent.") else tracestate attr = ATTR_MAP[key] if attr is not None: assert not header[attr], key @@ -125,127 +142,129 @@ def validate_outbound_payload(actual, expected, trusted_account_key): def target_wsgi_application(environ, start_response): transaction = current_transaction() - if not environ['.web_transaction']: + if not environ[".web_transaction"]: transaction.background_task = True - if environ['.raises_exception']: + if environ[".raises_exception"]: try: raise ValueError("oops") except: transaction.notice_error() - if '.inbound_headers' in environ: - transaction.accept_distributed_trace_headers( - environ['.inbound_headers'], - transport_type=environ['.transport_type'], + if ".inbound_headers" in environ: + accept_distributed_trace_headers( + environ[".inbound_headers"], + transport_type=environ[".transport_type"], ) payloads = [] - for _ in range(environ['.outbound_calls']): + for _ in range(environ[".outbound_calls"]): payloads.append([]) - transaction.insert_distributed_trace_headers(payloads[-1]) + insert_distributed_trace_headers(payloads[-1]) - start_response('200 OK', [('Content-Type', 'application/json')]) - return [json.dumps(payloads).encode('utf-8')] + start_response("200 OK", [("Content-Type", "application/json")]) + return [json.dumps(payloads).encode("utf-8")] test_application = webtest.TestApp(target_wsgi_application) def override_compute_sampled(override): - @transient_function_wrapper('newrelic.core.adaptive_sampler', - 'AdaptiveSampler.compute_sampled') + @transient_function_wrapper("newrelic.core.adaptive_sampler", "AdaptiveSampler.compute_sampled") def _override_compute_sampled(wrapped, instance, args, kwargs): if override: return True return wrapped(*args, **kwargs) + return _override_compute_sampled @pytest.mark.parametrize(_parameters, load_tests()) -def test_trace_context(test_name, trusted_account_key, account_id, - web_transaction, raises_exception, force_sampled_true, - span_events_enabled, transport_type, inbound_headers, - outbound_payloads, intrinsics, expected_metrics): - +def test_trace_context( + test_name, + trusted_account_key, + account_id, + web_transaction, + raises_exception, + force_sampled_true, + span_events_enabled, + transport_type, + inbound_headers, + outbound_payloads, + intrinsics, + expected_metrics, +): if test_name in XFAIL_TESTS: pytest.xfail("Waiting on cross agent tests update.") # Prepare assertions if not intrinsics: intrinsics = {} - common = intrinsics.get('common', {}) - common_required = common.get('expected', []) - common_forgone = common.get('unexpected', []) - common_exact = common.get('exact', {}) - - txn_intrinsics = intrinsics.get('Transaction', {}) - txn_event_required = {'agent': [], 'user': [], - 'intrinsic': txn_intrinsics.get('expected', [])} - txn_event_required['intrinsic'].extend(common_required) - txn_event_forgone = {'agent': [], 'user': [], - 'intrinsic': txn_intrinsics.get('unexpected', [])} - txn_event_forgone['intrinsic'].extend(common_forgone) - txn_event_exact = {'agent': {}, 'user': {}, - 'intrinsic': txn_intrinsics.get('exact', {})} - txn_event_exact['intrinsic'].update(common_exact) + common = intrinsics.get("common", {}) + common_required = common.get("expected", []) + common_forgone = common.get("unexpected", []) + common_exact = common.get("exact", {}) + + txn_intrinsics = intrinsics.get("Transaction", {}) + txn_event_required = {"agent": [], "user": [], "intrinsic": txn_intrinsics.get("expected", [])} + txn_event_required["intrinsic"].extend(common_required) + txn_event_forgone = {"agent": [], "user": [], "intrinsic": txn_intrinsics.get("unexpected", [])} + txn_event_forgone["intrinsic"].extend(common_forgone) + txn_event_exact = {"agent": {}, "user": {}, "intrinsic": txn_intrinsics.get("exact", {})} + txn_event_exact["intrinsic"].update(common_exact) override_settings = { - 'distributed_tracing.enabled': True, - 'span_events.enabled': span_events_enabled, - 'account_id': account_id, - 'trusted_account_key': trusted_account_key, + "distributed_tracing.enabled": True, + "span_events.enabled": span_events_enabled, + "account_id": account_id, + "trusted_account_key": trusted_account_key, } extra_environ = { - '.web_transaction': web_transaction, - '.raises_exception': raises_exception, - '.transport_type': transport_type, - '.outbound_calls': outbound_payloads and len(outbound_payloads) or 0, + ".web_transaction": web_transaction, + ".raises_exception": raises_exception, + ".transport_type": transport_type, + ".outbound_calls": outbound_payloads and len(outbound_payloads) or 0, } inbound_headers = inbound_headers and inbound_headers[0] or None - if transport_type != 'HTTP': - extra_environ['.inbound_headers'] = inbound_headers + if transport_type != "HTTP": + extra_environ[".inbound_headers"] = inbound_headers inbound_headers = None elif six.PY2 and inbound_headers: - inbound_headers = { - k.encode('utf-8'): v.encode('utf-8') - for k, v in inbound_headers.items()} - - @validate_transaction_metrics(test_name, - group="Uri", - rollup_metrics=expected_metrics, - background_task=not web_transaction) - @validate_transaction_event_attributes( - txn_event_required, txn_event_forgone, txn_event_exact) - @validate_attributes('intrinsic', common_required, common_forgone) + inbound_headers = {k.encode("utf-8"): v.encode("utf-8") for k, v in inbound_headers.items()} + + @validate_transaction_metrics( + test_name, group="Uri", rollup_metrics=expected_metrics, background_task=not web_transaction + ) + @validate_transaction_event_attributes(txn_event_required, txn_event_forgone, txn_event_exact) + @validate_attributes("intrinsic", common_required, common_forgone) @override_application_settings(override_settings) @override_compute_sampled(force_sampled_true) def _test(): return test_application.get( - '/' + test_name, + "/" + test_name, headers=inbound_headers, extra_environ=extra_environ, ) - if 'Span' in intrinsics: - span_intrinsics = intrinsics.get('Span') - span_expected = span_intrinsics.get('expected', []) + if "Span" in intrinsics: + span_intrinsics = intrinsics.get("Span") + span_expected = span_intrinsics.get("expected", []) span_expected.extend(common_required) - span_unexpected = span_intrinsics.get('unexpected', []) + span_unexpected = span_intrinsics.get("unexpected", []) span_unexpected.extend(common_forgone) - span_exact = span_intrinsics.get('exact', {}) + span_exact = span_intrinsics.get("exact", {}) span_exact.update(common_exact) - _test = validate_span_events(exact_intrinsics=span_exact, - expected_intrinsics=span_expected, - unexpected_intrinsics=span_unexpected)(_test) + _test = validate_span_events( + exact_intrinsics=span_exact, expected_intrinsics=span_expected, unexpected_intrinsics=span_unexpected + )(_test) elif not span_events_enabled: _test = validate_span_events(count=0)(_test) response = _test() - assert response.status == '200 OK' + assert response.status == "200 OK" payloads = response.json if outbound_payloads: assert len(payloads) == len(outbound_payloads) diff --git a/tests/framework_starlette/test_bg_tasks.py b/tests/framework_starlette/test_bg_tasks.py index 07a70131b..5e30fe32e 100644 --- a/tests/framework_starlette/test_bg_tasks.py +++ b/tests/framework_starlette/test_bg_tasks.py @@ -89,11 +89,20 @@ def _test(): # The bug was fixed in version 0.21.0 but re-occured in 0.23.1. # The bug was also not present on 0.20.1 to 0.23.1 if using Python3.7. - BUG_COMPLETELY_FIXED = (0, 21, 0) <= starlette_version < (0, 23, 1) or ( - (0, 20, 1) <= starlette_version < (0, 23, 1) and sys.version_info[:2] > (3, 7) + # The bug was fixed again in version 0.29.0 + BUG_COMPLETELY_FIXED = any( + ( + (0, 21, 0) <= starlette_version < (0, 23, 1), + (0, 20, 1) <= starlette_version < (0, 23, 1) and sys.version_info[:2] > (3, 7), + starlette_version >= (0, 29, 0), + ) + ) + BUG_PARTIALLY_FIXED = any( + ( + (0, 20, 1) <= starlette_version < (0, 21, 0), + (0, 23, 1) <= starlette_version < (0, 29, 0), + ) ) - BUG_PARTIALLY_FIXED = (0, 20, 1) <= starlette_version < (0, 21, 0) or starlette_version >= (0, 23, 1) - if BUG_COMPLETELY_FIXED: # Assert both web transaction and background task transactions are present. _test = validate_transaction_metrics( diff --git a/tests/messagebroker_confluentkafka/conftest.py b/tests/messagebroker_confluentkafka/conftest.py index e29596d55..fa86b6b3c 100644 --- a/tests/messagebroker_confluentkafka/conftest.py +++ b/tests/messagebroker_confluentkafka/conftest.py @@ -84,7 +84,7 @@ def producer(topic, client_type, json_serializer): @pytest.fixture(scope="function") -def consumer(topic, producer, client_type, json_deserializer): +def consumer(group_id, topic, producer, client_type, json_deserializer): from confluent_kafka import Consumer, DeserializingConsumer if client_type == "cimpl": @@ -93,7 +93,7 @@ def consumer(topic, producer, client_type, json_deserializer): "bootstrap.servers": BROKER, "auto.offset.reset": "earliest", "heartbeat.interval.ms": 1000, - "group.id": "test", + "group.id": group_id, } ) elif client_type == "serializer_function": @@ -102,7 +102,7 @@ def consumer(topic, producer, client_type, json_deserializer): "bootstrap.servers": BROKER, "auto.offset.reset": "earliest", "heartbeat.interval.ms": 1000, - "group.id": "test", + "group.id": group_id, "value.deserializer": lambda v, c: json.loads(v.decode("utf-8")), "key.deserializer": lambda v, c: json.loads(v.decode("utf-8")) if v is not None else None, } @@ -113,7 +113,7 @@ def consumer(topic, producer, client_type, json_deserializer): "bootstrap.servers": BROKER, "auto.offset.reset": "earliest", "heartbeat.interval.ms": 1000, - "group.id": "test", + "group.id": group_id, "value.deserializer": json_deserializer, "key.deserializer": json_deserializer, } @@ -181,6 +181,11 @@ def topic(): admin.delete_topics(new_topics) +@pytest.fixture(scope="session") +def group_id(): + return str(uuid.uuid4()) + + @pytest.fixture() def send_producer_message(topic, producer, serialize, client_type): callback_called = [] diff --git a/tests/messagebroker_kafkapython/conftest.py b/tests/messagebroker_kafkapython/conftest.py index becef31a0..de12f5830 100644 --- a/tests/messagebroker_kafkapython/conftest.py +++ b/tests/messagebroker_kafkapython/conftest.py @@ -86,7 +86,7 @@ def producer(client_type, json_serializer, json_callable_serializer): @pytest.fixture(scope="function") -def consumer(topic, producer, client_type, json_deserializer, json_callable_deserializer): +def consumer(group_id, topic, producer, client_type, json_deserializer, json_callable_deserializer): if client_type == "no_serializer": consumer = kafka.KafkaConsumer( topic, @@ -94,7 +94,7 @@ def consumer(topic, producer, client_type, json_deserializer, json_callable_dese auto_offset_reset="earliest", consumer_timeout_ms=100, heartbeat_interval_ms=1000, - group_id="test", + group_id=group_id, ) elif client_type == "serializer_function": consumer = kafka.KafkaConsumer( @@ -105,7 +105,7 @@ def consumer(topic, producer, client_type, json_deserializer, json_callable_dese auto_offset_reset="earliest", consumer_timeout_ms=100, heartbeat_interval_ms=1000, - group_id="test", + group_id=group_id, ) elif client_type == "callable_object": consumer = kafka.KafkaConsumer( @@ -116,7 +116,7 @@ def consumer(topic, producer, client_type, json_deserializer, json_callable_dese auto_offset_reset="earliest", consumer_timeout_ms=100, heartbeat_interval_ms=1000, - group_id="test", + group_id=group_id, ) elif client_type == "serializer_object": consumer = kafka.KafkaConsumer( @@ -127,7 +127,7 @@ def consumer(topic, producer, client_type, json_deserializer, json_callable_dese auto_offset_reset="earliest", consumer_timeout_ms=100, heartbeat_interval_ms=1000, - group_id="test", + group_id=group_id, ) yield consumer @@ -202,6 +202,11 @@ def topic(): admin.delete_topics([topic]) +@pytest.fixture(scope="session") +def group_id(): + return str(uuid.uuid4()) + + @pytest.fixture() def send_producer_message(topic, producer, serialize): def _test(): diff --git a/tests/mlmodel_sklearn/test_ml_model.py b/tests/mlmodel_sklearn/test_ml_model.py index 65fb1eabb..8302e0f72 100644 --- a/tests/mlmodel_sklearn/test_ml_model.py +++ b/tests/mlmodel_sklearn/test_ml_model.py @@ -100,12 +100,20 @@ def __init__( ) def fit(self, X, y, sample_weight=None, check_input=True): - return super(CustomTestModel, self).fit( - X, - y, - sample_weight=sample_weight, - check_input=check_input, - ) + if hasattr(super(CustomTestModel, self), "_fit"): + return self._fit( + X, + y, + sample_weight=sample_weight, + check_input=check_input, + ) + else: + return super(CustomTestModel, self).fit( + X, + y, + sample_weight=sample_weight, + check_input=check_input, + ) def predict(self, X, check_input=True): return super(CustomTestModel, self).predict(X, check_input=check_input) diff --git a/tests/template_jinja2/conftest.py b/tests/template_jinja2/conftest.py new file mode 100644 index 000000000..a6922078d --- /dev/null +++ b/tests/template_jinja2/conftest.py @@ -0,0 +1,30 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) + +_default_settings = { + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, +} + +collector_agent_registration = collector_agent_registration_fixture( + app_name="Python Agent Test (template_jinja2)", default_settings=_default_settings +) diff --git a/tests/template_jinja2/test_jinja2.py b/tests/template_jinja2/test_jinja2.py new file mode 100644 index 000000000..c64dac923 --- /dev/null +++ b/tests/template_jinja2/test_jinja2.py @@ -0,0 +1,41 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from jinja2 import Template +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task + + +@validate_transaction_metrics( + "test_render", + background_task=True, + scoped_metrics=( + ("Template/Render/