From 310d8bb503ba6a6a3ee243d4f606d5fcd6f126ec Mon Sep 17 00:00:00 2001 From: Itay Gibel Date: Wed, 3 Nov 2021 12:50:55 +0200 Subject: [PATCH] kafka-python instrumentation --- CHANGELOG.md | 3 + instrumentation/README.md | 1 + .../LICENSE | 201 ++++++++++++++++ .../MANIFEST.in | 9 + .../README.rst | 22 ++ .../setup.cfg | 57 +++++ .../setup.py | 89 ++++++++ .../instrumentation/kafka/__init__.py | 95 ++++++++ .../instrumentation/kafka/package.py | 16 ++ .../instrumentation/kafka/utils.py | 215 ++++++++++++++++++ .../instrumentation/kafka/version.py | 15 ++ .../tests/__init__.py | 0 .../tests/test_instrumentation.py | 36 +++ .../tests/test_utils.py | 151 ++++++++++++ .../instrumentation/bootstrap_gen.py | 4 + tox.ini | 9 + 16 files changed, 923 insertions(+) create mode 100644 instrumentation/opentelemetry-instrumentation-kafka-python/LICENSE create mode 100644 instrumentation/opentelemetry-instrumentation-kafka-python/MANIFEST.in create mode 100644 instrumentation/opentelemetry-instrumentation-kafka-python/README.rst create mode 100644 instrumentation/opentelemetry-instrumentation-kafka-python/setup.cfg create mode 100644 instrumentation/opentelemetry-instrumentation-kafka-python/setup.py create mode 100644 instrumentation/opentelemetry-instrumentation-kafka-python/src/opentelemetry/instrumentation/kafka/__init__.py create mode 100644 instrumentation/opentelemetry-instrumentation-kafka-python/src/opentelemetry/instrumentation/kafka/package.py create mode 100644 instrumentation/opentelemetry-instrumentation-kafka-python/src/opentelemetry/instrumentation/kafka/utils.py create mode 100644 instrumentation/opentelemetry-instrumentation-kafka-python/src/opentelemetry/instrumentation/kafka/version.py create mode 100644 instrumentation/opentelemetry-instrumentation-kafka-python/tests/__init__.py create mode 100644 instrumentation/opentelemetry-instrumentation-kafka-python/tests/test_instrumentation.py create mode 100644 instrumentation/opentelemetry-instrumentation-kafka-python/tests/test_utils.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 46f42fd58b..566ded01ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `opentelemetry-instrumentation-tornado` Add support instrumentation for Tornado 5.1.1 ([#812](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/812)) +- `opentelemetry-instrumentation-kafka-python` added kafka-python module instrumentation. +([#814](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/814)) + ## [1.7.1-0.26b1](https://github.com/open-telemetry/opentelemetry-python/releases/tag/v1.7.0-0.26b0) - 2021-11-11 - `opentelemetry-instrumentation-aws-lambda` Add instrumentation for AWS Lambda Service - pkg metadata files (Part 1/2) diff --git a/instrumentation/README.md b/instrumentation/README.md index fb8a9638b5..efcc0eceab 100644 --- a/instrumentation/README.md +++ b/instrumentation/README.md @@ -18,6 +18,7 @@ | [opentelemetry-instrumentation-grpc](./opentelemetry-instrumentation-grpc) | grpcio ~= 1.27 | | [opentelemetry-instrumentation-httpx](./opentelemetry-instrumentation-httpx) | httpx >= 0.18.0, < 0.19.0 | | [opentelemetry-instrumentation-jinja2](./opentelemetry-instrumentation-jinja2) | jinja2 >= 2.7, < 4.0 | +| [opentelemetry-instrumentation-kafka-python](./opentelemetry-instrumentation-kafka-python) | kafka-python >= 2.0 | | [opentelemetry-instrumentation-logging](./opentelemetry-instrumentation-logging) | logging | | [opentelemetry-instrumentation-mysql](./opentelemetry-instrumentation-mysql) | mysql-connector-python ~= 8.0 | | [opentelemetry-instrumentation-pika](./opentelemetry-instrumentation-pika) | pika >= 1.1.0 | diff --git a/instrumentation/opentelemetry-instrumentation-kafka-python/LICENSE b/instrumentation/opentelemetry-instrumentation-kafka-python/LICENSE new file mode 100644 index 0000000000..1ef7dad2c5 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-kafka-python/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright The OpenTelemetry Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/instrumentation/opentelemetry-instrumentation-kafka-python/MANIFEST.in b/instrumentation/opentelemetry-instrumentation-kafka-python/MANIFEST.in new file mode 100644 index 0000000000..aed3e33273 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-kafka-python/MANIFEST.in @@ -0,0 +1,9 @@ +graft src +graft tests +global-exclude *.pyc +global-exclude *.pyo +global-exclude __pycache__/* +include CHANGELOG.md +include MANIFEST.in +include README.rst +include LICENSE diff --git a/instrumentation/opentelemetry-instrumentation-kafka-python/README.rst b/instrumentation/opentelemetry-instrumentation-kafka-python/README.rst new file mode 100644 index 0000000000..4176ba23d5 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-kafka-python/README.rst @@ -0,0 +1,22 @@ +OpenTelemetry kafka-python integration +================================= + +|pypi| + +.. |pypi| image:: https://badge.fury.io/py/opentelemetry-instrumentation-kafka-python.svg + :target: https://pypi.org/project/opentelemetry-instrumentation-kafka-python/ + +Installation +------------ + +:: + + pip install opentelemetry-instrumentation-kafka-python + + +References +---------- + +* `OpenTelemetry logging integration `_ +* `OpenTelemetry Project `_ +* `OpenTelemetry Python Examples `_ diff --git a/instrumentation/opentelemetry-instrumentation-kafka-python/setup.cfg b/instrumentation/opentelemetry-instrumentation-kafka-python/setup.cfg new file mode 100644 index 0000000000..f029b10e11 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-kafka-python/setup.cfg @@ -0,0 +1,57 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +[metadata] +name = opentelemetry-instrumentation-kafka-python +description = OpenTelemetry Kafka-Python instrumentation +long_description = file: README.rst +long_description_content_type = text/x-rst +author = OpenTelemetry Authors +author_email = cncf-opentelemetry-contributors@lists.cncf.io +url = https://github.com/open-telemetry/opentelemetry-python-contrib/tree/master/instrumentation/opentelemetry-instrumentation-kafka-python +platforms = any +license = Apache-2.0 +classifiers = + Development Status :: 4 - Beta + Intended Audience :: Developers + License :: OSI Approved :: Apache Software License + Programming Language :: Python + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + +[options] +python_requires = >=3.6 +package_dir= + =src +packages=find_namespace: +install_requires = + opentelemetry-api ~= 1.5 + opentelemetry-instrumentation == 0.26b1 + opentelemetry-semantic-conventions == 0.26b1 + +[options.extras_require] +test = + wrapt >= 1.0.0, < 2.0.0 + opentelemetry-test-utils == 0.26b1 + +[options.entry_points] +opentelemetry_instrumentor = + kafka = opentelemetry.instrumentation.kafka:KafkaInstrumentor + +[options.packages.find] +where = src diff --git a/instrumentation/opentelemetry-instrumentation-kafka-python/setup.py b/instrumentation/opentelemetry-instrumentation-kafka-python/setup.py new file mode 100644 index 0000000000..4d6bbbabb4 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-kafka-python/setup.py @@ -0,0 +1,89 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# DO NOT EDIT. THIS FILE WAS AUTOGENERATED FROM templates/instrumentation_setup.py.txt. +# RUN `python scripts/generate_setup.py` TO REGENERATE. + + +import distutils.cmd +import json +import os +from configparser import ConfigParser + +import setuptools + +config = ConfigParser() +config.read("setup.cfg") + +# We provide extras_require parameter to setuptools.setup later which +# overwrites the extra_require section from setup.cfg. To support extra_require +# secion in setup.cfg, we load it here and merge it with the extra_require param. +extras_require = {} +if "options.extras_require" in config: + for key, value in config["options.extras_require"].items(): + extras_require[key] = [v for v in value.split("\n") if v.strip()] + +BASE_DIR = os.path.dirname(__file__) +PACKAGE_INFO = {} + +VERSION_FILENAME = os.path.join( + BASE_DIR, "src", "opentelemetry", "instrumentation", "kafka", "version.py" +) +with open(VERSION_FILENAME, encoding="utf-8") as f: + exec(f.read(), PACKAGE_INFO) + +PACKAGE_FILENAME = os.path.join( + BASE_DIR, "src", "opentelemetry", "instrumentation", "kafka", "package.py" +) +with open(PACKAGE_FILENAME, encoding="utf-8") as f: + exec(f.read(), PACKAGE_INFO) + +# Mark any instruments/runtime dependencies as test dependencies as well. +extras_require["instruments"] = PACKAGE_INFO["_instruments"] +test_deps = extras_require.get("test", []) +for dep in extras_require["instruments"]: + test_deps.append(dep) + +extras_require["test"] = test_deps + + +class JSONMetadataCommand(distutils.cmd.Command): + + description = ( + "print out package metadata as JSON. This is used by OpenTelemetry dev scripts to ", + "auto-generate code in other places", + ) + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + metadata = { + "name": config["metadata"]["name"], + "version": PACKAGE_INFO["__version__"], + "instruments": PACKAGE_INFO["_instruments"], + } + print(json.dumps(metadata)) + + +setuptools.setup( + cmdclass={"meta": JSONMetadataCommand}, + version=PACKAGE_INFO["__version__"], + extras_require=extras_require, +) diff --git a/instrumentation/opentelemetry-instrumentation-kafka-python/src/opentelemetry/instrumentation/kafka/__init__.py b/instrumentation/opentelemetry-instrumentation-kafka-python/src/opentelemetry/instrumentation/kafka/__init__.py new file mode 100644 index 0000000000..c3e0e35e7f --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-kafka-python/src/opentelemetry/instrumentation/kafka/__init__.py @@ -0,0 +1,95 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Instrument `kafka-python` to report instrumentation-kafka produced and consumed messages + +Usage +----- + +..code:: python + + from opentelemetry.instrumentation.kafka import KafkaInstrumentor + from kafka import KafkaProducer, KafkaConsumer + + # Instrument kafka + KafkaInstrumentor().instrument() + + # report a span of type producer with the default settings + producer = KafkaProducer(bootstrap_servers=['localhost:9092']) + producer.send('my-topic', b'raw_bytes') + + + # report a span of type consumer with the default settings + consumer = KafkaConsumer('my-topic', + group_id='my-group', + bootstrap_servers=['localhost:9092']) + for message in consumer: + # process message + +API +___ +""" +from typing import Collection + +import kafka +from wrapt import wrap_function_wrapper + +from opentelemetry import trace +from opentelemetry.instrumentation.instrumentor import BaseInstrumentor +from opentelemetry.instrumentation.kafka.package import _instruments +from opentelemetry.instrumentation.kafka.utils import ( + _wrap_next, + _wrap_send, + dummy_callback, +) +from opentelemetry.instrumentation.kafka.version import __version__ +from opentelemetry.instrumentation.utils import unwrap + + +class KafkaInstrumentor(BaseInstrumentor): + """An instrumentor for kafka module + See `BaseInstrumentor` + """ + + def instrumentation_dependencies(self) -> Collection[str]: + return _instruments + + def _instrument(self, **kwargs): + """Instruments the kafka module + + Args: + **kwargs: Optional arguments + ``tracer_provider``: a TracerProvider, defaults to global. + ``produce_hook``: a callable to be executed just before producing a message + ``consume _hook``: a callable to be executed just after consuming a message + """ + tracer_provider = kwargs.get("tracer_provider") + produce_hook = kwargs.get("produce_hook", dummy_callback) + consume_hook = kwargs.get("consume_hook", dummy_callback) + + tracer = trace.get_tracer( + __name__, __version__, tracer_provider=tracer_provider + ) + + wrap_function_wrapper( + "kafka", "KafkaProducer.send", _wrap_send(tracer, produce_hook) + ) + wrap_function_wrapper( + "kafka", "KafkaConsumer.__next__", _wrap_next(tracer, consume_hook) + ) + + def _uninstrument(self, **kwargs): + unwrap(kafka.KafkaProducer, "send") + unwrap(kafka.KafkaConsumer, "__next__") diff --git a/instrumentation/opentelemetry-instrumentation-kafka-python/src/opentelemetry/instrumentation/kafka/package.py b/instrumentation/opentelemetry-instrumentation-kafka-python/src/opentelemetry/instrumentation/kafka/package.py new file mode 100644 index 0000000000..04863fb7b4 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-kafka-python/src/opentelemetry/instrumentation/kafka/package.py @@ -0,0 +1,16 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +_instruments = ("kafka-python >= 2.0",) diff --git a/instrumentation/opentelemetry-instrumentation-kafka-python/src/opentelemetry/instrumentation/kafka/utils.py b/instrumentation/opentelemetry-instrumentation-kafka-python/src/opentelemetry/instrumentation/kafka/utils.py new file mode 100644 index 0000000000..ebcfe5c8bc --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-kafka-python/src/opentelemetry/instrumentation/kafka/utils.py @@ -0,0 +1,215 @@ +import json +from logging import getLogger +from typing import Callable, Dict, List, Optional + +from opentelemetry import trace +from opentelemetry.context import attach +from opentelemetry.propagate import extract, inject +from opentelemetry.propagators import textmap +from opentelemetry.semconv.trace import SpanAttributes +from opentelemetry.trace import Tracer, set_span_in_context +from opentelemetry.trace.span import Span + +_LOG = getLogger(__name__) + + +class KafkaPropertiesExtractor: + @staticmethod + def extract_bootstrap_servers(instance): + return instance.config.get("bootstrap_servers") + + @staticmethod + def _extract_argument(key, position, default_value, args, kwargs): + if len(args) > position: + return args[position] + return kwargs.get(key, default_value) + + @staticmethod + def extract_send_topic(args): + """extract topic from `send` method arguments in KafkaProducer class""" + if len(args) > 0: + return args[0] + return "unknown" + + @staticmethod + def extract_send_value(args, kwargs): + """extract value from `send` method arguments in KafkaProducer class""" + return KafkaPropertiesExtractor._extract_argument( + "value", 1, None, args, kwargs + ) + + @staticmethod + def extract_send_key(args, kwargs): + """extract key from `send` method arguments in KafkaProducer class""" + return KafkaPropertiesExtractor._extract_argument( + "key", 2, None, args, kwargs + ) + + @staticmethod + def extract_send_headers(args, kwargs): + """extract headers from `send` method arguments in KafkaProducer class""" + return KafkaPropertiesExtractor._extract_argument( + "headers", 3, None, args, kwargs + ) + + @staticmethod + def extract_send_partition(instance, args, kwargs): + """extract partition `send` method arguments, using the `_partition` method in KafkaProducer class""" + topic = KafkaPropertiesExtractor.extract_send_topic(args) + key = KafkaPropertiesExtractor.extract_send_key(args, kwargs) + value = KafkaPropertiesExtractor.extract_send_value(args, kwargs) + partition = KafkaPropertiesExtractor._extract_argument( + "partition", 4, None, args, kwargs + ) + key_bytes = instance._serialize( + instance.config["key_serializer"], topic, key + ) + value_bytes = instance._serialize( + instance.config["value_serializer"], topic, value + ) + valid_types = (bytes, bytearray, memoryview, type(None)) + if ( + type(key_bytes) not in valid_types + or type(value_bytes) not in valid_types + ): + return None + return instance._partition( + topic, partition, key, value, key_bytes, value_bytes + ) + + +HookT = Callable[[Span, List, Dict], None] + + +def dummy_callback(span, args, kwargs): + ... + + +class KafkaContextGetter(textmap.Getter): + def get( + self, carrier: textmap.CarrierT, requested_key: str + ) -> Optional[List[str]]: + if carrier is None: + return None + + for key, value in carrier: + if key == requested_key: + if value is not None: + return [value.decode()] + return None + + def keys(self, carrier: textmap.CarrierT) -> List[str]: + if carrier is None: + return [] + return [key for (key, value) in carrier] + + +class KafkaContextSetter(textmap.Setter): + def set(self, carrier: textmap.CarrierT, key: str, value: str) -> None: + if carrier is None or key is None: + return + + if value: + value = value.encode() + carrier.append((key, value)) + + +_kafka_getter = KafkaContextGetter() +_kafka_setter = KafkaContextSetter() + + +def _enrich_span( + span, bootstrap_servers: List[str], topic: str, partition: int +): + if span.is_recording(): + span.set_attribute(SpanAttributes.MESSAGING_SYSTEM, "kafka") + span.set_attribute(SpanAttributes.MESSAGING_DESTINATION, topic) + span.set_attribute(SpanAttributes.MESSAGING_KAFKA_PARTITION, partition) + span.set_attribute( + SpanAttributes.MESSAGING_URL, json.dumps(bootstrap_servers) + ) + + +def _get_span_name(operation: str, topic: str): + return f"kafka.{operation}: {topic}" + + +def _wrap_send(tracer: Tracer, produce_hook: HookT) -> Callable: + def _traced_send(func, instance, args, kwargs): + headers = KafkaPropertiesExtractor.extract_send_headers(args, kwargs) + if headers is None: + headers = [] + kwargs["headers"] = headers + + topic = KafkaPropertiesExtractor.extract_send_topic(args) + bootstrap_servers = KafkaPropertiesExtractor.extract_bootstrap_servers( + instance + ) + partition = KafkaPropertiesExtractor.extract_send_partition( + instance, args, kwargs + ) + span_name = _get_span_name("send", topic) + with tracer.start_as_current_span( + span_name, kind=trace.SpanKind.PRODUCER + ) as span: + _enrich_span(span, bootstrap_servers, topic, partition) + inject( + headers, + context=set_span_in_context(span), + setter=_kafka_setter, + ) + try: + produce_hook(span, args, kwargs) + except Exception as hook_exception: # pylint: disable=W0703 + _LOG.exception(hook_exception) + + return func(*args, **kwargs) + + return _traced_send + + +def _start_consume_span_with_extracted_context( + tracer: Tracer, headers: List, topic: str +) -> Span: + extracted_context = extract(headers, getter=_kafka_getter) + span_name = _get_span_name("consume", topic) + span = tracer.start_span( + span_name, context=extracted_context, kind=trace.SpanKind.CONSUMER + ) + new_context = set_span_in_context(span, extracted_context) + attach(new_context) + return span + + +def _wrap_next(tracer: Tracer, consume_hook: HookT) -> Callable: + def _traced_next(func, instance, args, kwargs): + # End the current span if exists before processing the next record + current_span = trace.get_current_span() + if current_span.is_recording() and current_span.name.startswith( + "kafka.consume" + ): + current_span.end() + + record = func(*args, **kwargs) + + if record: + headers = record.headers + topic = record.topic + bootstrap_servers = ( + KafkaPropertiesExtractor.extract_bootstrap_servers(instance) + ) + partition = record.partition + span = _start_consume_span_with_extracted_context( + tracer, headers, topic + ) + with trace.use_span(span): + _enrich_span(span, bootstrap_servers, topic, partition) + try: + consume_hook(span, args, kwargs) + except Exception as hook_exception: # pylint: disable=W0703 + _LOG.exception(hook_exception) + # We do not close the current span when returning to the caller, + # so the message processing logic will have the kafka span as parent + return record + + return _traced_next diff --git a/instrumentation/opentelemetry-instrumentation-kafka-python/src/opentelemetry/instrumentation/kafka/version.py b/instrumentation/opentelemetry-instrumentation-kafka-python/src/opentelemetry/instrumentation/kafka/version.py new file mode 100644 index 0000000000..a48aab26a5 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-kafka-python/src/opentelemetry/instrumentation/kafka/version.py @@ -0,0 +1,15 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "0.26b1" diff --git a/instrumentation/opentelemetry-instrumentation-kafka-python/tests/__init__.py b/instrumentation/opentelemetry-instrumentation-kafka-python/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/instrumentation/opentelemetry-instrumentation-kafka-python/tests/test_instrumentation.py b/instrumentation/opentelemetry-instrumentation-kafka-python/tests/test_instrumentation.py new file mode 100644 index 0000000000..7bc724fb78 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-kafka-python/tests/test_instrumentation.py @@ -0,0 +1,36 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from unittest import TestCase + +from kafka import KafkaConsumer, KafkaProducer +from wrapt import BoundFunctionWrapper + +from opentelemetry.instrumentation.kafka import KafkaInstrumentor + + +class TestKafka(TestCase): + def test_instrument_api(self) -> None: + instrumentation = KafkaInstrumentor() + + instrumentation.instrument() + self.assertTrue(isinstance(KafkaProducer.send, BoundFunctionWrapper)) + self.assertTrue( + isinstance(KafkaConsumer.__next__, BoundFunctionWrapper) + ) + + instrumentation.uninstrument() + self.assertFalse(isinstance(KafkaProducer.send, BoundFunctionWrapper)) + self.assertFalse( + isinstance(KafkaConsumer.__next__, BoundFunctionWrapper) + ) diff --git a/instrumentation/opentelemetry-instrumentation-kafka-python/tests/test_utils.py b/instrumentation/opentelemetry-instrumentation-kafka-python/tests/test_utils.py new file mode 100644 index 0000000000..264f34d7f4 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-kafka-python/tests/test_utils.py @@ -0,0 +1,151 @@ +from unittest import TestCase, mock + +from opentelemetry.instrumentation.kafka.utils import ( + _get_span_name, + _kafka_getter, + _kafka_setter, + _start_consume_span_with_extracted_context, + _wrap_next, + _wrap_send, +) +from opentelemetry.trace import SpanKind + + +class TestUtils(TestCase): + def setUp(self) -> None: + super().setUp() + self.topic_name = "test_topic" + self.args = [self.topic_name] + self.headers = [] + self.kwargs = {"partition": 0, "headers": self.headers} + + @mock.patch( + "opentelemetry.instrumentation.kafka.utils.KafkaPropertiesExtractor.extract_bootstrap_servers" + ) + @mock.patch( + "opentelemetry.instrumentation.kafka.utils.KafkaPropertiesExtractor.extract_send_partition" + ) + @mock.patch("opentelemetry.instrumentation.kafka.utils._enrich_span") + @mock.patch("opentelemetry.trace.set_span_in_context") + @mock.patch("opentelemetry.propagate.inject") + def test_wrap_send( + self, + inject: mock.MagicMock, + set_span_in_context: mock.MagicMock, + enrich_span: mock.MagicMock, + extract_send_partition: mock.MagicMock, + extract_bootstrap_servers: mock.MagicMock, + ): + tracer = mock.MagicMock() + produce_hook = mock.MagicMock() + original_send_callback = mock.MagicMock() + kafka_producer = mock.MagicMock() + expected_span_name = _get_span_name("send", self.topic_name) + + wrapped_send = _wrap_send(tracer, produce_hook) + retval = wrapped_send( + original_send_callback, kafka_producer, self.args, self.kwargs + ) + + extract_bootstrap_servers.assert_called_once_with(kafka_producer) + extract_send_partition.assert_called_once_with( + kafka_producer, self.args, self.kwargs + ) + tracer.start_as_current_span.assert_called_once_with( + expected_span_name, kind=SpanKind.PRODUCER + ) + + span = tracer.start_as_current_span().__enter__.return_value + enrich_span.assert_called_once_with( + span, + extract_bootstrap_servers.return_value, + self.topic_name, + extract_send_partition.return_value, + ) + + set_span_in_context.assert_called_once_with(span) + context = set_span_in_context.return_value + inject.assert_called_once_with( + self.headers, context=context, setter=_kafka_setter + ) + + produce_hook.assert_called_once_with(span, self.args, self.kwargs) + + original_send_callback.assert_called_once_with( + *self.args, **self.kwargs + ) + self.assertEqual(retval, original_send_callback.return_value) + + @mock.patch("opentelemetry.trace.get_current_span") + @mock.patch("opentelemetry.trace.use_span") + @mock.patch( + "opentelemetry.instrumentation.kafka.utils._start_consume_span_with_extracted_context" + ) + @mock.patch("opentelemetry.instrumentation.kafka.utils._enrich_span") + @mock.patch( + "opentelemetry.instrumentation.kafka.utils.KafkaPropertiesExtractor.extract_bootstrap_servers" + ) + def test_wrap_next( + self, + extract_bootstrap_servers: mock.MagicMock, + enrich_span: mock.MagicMock, + start_consume_span_with_extracted_context: mock.MagicMock, + use_span: mock.MagicMock, + get_current_span: mock.MagicMock, + ) -> None: + tracer = mock.MagicMock() + consume_hook = mock.MagicMock() + original_next_callback = mock.MagicMock() + kafka_consumer = mock.MagicMock() + + wrapped_next = _wrap_next(tracer, consume_hook) + record = wrapped_next( + original_next_callback, kafka_consumer, self.args, self.kwargs + ) + + extract_bootstrap_servers.assert_called_once_with(kafka_consumer) + bootstrap_servers = extract_bootstrap_servers.return_value + get_current_span.assert_called_once() + current_span = get_current_span.return_value + current_span.end.assert_called_once() + + original_next_callback.assert_called_once_with( + *self.args, **self.kwargs + ) + self.assertEqual(record, original_next_callback.return_value) + + start_consume_span_with_extracted_context.assert_called_once_with( + tracer, record.headers, record.topic + ) + span = start_consume_span_with_extracted_context.return_value + use_span.assert_called_once_with(span) + enrich_span.assert_called_once_with( + span, bootstrap_servers, record.topic, record.partition + ) + consume_hook.assert_called_once_with(span, self.args, self.kwargs) + + @mock.patch("opentelemetry.context.attach") + @mock.patch("opentelemetry.trace.set_span_in_context") + @mock.patch("opentelemetry.propagate.extract") + def test_start_consume_span_with_extracted_context( + self, + extract: mock.MagicMock, + set_span_in_context: mock.MagicMock, + attach: mock.MagicMock, + ): + tracer = mock.MagicMock() + expected_span_name = _get_span_name("consume", self.topic_name) + + _start_consume_span_with_extracted_context( + tracer, self.headers, self.topic_name + ) + + extract.assert_called_once_with(self.headers, _kafka_getter) + context = extract.return_value + tracer.start_span.assert_called_once_with( + expected_span_name, context=context, kind=SpanKind.CONSUMER + ) + span = tracer.start_span.return_value + set_span_in_context.assert_called_once_with(span, context) + new_context = set_span_in_context.return_value + attach.assert_called_once_with(new_context) diff --git a/opentelemetry-instrumentation/src/opentelemetry/instrumentation/bootstrap_gen.py b/opentelemetry-instrumentation/src/opentelemetry/instrumentation/bootstrap_gen.py index 27a1717ea0..205a763e46 100644 --- a/opentelemetry-instrumentation/src/opentelemetry/instrumentation/bootstrap_gen.py +++ b/opentelemetry-instrumentation/src/opentelemetry/instrumentation/bootstrap_gen.py @@ -76,6 +76,10 @@ "library": "jinja2 >= 2.7, < 4.0", "instrumentation": "opentelemetry-instrumentation-jinja2==0.26b1", }, + "kafka-python": { + "library": "kafka-python >= 2.0", + "instrumentation": "opentelemetry-instrumentation-kafka-python==0.26b1", + }, "mysql-connector-python": { "library": "mysql-connector-python ~= 8.0", "instrumentation": "opentelemetry-instrumentation-mysql==0.26b1", diff --git a/tox.ini b/tox.ini index 97673d1a47..e945267f92 100644 --- a/tox.ini +++ b/tox.ini @@ -185,6 +185,10 @@ envlist = py3{6,7,8,9,10}-test-instrumentation-pika pypy3-test-instrumentation-pika + ; opentelemetry-instrumentation-kafka-python + py3{6,7,8,9,10}-test-instrumentation-kafka-python + pypy3-test-instrumentation-kafka-python + lint docker-tests docs @@ -247,6 +251,7 @@ changedir = test-instrumentation-urllib3: instrumentation/opentelemetry-instrumentation-urllib3/tests test-instrumentation-grpc: instrumentation/opentelemetry-instrumentation-grpc/tests test-instrumentation-jinja2: instrumentation/opentelemetry-instrumentation-jinja2/tests + test-instrumentation-kafka-python: instrumentation/opentelemetry-instrumentation-kafka-python/tests test-instrumentation-logging: instrumentation/opentelemetry-instrumentation-logging/tests test-instrumentation-mysql: instrumentation/opentelemetry-instrumentation-mysql/tests test-instrumentation-pika: instrumentation/opentelemetry-instrumentation-pika/tests @@ -288,6 +293,8 @@ commands_pre = pika: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-pika[test] + kafka: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-kafka-python[test] + grpc: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-grpc[test] falcon{2,3},flask,django{1,2,3,4},pyramid,tornado,starlette,fastapi,aiohttp,asgi,requests,urllib,urllib3,wsgi: pip install {toxinidir}/util/opentelemetry-util-http[test] @@ -429,6 +436,7 @@ commands_pre = python -m pip install -e {toxinidir}/instrumentation/opentelemetry-instrumentation-redis[test] python -m pip install -e {toxinidir}/instrumentation/opentelemetry-instrumentation-fastapi[test] python -m pip install -e {toxinidir}/instrumentation/opentelemetry-instrumentation-jinja2[test] + python -m pip install -e {toxinidir}/instrumentation/opentelemetry-instrumentation-kafka-python[test] python -m pip install -e {toxinidir}/instrumentation/opentelemetry-instrumentation-logging[test] python -m pip install -e {toxinidir}/instrumentation/opentelemetry-instrumentation-pymemcache[test] python -m pip install -e {toxinidir}/instrumentation/opentelemetry-instrumentation-psycopg2[test] @@ -488,6 +496,7 @@ commands_pre = -e {toxinidir}/instrumentation/opentelemetry-instrumentation-asyncpg \ -e {toxinidir}/instrumentation/opentelemetry-instrumentation-celery \ -e {toxinidir}/instrumentation/opentelemetry-instrumentation-pika \ + -e {toxinidir}/instrumentation/opentelemetry-instrumentation-kafka-python \ -e {toxinidir}/instrumentation/opentelemetry-instrumentation-dbapi \ -e {toxinidir}/instrumentation/opentelemetry-instrumentation-mysql \ -e {toxinidir}/instrumentation/opentelemetry-instrumentation-psycopg2 \