feat: add plugin execution tracing to OpenTelemetry plugin

This commit adds plugin execution tracing capability to the OpenTelemetry plugin,
allowing users to trace individual plugin phases (rewrite, access, header_filter,
body_filter, log) as child spans of the main request trace.

Changes:
- Added trace_plugins configuration option (default: false, opt-in)
- Added plugin_span_kind configuration for observability provider compatibility
- Enhanced plugin execution with OpenTelemetry span creation and finishing
- Added comprehensive request context attributes to plugin spans
- Updated documentation with examples and usage instructions
- Added comprehensive test suite for the new functionality

Features:
- Plugin Phase Tracing: Creates child spans for each plugin phase execution
- Rich Context: Includes HTTP method, URI, hostname, user agent, route info, and service info
- Configurable: Can be enabled/disabled via trace_plugins configuration
- Span Kind Control: Supports internal (default) and server span kinds for observability provider compatibility
- Proper Hierarchy: Plugin spans are correctly nested under main request spans
- Performance: Minimal overhead when disabled (default behavior)

Configuration:
- trace_plugins: boolean (default: false) - Enable/disable plugin tracing
- plugin_span_kind: string (default: 'internal') - Span kind for plugin spans
  - 'internal': Standard internal operation (may be excluded from metrics)
  - 'server': Server-side operation (typically included in service-level metrics)

This addresses GitHub issue #12510 and provides end-to-end tracing visibility
for APISIX plugin execution phases.

Author: alibakhshoude-afk

README.md

@@ -123,6 +123,7 @@ A/B testing, canary release, blue-green deployment, limit rate, defense against
 - **OPS friendly**
 
   - Zipkin tracing: [Zipkin](docs/en/latest/plugins/zipkin.md)
+  - OpenTelemetry tracing: [OpenTelemetry](docs/en/latest/plugins/opentelemetry.md) with plugin execution tracing
   - Open source APM: support [Apache SkyWalking](docs/en/latest/plugins/skywalking.md)
   - Works with external service discovery: In addition to the built-in etcd, it also supports [Consul](docs/en/latest/discovery/consul.md), [Consul_kv](docs/en/latest/discovery/consul_kv.md), [Nacos](docs/en/latest/discovery/nacos.md), [Eureka](docs/en/latest/discovery/eureka.md) and [Zookeeper (CP)](https://github.com/api7/apisix-seed/blob/main/docs/en/latest/zookeeper.md).
   - Monitoring And Metrics: [Prometheus](docs/en/latest/plugins/prometheus.md)

apisix/plugin.lua

@@ -38,6 +38,7 @@ local tostring      = tostring
 local error         = error
 local getmetatable  = getmetatable
 local setmetatable  = setmetatable
+local string_format = string.format
 -- make linter happy to avoid error: getting the Lua global "load"
 -- luacheck: globals load, ignore lua_load
 local lua_load          = load
@@ -1169,6 +1170,9 @@ function _M.run_plugin(phase, plugins, api_ctx)
         return api_ctx
     end
 
+    -- Get OpenTelemetry plugin for tracing
+    local otel_plugin = _M.get("opentelemetry")
+
     if phase ~= "log"
         and phase ~= "header_filter"
         and phase ~= "body_filter"
@@ -1188,11 +1192,26 @@ function _M.run_plugin(phase, plugins, api_ctx)
                     goto CONTINUE
                 end
 
+                -- Start OpenTelemetry plugin span
+                if otel_plugin and otel_plugin.start_plugin_span then
+                    otel_plugin.start_plugin_span(api_ctx, plugins[i]["name"], phase)
+                end
+
                 run_meta_pre_function(conf, api_ctx, plugins[i]["name"])
                 plugin_run = true
                 api_ctx._plugin_name = plugins[i]["name"]
                 local code, body = phase_func(conf, api_ctx)
                 api_ctx._plugin_name = nil
+
+                -- Finish OpenTelemetry plugin span
+                if otel_plugin and otel_plugin.finish_plugin_span then
+                    local error_msg = nil
+                    if code and code >= 400 then
+                        error_msg = "plugin returned error code: " .. tostring(code)
+                    end
+                    otel_plugin.finish_plugin_span(api_ctx, plugins[i]["name"], phase, error_msg)
+                end
+
                 if code or body then
                     if is_http then
                         if code >= 400 then
@@ -1216,7 +1235,6 @@ function _M.run_plugin(phase, plugins, api_ctx)
                     end
                 end
             end
-
             ::CONTINUE::
         end
         return api_ctx, plugin_run
@@ -1226,11 +1244,26 @@ function _M.run_plugin(phase, plugins, api_ctx)
         local phase_func = plugins[i][phase]
         local conf = plugins[i + 1]
         if phase_func and meta_filter(api_ctx, plugins[i]["name"], conf) then
+            -- Start OpenTelemetry plugin span
+            if otel_plugin and otel_plugin.start_plugin_span then
+                otel_plugin.start_plugin_span(api_ctx, plugins[i]["name"], phase)
+            end
+
             plugin_run = true
             run_meta_pre_function(conf, api_ctx, plugins[i]["name"])
             api_ctx._plugin_name = plugins[i]["name"]
-            phase_func(conf, api_ctx)
+
+            local code, body = phase_func(conf, api_ctx)
             api_ctx._plugin_name = nil
+
+            -- Finish OpenTelemetry plugin span
+            if otel_plugin and otel_plugin.finish_plugin_span then
+                local error_msg = nil
+                if code and code >= 400 then
+                    error_msg = "plugin returned error code: " .. tostring(code)
+                end
+                otel_plugin.finish_plugin_span(api_ctx, plugins[i]["name"], phase, error_msg)
+            end
         end
     end