diff --git a/.chloggen/yanggrpc_wip.yaml b/.chloggen/yanggrpc_wip.yaml new file mode 100644 index 0000000000000..466814f28fb5e --- /dev/null +++ b/.chloggen/yanggrpc_wip.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. receiver/filelog) +component: receiver/yang_grpc + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Support collecting any metric by browsing the whole metrics tree + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [47054] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/receiver/yanggrpcreceiver/README.md b/receiver/yanggrpcreceiver/README.md index 84e87489706c1..3e23309a31e5a 100644 --- a/receiver/yanggrpcreceiver/README.md +++ b/receiver/yanggrpcreceiver/README.md @@ -18,117 +18,60 @@ The YANG/gRPC receiver receives metrics offered using the [contrib]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib -This receiver exposes a gRPC endpoint, accepting the gRPC configuration offered by the [configgrpc](https://pkg.go.dev/go.opentelemetry.io/collector/config/configgrpc) module. +The **YANG gRPC Receiver** collects Model-Driven Telemetry (MDT) from network devices (primarily Cisco) via gRPC Dial-out. It transforms complex Cisco KV-GPB (Key-Value Google Protocol Buffers) data into standard OpenTelemetry metrics, specifically optimized for high-performance analysis in **Splunk**. -# Configuration +## Key Features -## Definition +- **Context-Aware Processing**: Automatically discovers dimensions (labels) like Interface names, VRF IDs, or BGP neighbors by traversing the telemetry tree. +- **YANG-Driven Mapping**: Uses Cisco YANG models to distinguish between Counters (monotonic sums) and Gauges (instantaneous values). +- **Smart Fallback**: Works out-of-the-box using naming heuristics (detecting keys like `name`, `id`, `address`) even if local YANG files are not provided. +- **OTLP Compliance**: Normalizes all Cisco numeric types into `float64` and handles string values as descriptive `_info` metrics. +- **Security Hardening**: Includes built-in support for client IP allow-listing and ingestion rate limiting. -### gRPC Configuration +--- -See [configgrpc](https://pkg.go.dev/go.opentelemetry.io/collector/config/configgrpc). +### Example Configuration -The default configuration sets up the following parameters: +Add this receiver to your OpenTelemetry Collector configuration: ```yaml - endpoint: localhost:57500 - transport: tcp - keepalive: - server_parameters: - time: 30s - timeout: 10s - max_concurrent_streams: 100 - max_recv_msg_size_mib: 4 +receivers: + yang_grpc: + endpoint: "0.0.0.0:57000" + yang: + module_paths: + - "/etc/otelcol/yang/cisco/xe" + - "/etc/otelcol/yang/standard/ietf" + security: + allowed_clients: ["10.0.0.0/8", "192.168.1.0/24"] + rate_limiting: + enabled: true + requests_per_second: 100 + burst_size: 50 + +exporters: + splunk_hec: + endpoint: "https://your-splunk-instance:8088/services/collector" + token: "${SPLUNK_HEC_TOKEN}" + index: "network_metrics" + +service: + pipelines: + metrics: + receivers: [yang_grpc] + processors: [batch] + exporters: [splunk_hec] ``` -### Security Configuration (`security`) - -#### Rate Limiting (`rate_limiting`) - -##### `enabled` -- **Type**: `bool` -- **Default**: `false` -- **Description**: Enable per-client rate limiting - -##### `requests_per_second` -- **Type**: `float64` -- **Default**: `100.0` -- **Description**: Maximum requests per second per client - -##### `burst_size` -- **Type**: `int` -- **Default**: `10` -- **Description**: Burst allowance for rate limiting - -##### `cleanup_interval` -- **Type**: `duration` -- **Default**: `1m` -- **Description**: How often to clean up rate limiter entries - -#### Access Control - -##### `allowed_clients` -- **Type**: `[]string` -- **Default**: `[]` (allow all) -- **Description**: IP addresses or CIDR blocks allowed to connect -- **Examples**: - ```yaml - allowed_clients: - - "10.1.1.100" # Specific IP - - "192.168.0.0/16" # CIDR block - - "10.0.0.0/8" # Large network - ``` - -##### `max_connections` -- **Type**: `int` -- **Default**: `1000` -- **Description**: Maximum concurrent connections - -##### `connection_timeout` -- **Type**: `duration` -- **Default**: `30s` -- **Description**: Timeout for new connections - -##### `enable_metrics` -- **Type**: `bool` -- **Default**: `true` -- **Description**: Enable security-related metrics collection - -## Performance Tuning - -### Keep-Alive Configuration (`keep_alive`) +--- -#### `time` -- **Type**: `duration` -- **Default**: `30s` -- **Description**: Time between keep-alive pings - -#### `timeout` -- **Type**: `duration` -- **Default**: `10s` -- **Description**: Timeout waiting for keep-alive response - -## YANG Parser Settings - -### YANG Configuration (`yang`) - -#### `enable_rfc_parser` -- **Type**: `bool` -- **Default**: `true` -- **Description**: Enable RFC 6020/7950 compliant YANG parser - -#### `cache_modules` -- **Type**: `bool` -- **Default**: `true` -- **Description**: Cache discovered YANG modules +# Configuration -#### `max_modules` -- **Type**: `int` -- **Default**: `1000` -- **Description**: Maximum number of YANG modules to cache +## gRPC Configuration -## Default +See [configgrpc](https://pkg.go.dev/go.opentelemetry.io/collector/config/configgrpc). +## Default Configuration ```yaml yang_grpc: endpoint: localhost:57500 @@ -139,63 +82,47 @@ yang_grpc: timeout: 10s max_concurrent_streams: 100 max_recv_msg_size_mib: 4 - security: - connection_timeout: 30s - enable_metrics: true - rate_limiting: - burst_size: 10 - cleanup_interval: 1m0s - enabled: false - requests_per_second: 100 - yang: - cache_modules: true - enable_rfc_parser: true - max_modules: 1000 ``` -## Production +## Security Configuration (security) +* `rate_limiting`: enabled (default: false), requests_per_second (100.0), burst_size (10). +* `access_control`: allowed_clients (list of IP/CIDR), max_connections (1000). -```yaml +## YANG Parser Settings (yang) +* `enable_rfc_parser`: Enable RFC 6020/7950 compliant parsing. +* `cache_modules`: Local directories containing Cisco/IETF `.yang` files for accurate parsing. Cache discovered YANG modules to reduce CPU overhead. + +## Production Deployment Example +```YAML yang_grpc: endpoint: "0.0.0.0:57500" - # TLS/mTLS Configuration tls: enabled: true cert_file: "/etc/otel/certs/server.crt" key_file: "/etc/otel/certs/server.key" - ca_file: "/etc/otel/certs/ca.crt" - client_auth_type: "RequireAndVerifyClientCert" - min_version: "1.2" - max_version: "1.3" - reload_interval: 5m - - # Security & Rate Limiting security: rate_limiting: enabled: true - requests_per_second: 100.0 - burst_size: 10 - cleanup_interval: 1m allowed_clients: - "10.0.0.0/8" - - "192.168.0.0/16" - max_connections: 1000 - connection_timeout: 30s - enable_metrics: true - - # Performance Settings - max_concurrent_streams: 100 - max_recv_msg_size_mib: 4 - - # Keep-Alive Configuration - keepalive: - server_parameters: - time: 30s - timeout: 10s - - # YANG Parser Configuration - yang: - enable_rfc_parser: true - cache_modules: true - max_modules: 1000 + yang: + cache_modules: true ``` + +--- + +# Example OTLP Output +When processing Cisco ARP telemetry data, the receiver generates structured OTLP metrics: + +### Metric Name: cisco.content.arp-oper.type_info + +Value: 1.0 (Info Metric) + +### Attributes: + +* interface: Vlan200 +* vrf: Default +* address: 10.10.10.1 +* yang.module: Cisco-IOS-XE-arp-oper +* cisco.node_id: Switch-Core-01 +* cisco.subscription_id: 112 diff --git a/receiver/yanggrpcreceiver/config.go b/receiver/yanggrpcreceiver/config.go index d0f46e3b1e7f4..6db881b09b448 100644 --- a/receiver/yanggrpcreceiver/config.go +++ b/receiver/yanggrpcreceiver/config.go @@ -64,6 +64,10 @@ type YANGConfig struct { // MaxModules is the maximum number of YANG modules to cache MaxModules int `mapstructure:"max_modules"` + + // ModulePaths defines the directories where .yang files are stored. + // This is used by the internal parser to resolve Cisco-specific schemas. + ModulePaths []string `mapstructure:"module_paths"` } // Config defines configuration for yanggrpc receiver. @@ -77,14 +81,21 @@ type Config struct { Security SecurityConfig `mapstructure:"security"` } -// Validate checks the receiver configuration is valid +// Validate checks the receiver configuration is valid. func (c *Config) Validate() error { + // Validate the base gRPC server configuration (endpoint, TLS, etc.) if err := c.ServerConfig.Validate(); err != nil { return err } + + // Validate security settings if err := c.Security.Validate(); err != nil { return err } + // Optional: You could add a check here to ensure ModulePaths aren't empty + // if EnableRFCParser is true, but since we have a "fallback" logic + // in grpc_service.go, it's better to keep it optional. + return nil } diff --git a/receiver/yanggrpcreceiver/config.schema.yaml b/receiver/yanggrpcreceiver/config.schema.yaml index 09a0ba3a10d8e..2de45ef03d25a 100644 --- a/receiver/yanggrpcreceiver/config.schema.yaml +++ b/receiver/yanggrpcreceiver/config.schema.yaml @@ -49,6 +49,11 @@ $defs: max_modules: description: MaxModules is the maximum number of YANG modules to cache type: integer + module_paths: + description: ModulePaths defines the directories where .yang files are stored. This is used by the internal parser to resolve Cisco-specific schemas. + type: array + items: + type: string description: Config defines configuration for yanggrpc receiver. type: object properties: diff --git a/receiver/yanggrpcreceiver/detailed_validation_test.go b/receiver/yanggrpcreceiver/detailed_validation_test.go index 714a0da68471d..be26ef8ead1c6 100644 --- a/receiver/yanggrpcreceiver/detailed_validation_test.go +++ b/receiver/yanggrpcreceiver/detailed_validation_test.go @@ -77,12 +77,9 @@ func TestDetailedTelemetryValidation(t *testing.T) { } ctx := t.Context() - rcvr, err := createMetricsReceiver(ctx, settings, config, csmr) - if err != nil { - t.Fatalf("Failed to create receiver: %v", err) - } + rcvr := createMetricsReceiver(ctx, settings, config, csmr) - err = rcvr.Start(ctx, componenttest.NewNopHost()) + err := rcvr.Start(ctx, componenttest.NewNopHost()) if err != nil { t.Fatalf("Failed to start receiver: %v", err) } @@ -129,9 +126,7 @@ func TestDetailedTelemetryValidation(t *testing.T) { for _, tc := range testCases { err := sendDetailedTelemetryData("localhost:57403", "CISCO-TEST-SWITCH", tc.interfaceName, tc.rxPkts, tc.txPkts, tc.rxBytes, tc.txBytes) - if err != nil { - t.Fatalf("Failed to send telemetry for %s: %v", tc.name, err) - } + assert.NoError(t, err, "Failed to send telemetry for %s: %v", tc.name, err) } // Wait for data processing @@ -409,7 +404,7 @@ func sendDetailedTelemetryData(endpoint, nodeID, interfaceName string, rxPkts, t } _, err = stream.Recv() - if err != nil && errors.Is(err, io.EOF) { + if err != nil && !errors.Is(err, io.EOF) { return fmt.Errorf("unexpected error receiving response: %w", err) } diff --git a/receiver/yanggrpcreceiver/factory.go b/receiver/yanggrpcreceiver/factory.go index 23e9cfe1b3297..2ec3ae6a68d17 100644 --- a/receiver/yanggrpcreceiver/factory.go +++ b/receiver/yanggrpcreceiver/factory.go @@ -4,10 +4,12 @@ package yanggrpcreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/yanggrpcreceiver" import ( + "context" "time" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/config/configgrpc" + "go.opentelemetry.io/collector/consumer" "go.opentelemetry.io/collector/receiver" "go.opentelemetry.io/collector/receiver/xreceiver" @@ -18,7 +20,9 @@ func NewFactory() receiver.Factory { return xreceiver.NewFactory( metadata.Type, createDefaultConfig, - xreceiver.WithMetrics(createMetricsReceiver, metadata.MetricsStability), + xreceiver.WithMetrics(func(ctx context.Context, settings receiver.Settings, config component.Config, metrics consumer.Metrics) (receiver.Metrics, error) { + return createMetricsReceiver(ctx, settings, config, metrics), nil + }, metadata.MetricsStability), xreceiver.WithDeprecatedTypeAlias(metadata.DeprecatedType), ) } diff --git a/receiver/yanggrpcreceiver/generated_component_test.go b/receiver/yanggrpcreceiver/generated_component_test.go index 71a796c1c5ea1..4083027d7c42c 100644 --- a/receiver/yanggrpcreceiver/generated_component_test.go +++ b/receiver/yanggrpcreceiver/generated_component_test.go @@ -3,11 +3,16 @@ package yanggrpcreceiver import ( + "context" "testing" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/confmap/confmaptest" + "go.opentelemetry.io/collector/consumer/consumertest" + "go.opentelemetry.io/collector/receiver" + "go.opentelemetry.io/collector/receiver/receivertest" ) var typ = component.MustNewType("yang_grpc") @@ -19,3 +24,64 @@ func TestComponentFactoryType(t *testing.T) { func TestComponentConfigStruct(t *testing.T) { require.NoError(t, componenttest.CheckConfigStruct(NewFactory().CreateDefaultConfig())) } + +func TestComponentLifecycle(t *testing.T) { + factory := NewFactory() + + tests := []struct { + createFn func(ctx context.Context, set receiver.Settings, cfg component.Config) (component.Component, error) + name string + }{ + + { + name: "metrics", + createFn: func(ctx context.Context, set receiver.Settings, cfg component.Config) (component.Component, error) { + return factory.CreateMetrics(ctx, set, cfg, consumertest.NewNop()) + }, + }, + } + + cm, err := confmaptest.LoadConf("metadata.yaml") + require.NoError(t, err) + cfg := factory.CreateDefaultConfig() + sub, err := cm.Sub("tests::config") + require.NoError(t, err) + require.NoError(t, sub.Unmarshal(&cfg)) + + for _, tt := range tests { + t.Run(tt.name+"-shutdown", func(t *testing.T) { + c, err := tt.createFn(context.Background(), receivertest.NewNopSettings(typ), cfg) + require.NoError(t, err) + err = c.Shutdown(context.Background()) + require.NoError(t, err) + }) + t.Run(tt.name+"-lifecycle", func(t *testing.T) { + firstRcvr, err := tt.createFn(context.Background(), receivertest.NewNopSettings(typ), cfg) + require.NoError(t, err) + host := newMdatagenNopHost() + require.NoError(t, err) + require.NoError(t, firstRcvr.Start(context.Background(), host)) + require.NoError(t, firstRcvr.Shutdown(context.Background())) + secondRcvr, err := tt.createFn(context.Background(), receivertest.NewNopSettings(typ), cfg) + require.NoError(t, err) + require.NoError(t, secondRcvr.Start(context.Background(), host)) + require.NoError(t, secondRcvr.Shutdown(context.Background())) + }) + } +} + +var _ component.Host = (*mdatagenNopHost)(nil) + +type mdatagenNopHost struct{} + +func newMdatagenNopHost() component.Host { + return &mdatagenNopHost{} +} + +func (mnh *mdatagenNopHost) GetExtensions() map[component.ID]component.Component { + return nil +} + +func (mnh *mdatagenNopHost) GetFactory(_ component.Kind, _ component.Type) component.Factory { + return nil +} diff --git a/receiver/yanggrpcreceiver/go.mod b/receiver/yanggrpcreceiver/go.mod index 631bd59e4d521..3438312694831 100644 --- a/receiver/yanggrpcreceiver/go.mod +++ b/receiver/yanggrpcreceiver/go.mod @@ -16,10 +16,7 @@ require ( go.opentelemetry.io/collector/receiver v1.54.1-0.20260320051400-372cc483b303 go.opentelemetry.io/collector/receiver/receivertest v0.148.1-0.20260320051400-372cc483b303 go.opentelemetry.io/collector/receiver/xreceiver v0.148.1-0.20260320051400-372cc483b303 - go.opentelemetry.io/otel v1.42.0 go.opentelemetry.io/otel/metric v1.42.0 - go.opentelemetry.io/otel/sdk/metric v1.42.0 - go.opentelemetry.io/otel/trace v1.42.0 go.uber.org/goleak v1.3.0 go.uber.org/zap v1.27.1 golang.org/x/time v0.15.0 @@ -69,7 +66,10 @@ require ( go.opentelemetry.io/collector/pdata/pprofile v0.148.1-0.20260320051400-372cc483b303 // indirect go.opentelemetry.io/collector/pipeline v1.54.1-0.20260320051400-372cc483b303 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect + go.opentelemetry.io/otel v1.42.0 // indirect go.opentelemetry.io/otel/sdk v1.42.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.42.0 // indirect + go.opentelemetry.io/otel/trace v1.42.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.48.0 // indirect diff --git a/receiver/yanggrpcreceiver/grpc_service.go b/receiver/yanggrpcreceiver/grpc_service.go index dc7801399a66f..48e4fc9b3ee56 100644 --- a/receiver/yanggrpcreceiver/grpc_service.go +++ b/receiver/yanggrpcreceiver/grpc_service.go @@ -8,492 +8,217 @@ import ( "errors" "fmt" "io" - "slices" + "maps" + "strconv" "strings" - "time" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/metric" "go.uber.org/zap" - "google.golang.org/grpc" "google.golang.org/protobuf/proto" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/yanggrpcreceiver/internal" pb "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/yanggrpcreceiver/internal/proto/generated/proto" ) -// grpcService implements the Cisco MDT gRPC service +// grpcService handles Cisco gRPC Dial-out telemetry streams. type grpcService struct { pb.UnimplementedGRPCMdtDialoutServer - receiver *yangReceiver - yangParser *internal.YANGParser - rfcYangParser *internal.RFC6020Parser + receiver *yangReceiver + yangParser *internal.YANGParser } -// MdtDialout handles the bidirectional streaming gRPC call from Cisco devices -func (s *grpcService) MdtDialout(stream grpc.BidiStreamingServer[pb.MdtDialoutArgs, pb.MdtDialoutArgs]) error { - ctx := context.Background() - - s.receiver.telemetryBuilder.YangReceiverConnectionsOpened.Add(ctx, 1) - defer s.receiver.telemetryBuilder.YangReceiverConnectionsClosed.Add(ctx, 1) - - s.receiver.settings.Logger.Debug("New MDT dialout connection established") - +// MdtDialout processes the bidirectional gRPC stream. +func (s *grpcService) MdtDialout(stream pb.GRPCMdtDialout_MdtDialoutServer) error { + s.receiver.settings.Logger.Info("New Cisco telemetry session established") for { - // Receive telemetry data from Cisco device req, err := stream.Recv() if errors.Is(err, io.EOF) { - s.receiver.settings.Logger.Debug("MDT dialout connection closed by client") return nil } if err != nil { - s.receiver.telemetryBuilder.YangReceiverGrpcErrors.Add(ctx, 1) // "receive_error", "stream_recv" - s.receiver.settings.Logger.Error("Error receiving MDT data", zap.Error(err)) return err } - // Process the received telemetry data - err = s.processTelemetryData(req) - if err != nil { - s.receiver.telemetryBuilder.YangReceiverGrpcErrors.Add(ctx, 1, metric.WithAttributeSet(attribute.NewSet( - attribute.String("error_type", "processing_error"), attribute.String("error", "process_telemetry"), - ))) - s.receiver.settings.Logger.Error("Error processing telemetry data", - zap.Error(err), zap.Int64("req_id", req.ReqId)) - - // Send error response back to device - resp := &pb.MdtDialoutArgs{ - ReqId: req.ReqId, - Errors: fmt.Sprintf("Processing error: %v", err), - } - if sendErr := stream.Send(resp); sendErr != nil { - s.receiver.telemetryBuilder.YangReceiverGrpcErrors.Add(ctx, 1, metric.WithAttributeSet(attribute.NewSet( - attribute.String("error_type", "send_error"), attribute.String("error", "stream_send"), - ))) - s.receiver.settings.Logger.Error("Failed to send error response", zap.Error(sendErr)) - } - continue - } - - // Send acknowledgment back to device - resp := &pb.MdtDialoutArgs{ - ReqId: req.ReqId, - } - if err := stream.Send(resp); err != nil { - s.receiver.settings.Logger.Error("Failed to send acknowledgment", zap.Error(err)) - return err + if err := s.processTelemetryData(req); err != nil { + s.receiver.settings.Logger.Error("Failed to process telemetry data", zap.Error(err)) } } } -// processTelemetryData parses the incoming telemetry data and converts to OTEL metrics +// processTelemetryData unmarshals the GPBKV payload and triggers OTLP conversion. func (s *grpcService) processTelemetryData(req *pb.MdtDialoutArgs) error { - ctx := context.Background() - startTime := time.Now() - - // Record message received metrics - nodeID := "unknown" - subscriptionID := fmt.Sprintf("%d", req.ReqId) - - if len(req.Data) == 0 { - s.receiver.telemetryBuilder.YangReceiverMessagesDropped.Add(ctx, 1, metric.WithAttributeSet(attribute.NewSet( - attribute.String("node_id", nodeID), - attribute.String("subscription_id", subscriptionID), - attribute.String("reason", "empty_data"), - ))) - return errors.New("empty telemetry data") - } - - s.receiver.telemetryBuilder.YangReceiverMessagesReceived.Add(ctx, int64(len(req.Data)), metric.WithAttributeSet(attribute.NewSet( - attribute.String("node_id", nodeID), - attribute.String("subscription_id", subscriptionID), - ))) - - // Parse the telemetry message from the data field telemetryMsg := &pb.Telemetry{} if err := proto.Unmarshal(req.Data, telemetryMsg); err != nil { - s.receiver.telemetryBuilder.YangReceiverMessagesDropped.Add(ctx, 1, metric.WithAttributeSet(attribute.NewSet( - attribute.String("node_id", nodeID), - attribute.String("subscription_id", subscriptionID), - attribute.String("reason", "unmarshal_error"), - ))) - return fmt.Errorf("failed to unmarshal telemetry data: %w", err) + return fmt.Errorf("failed to unmarshal telemetry payload: %w", err) } - // Update nodeID with actual value - if telemetryMsg.GetNodeIdStr() != "" { - nodeID = telemetryMsg.GetNodeIdStr() - } - - // Convert to OTEL metrics metrics := s.convertToOTELMetrics(telemetryMsg) - if metrics.MetricCount() == 0 { - s.receiver.telemetryBuilder.YangReceiverMessagesDropped.Add(ctx, 1, metric.WithAttributeSet(attribute.NewSet( - attribute.String("node_id", nodeID), - attribute.String("subscription_id", subscriptionID), - attribute.String("reason", "no_metrics_extracted"), - ))) - s.receiver.settings.Logger.Warn("No metrics extracted from telemetry data", - zap.String("encoding_path", telemetryMsg.EncodingPath), - zap.String("node_id", telemetryMsg.GetNodeIdStr())) - return nil - } - - // Send metrics to the consumer - err := s.receiver.consumer.ConsumeMetrics(ctx, metrics) - if err != nil { - s.receiver.telemetryBuilder.YangReceiverMessagesDropped.Add(ctx, 1, metric.WithAttributeSet(attribute.NewSet( - attribute.String("node_id", nodeID), - attribute.String("subscription_id", subscriptionID), - attribute.String("reason", "consumer_error"), - ))) - return fmt.Errorf("failed to consume metrics: %w", err) - } - - // Record successful processing with timing - processingDuration := time.Since(startTime) - yangModule := s.extractYANGModule(telemetryMsg.EncodingPath) - s.receiver.telemetryBuilder.YangReceiverMessagesProcessed.Add(ctx, 1, metric.WithAttributeSet(attribute.NewSet( - attribute.String("node_id", nodeID), - attribute.String("subscription_id", subscriptionID), - attribute.String("yang_module", yangModule), - ))) - s.receiver.telemetryBuilder.YangReceiverProcessingDuration.Record(ctx, float64(processingDuration.Nanoseconds())/1000000) - - s.receiver.settings.Logger.Debug("Successfully processed telemetry data", - zap.Int64("req_id", req.ReqId), - zap.String("encoding_path", telemetryMsg.EncodingPath), - zap.Int("metric_count", metrics.MetricCount()), - zap.Duration("processing_duration", processingDuration)) - - return nil -} - -// extractYANGModule extracts the YANG module name from encoding path -func (*grpcService) extractYANGModule(encodingPath string) string { - // Example: "Cisco-IOS-XE-interfaces-oper:interfaces/interface/statistics" - if before, _, ok := strings.Cut(encodingPath, ":"); ok { - return before - } - return "unknown" + return s.receiver.consumer.ConsumeMetrics(context.Background(), metrics) } -// convertToOTELMetrics converts Cisco telemetry data to OpenTelemetry metrics format +// convertToOTELMetrics maps Cisco KV-GPB data to OTLP using a Telegraf-inspired +// approach: extract identifiers (tags) first, then emit measurements. func (s *grpcService) convertToOTELMetrics(telemetry *pb.Telemetry) pmetric.Metrics { - // RFC YANG Parser analysis for this encoding path - if s.rfcYangParser != nil { - rfcAnalysis := s.rfcYangParser.AnalyzeTelemetryPath(telemetry.EncodingPath) - if rfcAnalysis != nil && rfcAnalysis.IsValid { - fmt.Printf("\n=== RFC YANG ANALYSIS ===\n") - fmt.Printf("Encoding Path: %s\n", telemetry.EncodingPath) - fmt.Printf("Module: %s\n", rfcAnalysis.ModuleName) - fmt.Printf("XPath: %s\n", rfcAnalysis.XPath) - fmt.Printf("List Path: %s\n", rfcAnalysis.ListPath) - fmt.Printf("Semantic Type: %s\n", rfcAnalysis.SemanticType) - fmt.Printf("List Keys: %v\n", rfcAnalysis.ListKeys) - fmt.Printf("=========================\n\n") - } - } - metrics := pmetric.NewMetrics() - resourceMetrics := metrics.ResourceMetrics().AppendEmpty() + rm := metrics.ResourceMetrics().AppendEmpty() - // Set resource attributes - resource := resourceMetrics.Resource() - resourceAttrs := resource.Attributes() + resAttrs := rm.Resource().Attributes() + resAttrs.PutStr("cisco.node_id", telemetry.GetNodeIdStr()) + resAttrs.PutStr("cisco.encoding_path", telemetry.EncodingPath) - if nodeID := telemetry.GetNodeIdStr(); nodeID != "" { - resourceAttrs.PutStr("cisco.node_id", nodeID) - } - if subscriptionID := telemetry.GetSubscriptionIdStr(); subscriptionID != "" { - resourceAttrs.PutStr("cisco.subscription_id", subscriptionID) - } - resourceAttrs.PutStr("cisco.encoding_path", telemetry.EncodingPath) + sm := rm.ScopeMetrics().AppendEmpty() + timestamp := pcommon.Timestamp(telemetry.MsgTimestamp * 1000000) - scopeMetrics := resourceMetrics.ScopeMetrics().AppendEmpty() - scope := scopeMetrics.Scope() - scope.SetName("cisco_telemetry_receiver") - scope.SetVersion("1.0.0") + // Process each entry in DataGpbkv as a distinct row/object. + for _, field := range telemetry.DataGpbkv { + // Step 1: Initialize context bag with global metadata. + ctxBag := map[string]string{ + "node_id": telemetry.GetNodeIdStr(), + } - // Process kvGPB data if present - if len(telemetry.DataGpbkv) > 0 { - s.processKvGPBData(scopeMetrics, telemetry) - } + // Step 2: Pre-scan the entire tree for keys/identifiers (Telegraf logic). + // This ensures sibling branches like 'admin-status' can access 'interface-name'. + s.extractKeys(field, ctxBag) - // Process GPB table data if present - if telemetry.DataGpb != nil { - s.processGPBTableData(scopeMetrics, telemetry) + // Step 3: Walk the tree again to emit actual metrics using the enriched context. + s.emitMetrics(sm, field, "", timestamp, ctxBag) } return metrics } -// processKvGPBData processes key-value GPB formatted telemetry data -func (s *grpcService) processKvGPBData(scopeMetrics pmetric.ScopeMetrics, telemetry *pb.Telemetry) { - timestamp := pcommon.Timestamp(telemetry.MsgTimestamp * 1000000) // Convert milliseconds to nanoseconds - - for _, field := range telemetry.DataGpbkv { - s.processField(scopeMetrics, field, telemetry.EncodingPath, "", timestamp) - } -} // processField recursively processes a telemetry field and creates metrics -func (s *grpcService) processField(scopeMetrics pmetric.ScopeMetrics, field *pb.TelemetryField, basePath, pathPrefix string, timestamp pcommon.Timestamp) { - currentPath := pathPrefix - if currentPath != "" { - currentPath += "." - } - currentPath += field.Name - - // YANG analysis is now done within the metric creation methods - // If field has a value, create a metric - if field.ValueByType != nil { - m := scopeMetrics.Metrics().AppendEmpty() - - // Get YANG data type information for this field - yangDataType := s.yangParser.GetDataTypeForEncodingPath(basePath, field.Name) - - switch value := field.ValueByType.(type) { - case *pb.TelemetryField_Uint32Value: - s.createYANGAwareMetric(m, currentPath, basePath, float64(value.Uint32Value), timestamp, yangDataType) - case *pb.TelemetryField_Uint64Value: - s.createYANGAwareMetric(m, currentPath, basePath, float64(value.Uint64Value), timestamp, yangDataType) - case *pb.TelemetryField_Sint32Value: - s.createYANGAwareMetric(m, currentPath, basePath, float64(value.Sint32Value), timestamp, yangDataType) - case *pb.TelemetryField_Sint64Value: - s.createYANGAwareMetric(m, currentPath, basePath, float64(value.Sint64Value), timestamp, yangDataType) - case *pb.TelemetryField_DoubleValue: - s.createYANGAwareMetric(m, currentPath, basePath, value.DoubleValue, timestamp, yangDataType) - case *pb.TelemetryField_FloatValue: - s.createYANGAwareMetric(m, currentPath, basePath, float64(value.FloatValue), timestamp, yangDataType) - case *pb.TelemetryField_BoolValue: - val := 0.0 - if value.BoolValue { - val = 1.0 +// extractKeys recursively scans for string values that serve as identifiers. +func (s *grpcService) extractKeys(field *pb.TelemetryField, ctxBag map[string]string) { + val := formatValueToString(field) + if val != "" { + // If it's a string value, it's likely a dimension/tag. + if _, ok := field.ValueByType.(*pb.TelemetryField_StringValue); ok { + ctxBag[field.Name] = val + + // Common Cisco naming normalization. + lowName := strings.ToLower(field.Name) + if lowName == "name" || lowName == "interface-name" { + ctxBag["interface"] = val } - s.createYANGAwareMetric(m, currentPath, basePath, val, timestamp, yangDataType) - case *pb.TelemetryField_StringValue: - // For string values, create YANG-aware info m - s.createYANGAwareInfoMetric(m, currentPath, basePath, value.StringValue, timestamp, yangDataType) - default: - // Remove the metric we added if we can't handle the type - scopeMetrics.Metrics().RemoveIf(func(m pmetric.Metric) bool { - return m.Name() == "" - }) } } - - // Process nested fields recursively - for _, nestedField := range field.Fields { - s.processField(scopeMetrics, nestedField, basePath, currentPath, timestamp) + for _, child := range field.Fields { + s.extractKeys(child, ctxBag) } } -// processGPBTableData processes GPB table formatted telemetry data -func (s *grpcService) processGPBTableData(_ pmetric.ScopeMetrics, telemetry *pb.Telemetry) { - // For GPB table data, we would need specific protobuf definitions for each encoding path - // This is a placeholder implementation - s.receiver.settings.Logger.Debug("GPB table data processing not implemented", - zap.String("encoding_path", telemetry.EncodingPath)) -} - -// isKeyField checks if a field name is a key field based on YANG analysis -func (*grpcService) isKeyField(fieldName string, analysis *internal.PathAnalysis) bool { - if analysis == nil { - return false +// emitMetrics processes numerical values and emits OTLP metrics with the full context bag. +func (s *grpcService) emitMetrics(sm pmetric.ScopeMetrics, field *pb.TelemetryField, pathPrefix string, timestamp pcommon.Timestamp, ctxBag map[string]string) { + currentPath := pathPrefix + if currentPath != "" { + currentPath += "." } + currentPath += field.Name + + // Only emit metrics for leaf nodes (values) that are NOT in the 'keys' branch. + if field.ValueByType != nil && len(field.Fields) == 0 && !strings.HasPrefix(currentPath, "keys") { + m := sm.Metrics().AppendEmpty() + cleanName := strings.TrimPrefix(currentPath, "content.") + metricLabels := cloneCtxBag(ctxBag) - // Check if this field matches any known key fields for the path - for _, keyField := range analysis.Keys { - if fieldName == keyField { - return true + if strVal, ok := field.ValueByType.(*pb.TelemetryField_StringValue); ok { + // Step/Info metrics for string states (e.g., Up/Down). + createStepMetric(m, cleanName, strVal.StringValue, timestamp, metricLabels) + } else { + // Numeric metrics for counters and gauges. + createNumericMetric(m, cleanName, getNumericValue(field), timestamp, nil, metricLabels) } } - // Additional heuristics for common key patterns - commonKeys := []string{"name", "id", "index", "interface-name", "neighbor-id", "router-id"} - return slices.Contains(commonKeys, fieldName) -} - -// extractFieldName extracts the field name from a metric name path -func (*grpcService) extractFieldName(metricName string) string { - // Remove common prefixes and get the last component - parts := strings.Split(metricName, ".") - if len(parts) == 0 { - return metricName + for _, child := range field.Fields { + s.emitMetrics(sm, child, currentPath, timestamp, ctxBag) } - - // Get the last part, which is usually the field name - fieldName := parts[len(parts)-1] - - // Remove common suffixes like "_info" - fieldName = strings.TrimSuffix(fieldName, "_info") - - return fieldName } -// createYANGAwareMetric creates a metric with YANG data type awareness -func (s *grpcService) createYANGAwareMetric(metric pmetric.Metric, name, encodingPath string, value float64, timestamp pcommon.Timestamp, yangDataType *internal.YANGDataType) { - // Determine metric name and type based on YANG information - metricName := fmt.Sprintf("cisco.%s", name) - metricDescription := fmt.Sprintf("Cisco telemetry metric from %s", encodingPath) - metricUnit := "1" // Default unit - - if yangDataType != nil { - // Use YANG-provided description and units - if yangDataType.Description != "" { - metricDescription = yangDataType.Description - } - if yangDataType.Units != "" { - metricUnit = yangDataType.Units - } - - // Add YANG type to metric name for clarity - if yangDataType.Type != "" { - metricName = fmt.Sprintf("cisco.%s", name) - } - } - - metric.SetName(metricName) - metric.SetDescription(metricDescription) - metric.SetUnit(metricUnit) - - // Determine if this should be a counter or gauge based on YANG data type - if yangDataType != nil && yangDataType.IsCounterType() { - // Create a sum metric for counters (monotonic increasing) - sum := metric.SetEmptySum() +// createNumericMetric populates a NumberDataPoint. +func createNumericMetric(m pmetric.Metric, name string, val float64, ts pcommon.Timestamp, yType *internal.YANGDataType, ctx map[string]string) { + m.SetName("cisco." + name) + if yType != nil && yType.IsCounterType() { + sum := m.SetEmptySum() sum.SetIsMonotonic(true) sum.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) - dp := sum.DataPoints().AppendEmpty() - dp.SetDoubleValue(value) - dp.SetTimestamp(timestamp) - - // Add attributes - s.addYANGAttributes(dp.Attributes(), encodingPath, yangDataType, name) + dp.SetDoubleValue(val) + dp.SetTimestamp(ts) + applyCtxBag(dp.Attributes(), ctx) } else { - // Create a gauge metric for everything else - gauge := metric.SetEmptyGauge() - dp := gauge.DataPoints().AppendEmpty() - dp.SetDoubleValue(value) - dp.SetTimestamp(timestamp) - - // Add attributes - s.addYANGAttributes(dp.Attributes(), encodingPath, yangDataType, name) + dp := m.SetEmptyGauge().DataPoints().AppendEmpty() + dp.SetDoubleValue(val) + dp.SetTimestamp(ts) + applyCtxBag(dp.Attributes(), ctx) } } -// createYANGAwareInfoMetric creates an info metric with YANG data type awareness -func (s *grpcService) createYANGAwareInfoMetric(metric pmetric.Metric, name, encodingPath, value string, timestamp pcommon.Timestamp, yangDataType *internal.YANGDataType) { - metricName := fmt.Sprintf("cisco.%s_info", name) - metricDescription := fmt.Sprintf("Cisco telemetry info metric from %s", encodingPath) - - if yangDataType != nil && yangDataType.Description != "" { - metricDescription = yangDataType.Description - } - - metric.SetName(metricName) - metric.SetDescription(metricDescription) - metric.SetUnit("1") - - gauge := metric.SetEmptyGauge() - dp := gauge.DataPoints().AppendEmpty() - dp.SetDoubleValue(1.0) // Info metrics always have value 1 - dp.SetTimestamp(timestamp) - - // Add the string value as an attribute - dp.Attributes().PutStr("value", value) - - // Add YANG attributes - s.addYANGAttributes(dp.Attributes(), encodingPath, yangDataType, name) +// createStepMetric creates an "Info" metric where the actual value is an attribute. +func createStepMetric(m pmetric.Metric, name, val string, ts pcommon.Timestamp, ctx map[string]string) { + m.SetName("cisco." + name + "_info") + dp := m.SetEmptyGauge().DataPoints().AppendEmpty() + dp.SetDoubleValue(1.0) + dp.SetTimestamp(ts) + dp.Attributes().PutStr("value", val) + applyCtxBag(dp.Attributes(), ctx) } -// addYANGAttributes adds YANG-derived attributes to metric data points -func (s *grpcService) addYANGAttributes(attrs pcommon.Map, encodingPath string, yangDataType *internal.YANGDataType, fieldName string) { - // Always add encoding path - attrs.PutStr("encoding_path", encodingPath) - - // Use RFC-compliant YANG parser for enhanced analysis - rfcAnalysis := s.rfcYangParser.AnalyzeTelemetryPath(encodingPath) - if rfcAnalysis != nil && rfcAnalysis.IsValid { - if rfcAnalysis.ModuleName != "" { - attrs.PutStr("yang.module", rfcAnalysis.ModuleName) - } - if rfcAnalysis.ListPath != "" { - attrs.PutStr("yang.list_path", rfcAnalysis.ListPath) - } - if rfcAnalysis.SemanticType != "" { - attrs.PutStr("yang.semantic_type", rfcAnalysis.SemanticType) - } - - // Add list key information - if len(rfcAnalysis.ListKeys) > 0 { - // Check if current field is a key field - for _, key := range rfcAnalysis.ListKeys { - if strings.Contains(fieldName, key) { - attrs.PutStr("yang.is_key", "true") - attrs.PutStr("yang.key_type", key) - break - } - } - } - - // Add inferred data type from RFC parser - if len(rfcAnalysis.PathSegments) > 0 { - leafName := rfcAnalysis.PathSegments[len(rfcAnalysis.PathSegments)-1] - inferredType := s.rfcYangParser.InferDataTypeFromPath(leafName) - if inferredType != nil && inferredType.Name != "" { - attrs.PutStr("yang.data_type", inferredType.Name) - } - } - } - - // Fallback to basic YANG parser analysis if RFC parser fails - if rfcAnalysis == nil || !rfcAnalysis.IsValid { - analysis := s.yangParser.AnalyzeEncodingPath(encodingPath) - if analysis != nil { - if analysis.ModuleName != "" { - attrs.PutStr("yang.module", analysis.ModuleName) - } - if analysis.ListPath != "" { - attrs.PutStr("yang.list_path", analysis.ListPath) - } - - // Check if this is a key field - if s.isKeyField(fieldName, analysis) { - attrs.PutStr("yang.is_key", "true") - attrs.PutStr("yang.key_type", fieldName) - } - } - } +// getNumericValue extracts float64 from various protobuf types. +func getNumericValue(f *pb.TelemetryField) float64 { + switch v := f.ValueByType.(type) { + case *pb.TelemetryField_Uint32Value: + return float64(v.Uint32Value) + case *pb.TelemetryField_Uint64Value: + return float64(v.Uint64Value) + case *pb.TelemetryField_Sint32Value: + return float64(v.Sint32Value) + case *pb.TelemetryField_Sint64Value: + return float64(v.Sint64Value) + case *pb.TelemetryField_DoubleValue: + return v.DoubleValue + case *pb.TelemetryField_FloatValue: + return float64(v.FloatValue) + case *pb.TelemetryField_BoolValue: + if v.BoolValue { + return 1.0 + } + return 0.0 + } + return 0.0 +} - // Add YANG data type information from basic parser - if yangDataType != nil { - if yangDataType.Type != "" { - attrs.PutStr("yang.data_type", yangDataType.Type) - } - if yangDataType.Units != "" { - attrs.PutStr("yang.units", yangDataType.Units) - } - if yangDataType.Description != "" { - attrs.PutStr("yang.description", yangDataType.Description) - } +// formatValueToString converts protobuf field values to string for labels. +func formatValueToString(f *pb.TelemetryField) string { + if f.ValueByType == nil { + return "" + } + switch v := f.ValueByType.(type) { + case *pb.TelemetryField_StringValue: + return v.StringValue + case *pb.TelemetryField_Uint32Value: + return strconv.FormatUint(uint64(v.Uint32Value), 10) + case *pb.TelemetryField_Uint64Value: + return strconv.FormatUint(v.Uint64Value, 10) + case *pb.TelemetryField_Sint32Value: + return strconv.FormatInt(int64(v.Sint32Value), 10) + case *pb.TelemetryField_Sint64Value: + return strconv.FormatInt(v.Sint64Value, 10) + } + return "" +} - // Add semantic information - if yangDataType.IsCounterType() { - attrs.PutStr("yang.semantic_type", "counter") - } else if yangDataType.IsGaugeType() { - attrs.PutStr("yang.semantic_type", "gauge") - } +func cloneCtxBag(in map[string]string) map[string]string { + out := make(map[string]string, len(in)) + maps.Copy(in, out) + return out +} - // Add range information if available - if yangDataType.Range != nil { - if yangDataType.Range.Min != nil { - attrs.PutInt("yang.min_value", *yangDataType.Range.Min) - } - if yangDataType.Range.Max != nil { - attrs.PutInt("yang.max_value", *yangDataType.Range.Max) - } +func applyCtxBag(attrs pcommon.Map, ctx map[string]string) { + for k, v := range ctx { + if v != "" { + attrs.PutStr(k, v) } } } diff --git a/receiver/yanggrpcreceiver/grpc_service_coverage_test.go b/receiver/yanggrpcreceiver/grpc_service_coverage_test.go index 139935800221f..cc39673436a28 100644 --- a/receiver/yanggrpcreceiver/grpc_service_coverage_test.go +++ b/receiver/yanggrpcreceiver/grpc_service_coverage_test.go @@ -21,8 +21,7 @@ func TestMdtDialout_BasicFlow(t *testing.T) { consumer := &consumertest.MetricsSink{} settings := createTestSettings() - receiver, err := createMetricsReceiver(t.Context(), settings, config, consumer) - require.NoError(t, err) + receiver := createMetricsReceiver(t.Context(), settings, config, consumer) yangParser := internal.NewYANGParser() yangParser.LoadBuiltinModules() @@ -44,8 +43,7 @@ func TestProcessTelemetryData_ErrorHandling(t *testing.T) { consumer := &consumertest.MetricsSink{} settings := createTestSettings() - receiver, err := createMetricsReceiver(t.Context(), settings, config, consumer) - require.NoError(t, err) + receiver := createMetricsReceiver(t.Context(), settings, config, consumer) yangParser := internal.NewYANGParser() yangParser.LoadBuiltinModules() @@ -60,9 +58,8 @@ func TestProcessTelemetryData_ErrorHandling(t *testing.T) { ReqId: 12345, Data: []byte{}, // Empty data } - err = service.processTelemetryData(emptyMsg) - assert.Error(t, err) - assert.Contains(t, err.Error(), "empty telemetry data") + err := service.processTelemetryData(emptyMsg) + assert.NoError(t, err) // Test with invalid protobuf data invalidMsg := &pb.MdtDialoutArgs{ @@ -71,7 +68,7 @@ func TestProcessTelemetryData_ErrorHandling(t *testing.T) { } err = service.processTelemetryData(invalidMsg) assert.Error(t, err) - assert.Contains(t, err.Error(), "failed to unmarshal telemetry data") + assert.Contains(t, err.Error(), "cannot parse invalid wire-format data") // Test with valid telemetry data (simple case) validTelemetry := &pb.Telemetry{ @@ -90,51 +87,12 @@ func TestProcessTelemetryData_ErrorHandling(t *testing.T) { assert.NoError(t, err) } -// Add more coverage for helper methods that show 0% coverage -func TestGrpcServiceHelpers(t *testing.T) { - config := createValidTestConfig() - consumer := &consumertest.MetricsSink{} - settings := createTestSettings() - - receiver, err := createMetricsReceiver(t.Context(), settings, config, consumer) - require.NoError(t, err) - - yangParser := internal.NewYANGParser() - yangParser.LoadBuiltinModules() - - service := &grpcService{ - receiver: receiver.(*yangReceiver), - yangParser: yangParser, - } - - // Test extractFieldName method (currently 0% coverage) - fieldName := service.extractFieldName("test.path.field-name") - assert.Equal(t, "field-name", fieldName) - - fieldName = service.extractFieldName("simple-field") - assert.Equal(t, "simple-field", fieldName) - - fieldName = service.extractFieldName("field_info") - assert.Equal(t, "field", fieldName) // Should remove _info suffix - - // Test extractYANGModule method - module := service.extractYANGModule("Cisco-IOS-XE-interfaces-oper:interfaces") - assert.Equal(t, "Cisco-IOS-XE-interfaces-oper", module) - - module = service.extractYANGModule("simple-path") - assert.Equal(t, "unknown", module) // Returns "unknown" if no colon found - - module = service.extractYANGModule("") - assert.Equal(t, "unknown", module) -} - func TestConvertToOTELMetrics(t *testing.T) { config := createValidTestConfig() consumer := &consumertest.MetricsSink{} settings := createTestSettings() - receiver, err := createMetricsReceiver(t.Context(), settings, config, consumer) - require.NoError(t, err) + receiver := createMetricsReceiver(t.Context(), settings, config, consumer) yangParser := internal.NewYANGParser() yangParser.LoadBuiltinModules() diff --git a/receiver/yanggrpcreceiver/grpc_service_test.go b/receiver/yanggrpcreceiver/grpc_service_test.go index c2602d26639e6..26dee06941395 100644 --- a/receiver/yanggrpcreceiver/grpc_service_test.go +++ b/receiver/yanggrpcreceiver/grpc_service_test.go @@ -28,19 +28,14 @@ func TestGrpcService_ProcessTelemetryData(t *testing.T) { mockConsumer := &consumertest.MetricsSink{} settings := createTestSettings() - ctr, err := createMetricsReceiver(t.Context(), settings, config, mockConsumer) - if err != nil { - t.Fatalf("Failed to create receiver: %v", err) - } + ctr := createMetricsReceiver(t.Context(), settings, config, mockConsumer) // Create gRPC service with YANG parser yangParser := internal.NewYANGParser() yangParser.LoadBuiltinModules() - rfcYangParser := internal.NewRFC6020Parser() service := &grpcService{ - receiver: ctr.(*yangReceiver), - yangParser: yangParser, - rfcYangParser: rfcYangParser, + receiver: ctr.(*yangReceiver), + yangParser: yangParser, } // Create test telemetry data @@ -199,10 +194,7 @@ func TestKvGPBDataParsing(t *testing.T) { }, } - ctr, err := createMetricsReceiver(t.Context(), settings, config, mockConsumer) - if err != nil { - t.Fatalf("Failed to create receiver: %v", err) - } + ctr := createMetricsReceiver(t.Context(), settings, config, mockConsumer) yangParser := internal.NewYANGParser() yangParser.LoadBuiltinModules() diff --git a/receiver/yanggrpcreceiver/internal/metadata/generated_telemetry.go b/receiver/yanggrpcreceiver/internal/metadata/generated_telemetry.go deleted file mode 100644 index 2eddb9d6d43e4..0000000000000 --- a/receiver/yanggrpcreceiver/internal/metadata/generated_telemetry.go +++ /dev/null @@ -1,124 +0,0 @@ -// Code generated by mdatagen. DO NOT EDIT. - -package metadata - -import ( - "errors" - "sync" - - "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/otel/metric" - "go.opentelemetry.io/otel/trace" -) - -func Meter(settings component.TelemetrySettings) metric.Meter { - return settings.MeterProvider.Meter("github.com/open-telemetry/opentelemetry-collector-contrib/receiver/yanggrpcreceiver") -} - -func Tracer(settings component.TelemetrySettings) trace.Tracer { - return settings.TracerProvider.Tracer("github.com/open-telemetry/opentelemetry-collector-contrib/receiver/yanggrpcreceiver") -} - -// TelemetryBuilder provides an interface for components to report telemetry -// as defined in metadata and user config. -type TelemetryBuilder struct { - meter metric.Meter - mu sync.Mutex - registrations []metric.Registration - YangReceiverBytesReceived metric.Int64Counter - YangReceiverConnectionsClosed metric.Int64UpDownCounter - YangReceiverConnectionsOpened metric.Int64UpDownCounter - YangReceiverGrpcErrors metric.Int64Counter - YangReceiverMessagesDropped metric.Int64Counter - YangReceiverMessagesProcessed metric.Int64Counter - YangReceiverMessagesReceived metric.Int64Counter - YangReceiverProcessingDuration metric.Float64Histogram - YangReceiverYangModulesDiscovered metric.Int64UpDownCounter -} - -// TelemetryBuilderOption applies changes to default builder. -type TelemetryBuilderOption interface { - apply(*TelemetryBuilder) -} - -type telemetryBuilderOptionFunc func(mb *TelemetryBuilder) - -func (tbof telemetryBuilderOptionFunc) apply(mb *TelemetryBuilder) { - tbof(mb) -} - -// Shutdown unregister all registered callbacks for async instruments. -func (builder *TelemetryBuilder) Shutdown() { - builder.mu.Lock() - defer builder.mu.Unlock() - for _, reg := range builder.registrations { - reg.Unregister() - } -} - -// NewTelemetryBuilder provides a struct with methods to update all internal telemetry -// for a component -func NewTelemetryBuilder(settings component.TelemetrySettings, options ...TelemetryBuilderOption) (*TelemetryBuilder, error) { - builder := TelemetryBuilder{} - for _, op := range options { - op.apply(&builder) - } - builder.meter = Meter(settings) - var err, errs error - builder.YangReceiverBytesReceived, err = builder.meter.Int64Counter( - "otelcol_yang_receiver_bytes_received", - metric.WithDescription("Total bytes received from telemetry connections [Development]"), - metric.WithUnit("By"), - ) - errs = errors.Join(errs, err) - builder.YangReceiverConnectionsClosed, err = builder.meter.Int64UpDownCounter( - "otelcol_yang_receiver_connections_closed", - metric.WithDescription("Number of gRPC connections closed [Development]"), - metric.WithUnit("{connections}"), - ) - errs = errors.Join(errs, err) - builder.YangReceiverConnectionsOpened, err = builder.meter.Int64UpDownCounter( - "otelcol_yang_receiver_connections_opened", - metric.WithDescription("Number of gRPC connections opened [Development]"), - metric.WithUnit("{connections}"), - ) - errs = errors.Join(errs, err) - builder.YangReceiverGrpcErrors, err = builder.meter.Int64Counter( - "otelcol_yang_receiver_grpc_errors", - metric.WithDescription("Number of gRPC errors encountered [Development]"), - metric.WithUnit("{errors}"), - ) - errs = errors.Join(errs, err) - builder.YangReceiverMessagesDropped, err = builder.meter.Int64Counter( - "otelcol_yang_receiver_messages_dropped", - metric.WithDescription("Number of telemetry messages dropped due to errors [Development]"), - metric.WithUnit("{messages}"), - ) - errs = errors.Join(errs, err) - builder.YangReceiverMessagesProcessed, err = builder.meter.Int64Counter( - "otelcol_yang_receiver_messages_processed", - metric.WithDescription("Number of telemetry messages successfully processed [Development]"), - metric.WithUnit("{messages}"), - ) - errs = errors.Join(errs, err) - builder.YangReceiverMessagesReceived, err = builder.meter.Int64Counter( - "otelcol_yang_receiver_messages_received", - metric.WithDescription("Number of telemetry messages received [Development]"), - metric.WithUnit("{messages}"), - ) - errs = errors.Join(errs, err) - builder.YangReceiverProcessingDuration, err = builder.meter.Float64Histogram( - "otelcol_yang_receiver_processing_duration", - metric.WithDescription("Time spent processing telemetry messages [Development]"), - metric.WithUnit("ms"), - metric.WithExplicitBucketBoundaries([]float64{0.1, 0.5, 1, 5, 10, 50, 100, 500, 1000}...), - ) - errs = errors.Join(errs, err) - builder.YangReceiverYangModulesDiscovered, err = builder.meter.Int64UpDownCounter( - "otelcol_yang_receiver_yang_modules_discovered", - metric.WithDescription("Number of unique YANG modules discovered [Development]"), - metric.WithUnit("{errors}"), - ) - errs = errors.Join(errs, err) - return &builder, errs -} diff --git a/receiver/yanggrpcreceiver/internal/metadata/generated_telemetry_test.go b/receiver/yanggrpcreceiver/internal/metadata/generated_telemetry_test.go deleted file mode 100644 index 84f4406e62bfd..0000000000000 --- a/receiver/yanggrpcreceiver/internal/metadata/generated_telemetry_test.go +++ /dev/null @@ -1,73 +0,0 @@ -// Code generated by mdatagen. DO NOT EDIT. - -package metadata - -import ( - "testing" - - "github.com/stretchr/testify/require" - "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/component/componenttest" - "go.opentelemetry.io/otel/metric" - embeddedmetric "go.opentelemetry.io/otel/metric/embedded" - noopmetric "go.opentelemetry.io/otel/metric/noop" - "go.opentelemetry.io/otel/trace" - embeddedtrace "go.opentelemetry.io/otel/trace/embedded" - nooptrace "go.opentelemetry.io/otel/trace/noop" -) - -type mockMeter struct { - noopmetric.Meter - name string -} -type mockMeterProvider struct { - embeddedmetric.MeterProvider -} - -func (m mockMeterProvider) Meter(name string, opts ...metric.MeterOption) metric.Meter { - return mockMeter{name: name} -} - -type mockTracer struct { - nooptrace.Tracer - name string -} - -type mockTracerProvider struct { - embeddedtrace.TracerProvider -} - -func (m mockTracerProvider) Tracer(name string, opts ...trace.TracerOption) trace.Tracer { - return mockTracer{name: name} -} - -func TestProviders(t *testing.T) { - set := component.TelemetrySettings{ - MeterProvider: mockMeterProvider{}, - TracerProvider: mockTracerProvider{}, - } - - meter := Meter(set) - if m, ok := meter.(mockMeter); ok { - require.Equal(t, "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/yanggrpcreceiver", m.name) - } else { - require.Fail(t, "returned Meter not mockMeter") - } - - tracer := Tracer(set) - if m, ok := tracer.(mockTracer); ok { - require.Equal(t, "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/yanggrpcreceiver", m.name) - } else { - require.Fail(t, "returned Meter not mockTracer") - } -} - -func TestNewTelemetryBuilder(t *testing.T) { - set := componenttest.NewNopTelemetrySettings() - applied := false - _, err := NewTelemetryBuilder(set, telemetryBuilderOptionFunc(func(b *TelemetryBuilder) { - applied = true - })) - require.NoError(t, err) - require.True(t, applied) -} diff --git a/receiver/yanggrpcreceiver/internal/metadatatest/generated_telemetrytest.go b/receiver/yanggrpcreceiver/internal/metadatatest/generated_telemetrytest.go deleted file mode 100644 index c5fef8546396b..0000000000000 --- a/receiver/yanggrpcreceiver/internal/metadatatest/generated_telemetrytest.go +++ /dev/null @@ -1,165 +0,0 @@ -// Code generated by mdatagen. DO NOT EDIT. - -package metadatatest - -import ( - "testing" - - "github.com/stretchr/testify/require" - "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/component/componenttest" - "go.opentelemetry.io/collector/receiver" - "go.opentelemetry.io/collector/receiver/receivertest" - "go.opentelemetry.io/otel/sdk/metric/metricdata" - "go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest" -) - -func NewSettings(tt *componenttest.Telemetry) receiver.Settings { - set := receivertest.NewNopSettings(receivertest.NopType) - set.ID = component.NewID(component.MustNewType("yang_grpc")) - set.TelemetrySettings = tt.NewTelemetrySettings() - return set -} - -func AssertEqualYangReceiverBytesReceived(t *testing.T, tt *componenttest.Telemetry, dps []metricdata.DataPoint[int64], opts ...metricdatatest.Option) { - want := metricdata.Metrics{ - Name: "otelcol_yang_receiver_bytes_received", - Description: "Total bytes received from telemetry connections [Development]", - Unit: "By", - Data: metricdata.Sum[int64]{ - Temporality: metricdata.CumulativeTemporality, - IsMonotonic: true, - DataPoints: dps, - }, - } - got, err := tt.GetMetric("otelcol_yang_receiver_bytes_received") - require.NoError(t, err) - metricdatatest.AssertEqual(t, want, got, opts...) -} - -func AssertEqualYangReceiverConnectionsClosed(t *testing.T, tt *componenttest.Telemetry, dps []metricdata.DataPoint[int64], opts ...metricdatatest.Option) { - want := metricdata.Metrics{ - Name: "otelcol_yang_receiver_connections_closed", - Description: "Number of gRPC connections closed [Development]", - Unit: "{connections}", - Data: metricdata.Sum[int64]{ - Temporality: metricdata.CumulativeTemporality, - IsMonotonic: false, - DataPoints: dps, - }, - } - got, err := tt.GetMetric("otelcol_yang_receiver_connections_closed") - require.NoError(t, err) - metricdatatest.AssertEqual(t, want, got, opts...) -} - -func AssertEqualYangReceiverConnectionsOpened(t *testing.T, tt *componenttest.Telemetry, dps []metricdata.DataPoint[int64], opts ...metricdatatest.Option) { - want := metricdata.Metrics{ - Name: "otelcol_yang_receiver_connections_opened", - Description: "Number of gRPC connections opened [Development]", - Unit: "{connections}", - Data: metricdata.Sum[int64]{ - Temporality: metricdata.CumulativeTemporality, - IsMonotonic: false, - DataPoints: dps, - }, - } - got, err := tt.GetMetric("otelcol_yang_receiver_connections_opened") - require.NoError(t, err) - metricdatatest.AssertEqual(t, want, got, opts...) -} - -func AssertEqualYangReceiverGrpcErrors(t *testing.T, tt *componenttest.Telemetry, dps []metricdata.DataPoint[int64], opts ...metricdatatest.Option) { - want := metricdata.Metrics{ - Name: "otelcol_yang_receiver_grpc_errors", - Description: "Number of gRPC errors encountered [Development]", - Unit: "{errors}", - Data: metricdata.Sum[int64]{ - Temporality: metricdata.CumulativeTemporality, - IsMonotonic: true, - DataPoints: dps, - }, - } - got, err := tt.GetMetric("otelcol_yang_receiver_grpc_errors") - require.NoError(t, err) - metricdatatest.AssertEqual(t, want, got, opts...) -} - -func AssertEqualYangReceiverMessagesDropped(t *testing.T, tt *componenttest.Telemetry, dps []metricdata.DataPoint[int64], opts ...metricdatatest.Option) { - want := metricdata.Metrics{ - Name: "otelcol_yang_receiver_messages_dropped", - Description: "Number of telemetry messages dropped due to errors [Development]", - Unit: "{messages}", - Data: metricdata.Sum[int64]{ - Temporality: metricdata.CumulativeTemporality, - IsMonotonic: true, - DataPoints: dps, - }, - } - got, err := tt.GetMetric("otelcol_yang_receiver_messages_dropped") - require.NoError(t, err) - metricdatatest.AssertEqual(t, want, got, opts...) -} - -func AssertEqualYangReceiverMessagesProcessed(t *testing.T, tt *componenttest.Telemetry, dps []metricdata.DataPoint[int64], opts ...metricdatatest.Option) { - want := metricdata.Metrics{ - Name: "otelcol_yang_receiver_messages_processed", - Description: "Number of telemetry messages successfully processed [Development]", - Unit: "{messages}", - Data: metricdata.Sum[int64]{ - Temporality: metricdata.CumulativeTemporality, - IsMonotonic: true, - DataPoints: dps, - }, - } - got, err := tt.GetMetric("otelcol_yang_receiver_messages_processed") - require.NoError(t, err) - metricdatatest.AssertEqual(t, want, got, opts...) -} - -func AssertEqualYangReceiverMessagesReceived(t *testing.T, tt *componenttest.Telemetry, dps []metricdata.DataPoint[int64], opts ...metricdatatest.Option) { - want := metricdata.Metrics{ - Name: "otelcol_yang_receiver_messages_received", - Description: "Number of telemetry messages received [Development]", - Unit: "{messages}", - Data: metricdata.Sum[int64]{ - Temporality: metricdata.CumulativeTemporality, - IsMonotonic: true, - DataPoints: dps, - }, - } - got, err := tt.GetMetric("otelcol_yang_receiver_messages_received") - require.NoError(t, err) - metricdatatest.AssertEqual(t, want, got, opts...) -} - -func AssertEqualYangReceiverProcessingDuration(t *testing.T, tt *componenttest.Telemetry, dps []metricdata.HistogramDataPoint[float64], opts ...metricdatatest.Option) { - want := metricdata.Metrics{ - Name: "otelcol_yang_receiver_processing_duration", - Description: "Time spent processing telemetry messages [Development]", - Unit: "ms", - Data: metricdata.Histogram[float64]{ - Temporality: metricdata.CumulativeTemporality, - DataPoints: dps, - }, - } - got, err := tt.GetMetric("otelcol_yang_receiver_processing_duration") - require.NoError(t, err) - metricdatatest.AssertEqual(t, want, got, opts...) -} - -func AssertEqualYangReceiverYangModulesDiscovered(t *testing.T, tt *componenttest.Telemetry, dps []metricdata.DataPoint[int64], opts ...metricdatatest.Option) { - want := metricdata.Metrics{ - Name: "otelcol_yang_receiver_yang_modules_discovered", - Description: "Number of unique YANG modules discovered [Development]", - Unit: "{errors}", - Data: metricdata.Sum[int64]{ - Temporality: metricdata.CumulativeTemporality, - IsMonotonic: false, - DataPoints: dps, - }, - } - got, err := tt.GetMetric("otelcol_yang_receiver_yang_modules_discovered") - require.NoError(t, err) - metricdatatest.AssertEqual(t, want, got, opts...) -} diff --git a/receiver/yanggrpcreceiver/internal/metadatatest/generated_telemetrytest_test.go b/receiver/yanggrpcreceiver/internal/metadatatest/generated_telemetrytest_test.go deleted file mode 100644 index 60339b33ca9f3..0000000000000 --- a/receiver/yanggrpcreceiver/internal/metadatatest/generated_telemetrytest_test.go +++ /dev/null @@ -1,60 +0,0 @@ -// Code generated by mdatagen. DO NOT EDIT. - -package metadatatest - -import ( - "context" - "testing" - - "github.com/stretchr/testify/require" - "go.opentelemetry.io/collector/component/componenttest" - "go.opentelemetry.io/otel/sdk/metric/metricdata" - "go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest" - - "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/yanggrpcreceiver/internal/metadata" -) - -func TestSetupTelemetry(t *testing.T) { - testTel := componenttest.NewTelemetry() - tb, err := metadata.NewTelemetryBuilder(testTel.NewTelemetrySettings()) - require.NoError(t, err) - defer tb.Shutdown() - tb.YangReceiverBytesReceived.Add(context.Background(), 1) - tb.YangReceiverConnectionsClosed.Add(context.Background(), 1) - tb.YangReceiverConnectionsOpened.Add(context.Background(), 1) - tb.YangReceiverGrpcErrors.Add(context.Background(), 1) - tb.YangReceiverMessagesDropped.Add(context.Background(), 1) - tb.YangReceiverMessagesProcessed.Add(context.Background(), 1) - tb.YangReceiverMessagesReceived.Add(context.Background(), 1) - tb.YangReceiverProcessingDuration.Record(context.Background(), 1) - tb.YangReceiverYangModulesDiscovered.Add(context.Background(), 1) - AssertEqualYangReceiverBytesReceived(t, testTel, - []metricdata.DataPoint[int64]{{Value: 1}}, - metricdatatest.IgnoreTimestamp()) - AssertEqualYangReceiverConnectionsClosed(t, testTel, - []metricdata.DataPoint[int64]{{Value: 1}}, - metricdatatest.IgnoreTimestamp()) - AssertEqualYangReceiverConnectionsOpened(t, testTel, - []metricdata.DataPoint[int64]{{Value: 1}}, - metricdatatest.IgnoreTimestamp()) - AssertEqualYangReceiverGrpcErrors(t, testTel, - []metricdata.DataPoint[int64]{{Value: 1}}, - metricdatatest.IgnoreTimestamp()) - AssertEqualYangReceiverMessagesDropped(t, testTel, - []metricdata.DataPoint[int64]{{Value: 1}}, - metricdatatest.IgnoreTimestamp()) - AssertEqualYangReceiverMessagesProcessed(t, testTel, - []metricdata.DataPoint[int64]{{Value: 1}}, - metricdatatest.IgnoreTimestamp()) - AssertEqualYangReceiverMessagesReceived(t, testTel, - []metricdata.DataPoint[int64]{{Value: 1}}, - metricdatatest.IgnoreTimestamp()) - AssertEqualYangReceiverProcessingDuration(t, testTel, - []metricdata.HistogramDataPoint[float64]{{}}, metricdatatest.IgnoreValue(), - metricdatatest.IgnoreTimestamp()) - AssertEqualYangReceiverYangModulesDiscovered(t, testTel, - []metricdata.DataPoint[int64]{{Value: 1}}, - metricdatatest.IgnoreTimestamp()) - - require.NoError(t, testTel.Shutdown(context.Background())) -} diff --git a/receiver/yanggrpcreceiver/internal/rfc_yang_parser.go b/receiver/yanggrpcreceiver/internal/rfc_yang_parser.go deleted file mode 100644 index 3348979a3519f..0000000000000 --- a/receiver/yanggrpcreceiver/internal/rfc_yang_parser.go +++ /dev/null @@ -1,1515 +0,0 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 - -package internal // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/yanggrpcreceiver/internal" - -import ( - "encoding/json" - "fmt" - "log" - "os" - "regexp" - "sort" - "strconv" - "strings" -) - -// RFC6020Parser implements RFC 6020 (YANG 1.0) and RFC 7950 (YANG 1.1) compliant YANG parsing -type RFC6020Parser struct { - modules map[string]*RFC6020Module - builtinTypes map[string]*RFC6020BuiltinType - typeRestrictions map[string]*RFC6020TypeRestriction - logger *log.Logger -} - -// RFC6020Module represents a complete YANG module based on RFC specifications -type RFC6020Module struct { - // Module header statements (RFC 7.1) - Name string `json:"name"` - Namespace string `json:"namespace"` - Prefix string `json:"prefix"` - YangVersion string `json:"yang_version"` // "1" or "1.1" - Organization string `json:"organization"` - Contact string `json:"contact"` - Description string `json:"description"` - Reference string `json:"reference"` - - // Import/Include statements (RFC 7.1.5, 7.1.6) - Imports map[string]*RFC6020Import `json:"imports"` - Includes map[string]*RFC6020Include `json:"includes"` - - // Revision history (RFC 7.1.9) - Revisions []*RFC6020Revision `json:"revisions"` - - // Type definitions (RFC 7.3) - Typedefs map[string]*RFC6020Typedef `json:"typedefs"` - - // Groupings (RFC 7.12) - Groupings map[string]*RFC6020Grouping `json:"groupings"` - - // Features (RFC 7.20.1) - Features map[string]*RFC6020Feature `json:"features"` - - // Data nodes (RFC 4.2.2) - DataNodes map[string]*RFC6020DataNode `json:"data_nodes"` - - // Semantic analysis results - KeyedPaths map[string]string `json:"keyed_paths"` // path -> primary key - ListKeys map[string][]string `json:"list_keys"` // list path -> all keys - DataTypes map[string]*RFC6020ResolvedType `json:"data_types"` // field path -> resolved type - Counters []string `json:"counters"` // paths that are counter semantics - Gauges []string `json:"gauges"` // paths that are gauge semantics - ConfigPaths []string `json:"config_paths"` // configuration data paths - StatePaths []string `json:"state_paths"` // state data paths -} - -// RFC6020Import represents an import statement (RFC 7.1.5) -type RFC6020Import struct { - Module string `json:"module"` - Prefix string `json:"prefix"` - RevisionDate string `json:"revision_date,omitempty"` - Description string `json:"description,omitempty"` - Reference string `json:"reference,omitempty"` -} - -// RFC6020Include represents an include statement (RFC 7.1.6) -type RFC6020Include struct { - Submodule string `json:"submodule"` - RevisionDate string `json:"revision_date,omitempty"` - Description string `json:"description,omitempty"` - Reference string `json:"reference,omitempty"` -} - -// RFC6020Revision represents a revision statement (RFC 7.1.9) -type RFC6020Revision struct { - Date string `json:"date"` - Description string `json:"description,omitempty"` - Reference string `json:"reference,omitempty"` -} - -// RFC6020Typedef represents a typedef statement (RFC 7.3) -type RFC6020Typedef struct { - Name string `json:"name"` - Type *RFC6020Type `json:"type"` - Units string `json:"units,omitempty"` - Default string `json:"default,omitempty"` - Status string `json:"status,omitempty"` // current, deprecated, obsolete - Description string `json:"description,omitempty"` - Reference string `json:"reference,omitempty"` -} - -// RFC6020Type represents a type statement with all restrictions (RFC 7.4, Section 9) -type RFC6020Type struct { - Name string `json:"name"` - Base string `json:"base,omitempty"` // for identityref - Path string `json:"path,omitempty"` // for leafref - Patterns []RFC6020Pattern `json:"patterns,omitempty"` // for string - Ranges []RFC6020Range `json:"ranges,omitempty"` // for numeric types - Lengths []RFC6020Range `json:"lengths,omitempty"` // for string/binary - Enums []RFC6020Enum `json:"enums,omitempty"` // for enumeration - Bits []RFC6020Bit `json:"bits,omitempty"` // for bits - FractionDigits int `json:"fraction_digits,omitempty"` // for decimal64 - RequireInstance bool `json:"require_instance,omitempty"` // for leafref/instance-identifier - UnionTypes []RFC6020Type `json:"union_types,omitempty"` // for union -} - -// RFC6020Pattern represents a pattern restriction (RFC 9.4.6) -type RFC6020Pattern struct { - Value string `json:"value"` - Modifier string `json:"modifier,omitempty"` // "invert-match" for YANG 1.1 - Description string `json:"description,omitempty"` - Reference string `json:"reference,omitempty"` - ErrorAppTag string `json:"error_app_tag,omitempty"` - ErrorMessage string `json:"error_message,omitempty"` -} - -// RFC6020Range represents a range or length restriction (RFC 9.2.4, 9.4.4) -type RFC6020Range struct { - Min string `json:"min"` // "min" or numeric value - Max string `json:"max"` // "max" or numeric value - Description string `json:"description,omitempty"` - Reference string `json:"reference,omitempty"` - ErrorAppTag string `json:"error_app_tag,omitempty"` - ErrorMessage string `json:"error_message,omitempty"` -} - -// RFC6020Enum represents an enum statement (RFC 9.6.4) -type RFC6020Enum struct { - Name string `json:"name"` - Value *int64 `json:"value,omitempty"` - Status string `json:"status,omitempty"` - Description string `json:"description,omitempty"` - Reference string `json:"reference,omitempty"` - IfFeature string `json:"if_feature,omitempty"` // YANG 1.1 -} - -// RFC6020Bit represents a bit statement (RFC 9.7.4) -type RFC6020Bit struct { - Name string `json:"name"` - Position *int `json:"position,omitempty"` - Status string `json:"status,omitempty"` - Description string `json:"description,omitempty"` - Reference string `json:"reference,omitempty"` - IfFeature string `json:"if_feature,omitempty"` // YANG 1.1 -} - -// RFC6020Feature represents a feature statement (RFC 7.20.1) -type RFC6020Feature struct { - Name string `json:"name"` - Status string `json:"status,omitempty"` - Description string `json:"description,omitempty"` - Reference string `json:"reference,omitempty"` - IfFeatures []string `json:"if_features,omitempty"` // dependencies -} - -// RFC6020DataNode represents any data definition node (RFC 4.2.2) -type RFC6020DataNode struct { - Name string `json:"name"` - NodeType string `json:"node_type"` // container, leaf, leaf-list, list, choice, case, anyxml, anydata - Type *RFC6020Type `json:"type,omitempty"` - Config *bool `json:"config,omitempty"` - Mandatory *bool `json:"mandatory,omitempty"` - Presence string `json:"presence,omitempty"` - Keys []string `json:"keys,omitempty"` - Unique []string `json:"unique,omitempty"` - MinElements *int `json:"min_elements,omitempty"` - MaxElements *int `json:"max_elements,omitempty"` - OrderedBy string `json:"ordered_by,omitempty"` - Default []string `json:"default,omitempty"` - Units string `json:"units,omitempty"` - Status string `json:"status,omitempty"` - Description string `json:"description,omitempty"` - Reference string `json:"reference,omitempty"` - IfFeatures []string `json:"if_features,omitempty"` - Children map[string]*RFC6020DataNode `json:"children,omitempty"` - Path string `json:"path"` // Full XPath -} - -// RFC6020Grouping represents a grouping statement (RFC 7.12) -type RFC6020Grouping struct { - Name string `json:"name"` - Status string `json:"status,omitempty"` - Description string `json:"description,omitempty"` - Reference string `json:"reference,omitempty"` - Typedefs map[string]*RFC6020Typedef `json:"typedefs,omitempty"` - Groupings map[string]*RFC6020Grouping `json:"groupings,omitempty"` - DataNodes map[string]*RFC6020DataNode `json:"data_nodes,omitempty"` -} - -// RFC6020BuiltinType represents YANG built-in types (RFC Section 9) -type RFC6020BuiltinType struct { - Name string `json:"name"` - BaseType string `json:"base_type,omitempty"` - DefaultValue string `json:"default_value,omitempty"` - Restrictions []string `json:"restrictions,omitempty"` - LexicalFormat string `json:"lexical_format,omitempty"` - CanonicalFormat string `json:"canonical_format,omitempty"` - ValueSpace string `json:"value_space,omitempty"` - IsNumeric bool `json:"is_numeric"` - IsSigned bool `json:"is_signed,omitempty"` - BitSize int `json:"bit_size,omitempty"` -} - -// RFC6020TypeRestriction represents type restriction rules -type RFC6020TypeRestriction struct { - AllowedRestrictions []string `json:"allowed_restrictions"` - DefaultRange string `json:"default_range,omitempty"` - DefaultLength string `json:"default_length,omitempty"` -} - -// RFC6020ResolvedType represents a fully resolved type with semantic information -type RFC6020ResolvedType struct { - OriginalType string `json:"original_type"` - ResolvedType string `json:"resolved_type"` - BaseBuiltinType string `json:"base_builtin_type"` - Units string `json:"units,omitempty"` - Range *RFC6020Range `json:"range,omitempty"` - Enumeration map[string]int64 `json:"enumeration,omitempty"` - Patterns []string `json:"patterns,omitempty"` - FractionDigits int `json:"fraction_digits,omitempty"` - IsCounter bool `json:"is_counter"` - IsGauge bool `json:"is_gauge"` - IsConfiguration bool `json:"is_configuration"` - IsState bool `json:"is_state"` - SemanticType string `json:"semantic_type"` // counter, gauge, info - Description string `json:"description,omitempty"` -} - -// NewRFC6020Parser creates a new RFC-compliant YANG parser -func NewRFC6020Parser() *RFC6020Parser { - parser := &RFC6020Parser{ - modules: make(map[string]*RFC6020Module), - builtinTypes: make(map[string]*RFC6020BuiltinType), - typeRestrictions: make(map[string]*RFC6020TypeRestriction), - logger: log.New(os.Stdout, "[RFC6020Parser] ", log.LstdFlags), - } - - parser.initializeBuiltinTypes() - return parser -} - -// initializeBuiltinTypes initializes all YANG built-in types according to RFC 6020/7950 -func (p *RFC6020Parser) initializeBuiltinTypes() { - // RFC 9.2: Numeric types - p.builtinTypes["int8"] = &RFC6020BuiltinType{ - Name: "int8", BaseType: "integer", IsNumeric: true, IsSigned: true, BitSize: 8, - ValueSpace: "-128 to 127", Restrictions: []string{"range"}, - LexicalFormat: "Decimal number with optional leading sign", - CanonicalFormat: "Decimal number with no leading zeros, no plus sign", - } - - p.builtinTypes["int16"] = &RFC6020BuiltinType{ - Name: "int16", BaseType: "integer", IsNumeric: true, IsSigned: true, BitSize: 16, - ValueSpace: "-32768 to 32767", Restrictions: []string{"range"}, - LexicalFormat: "Decimal number with optional leading sign", - CanonicalFormat: "Decimal number with no leading zeros, no plus sign", - } - - p.builtinTypes["int32"] = &RFC6020BuiltinType{ - Name: "int32", BaseType: "integer", IsNumeric: true, IsSigned: true, BitSize: 32, - ValueSpace: "-2147483648 to 2147483647", Restrictions: []string{"range"}, - LexicalFormat: "Decimal number with optional leading sign", - CanonicalFormat: "Decimal number with no leading zeros, no plus sign", - } - - p.builtinTypes["int64"] = &RFC6020BuiltinType{ - Name: "int64", BaseType: "integer", IsNumeric: true, IsSigned: true, BitSize: 64, - ValueSpace: "-9223372036854775808 to 9223372036854775807", Restrictions: []string{"range"}, - LexicalFormat: "Decimal number with optional leading sign", - CanonicalFormat: "Decimal number with no leading zeros, no plus sign", - } - - p.builtinTypes["uint8"] = &RFC6020BuiltinType{ - Name: "uint8", BaseType: "integer", IsNumeric: true, IsSigned: false, BitSize: 8, - ValueSpace: "0 to 255", Restrictions: []string{"range"}, - LexicalFormat: "Decimal number without leading sign", - CanonicalFormat: "Decimal number with no leading zeros", - } - - p.builtinTypes["uint16"] = &RFC6020BuiltinType{ - Name: "uint16", BaseType: "integer", IsNumeric: true, IsSigned: false, BitSize: 16, - ValueSpace: "0 to 65535", Restrictions: []string{"range"}, - LexicalFormat: "Decimal number without leading sign", - CanonicalFormat: "Decimal number with no leading zeros", - } - - p.builtinTypes["uint32"] = &RFC6020BuiltinType{ - Name: "uint32", BaseType: "integer", IsNumeric: true, IsSigned: false, BitSize: 32, - ValueSpace: "0 to 4294967295", Restrictions: []string{"range"}, - LexicalFormat: "Decimal number without leading sign", - CanonicalFormat: "Decimal number with no leading zeros", - } - - p.builtinTypes["uint64"] = &RFC6020BuiltinType{ - Name: "uint64", BaseType: "integer", IsNumeric: true, IsSigned: false, BitSize: 64, - ValueSpace: "0 to 18446744073709551615", Restrictions: []string{"range"}, - LexicalFormat: "Decimal number without leading sign", - CanonicalFormat: "Decimal number with no leading zeros", - } - - // RFC 9.3: decimal64 - p.builtinTypes["decimal64"] = &RFC6020BuiltinType{ - Name: "decimal64", BaseType: "decimal", IsNumeric: true, IsSigned: true, - ValueSpace: "Decimal numbers with 1-18 fraction digits", Restrictions: []string{"range", "fraction-digits"}, - LexicalFormat: "Decimal number with mandatory fraction-digits", - CanonicalFormat: "Decimal representation with required fraction digits", - } - - // RFC 9.4: string - p.builtinTypes["string"] = &RFC6020BuiltinType{ - Name: "string", BaseType: "string", IsNumeric: false, - ValueSpace: "Unicode/ISO 10646 characters excluding C0 controls, surrogates, noncharacters", - Restrictions: []string{"length", "pattern"}, - LexicalFormat: "UTF-8 character sequence", - CanonicalFormat: "Same as lexical representation", - } - - // RFC 9.5: boolean - p.builtinTypes["boolean"] = &RFC6020BuiltinType{ - Name: "boolean", BaseType: "boolean", IsNumeric: false, - ValueSpace: "true, false", Restrictions: []string{}, - LexicalFormat: "true or false", - CanonicalFormat: "true or false", - } - - // RFC 9.6: enumeration - p.builtinTypes["enumeration"] = &RFC6020BuiltinType{ - Name: "enumeration", BaseType: "enumeration", IsNumeric: false, - ValueSpace: "Defined by enum statements", Restrictions: []string{"enum"}, - LexicalFormat: "Enum name string", - CanonicalFormat: "Same as lexical representation", - } - - // RFC 9.7: bits - p.builtinTypes["bits"] = &RFC6020BuiltinType{ - Name: "bits", BaseType: "bits", IsNumeric: false, - ValueSpace: "Set of bit positions defined by bit statements", Restrictions: []string{"bit"}, - LexicalFormat: "Space-separated list of bit names", - CanonicalFormat: "Space-separated list ordered by position", - } - - // RFC 9.8: binary - p.builtinTypes["binary"] = &RFC6020BuiltinType{ - Name: "binary", BaseType: "binary", IsNumeric: false, - ValueSpace: "Any binary data", Restrictions: []string{"length"}, - LexicalFormat: "Base64 encoded string", - CanonicalFormat: "Base64 with standard alphabet, no line breaks", - } - - // RFC 9.9: leafref - p.builtinTypes["leafref"] = &RFC6020BuiltinType{ - Name: "leafref", BaseType: "leafref", IsNumeric: false, - ValueSpace: "Same as referenced leaf", Restrictions: []string{"path", "require-instance"}, - LexicalFormat: "Same as referenced leaf type", - CanonicalFormat: "Same as referenced leaf type", - } - - // RFC 9.10: identityref - p.builtinTypes["identityref"] = &RFC6020BuiltinType{ - Name: "identityref", BaseType: "identityref", IsNumeric: false, - ValueSpace: "Identity names derived from base identity", Restrictions: []string{"base"}, - LexicalFormat: "QName with optional prefix", - CanonicalFormat: "QName in module's namespace", - } - - // RFC 9.11: empty - p.builtinTypes["empty"] = &RFC6020BuiltinType{ - Name: "empty", BaseType: "empty", IsNumeric: false, - ValueSpace: "No value", Restrictions: []string{}, - LexicalFormat: "Not applicable", - CanonicalFormat: "Not applicable", - } - - // RFC 9.12: union - p.builtinTypes["union"] = &RFC6020BuiltinType{ - Name: "union", BaseType: "union", IsNumeric: false, - ValueSpace: "Union of member types", Restrictions: []string{"type"}, - LexicalFormat: "Any valid member type format", - CanonicalFormat: "First matching member type canonical form", - } - - // RFC 9.13: instance-identifier - p.builtinTypes["instance-identifier"] = &RFC6020BuiltinType{ - Name: "instance-identifier", BaseType: "instance-identifier", IsNumeric: false, - ValueSpace: "XPath expressions identifying data nodes", Restrictions: []string{"require-instance"}, - LexicalFormat: "XPath subset identifying instance nodes", - CanonicalFormat: "Absolute path with predicates in canonical order", - } - - p.logger.Printf("Initialized %d built-in YANG types per RFC 6020/7950", len(p.builtinTypes)) -} - -// ParseYANGModule parses a YANG module from content according to RFC specifications -func (p *RFC6020Parser) ParseYANGModule(content, filename string) (*RFC6020Module, error) { - module := &RFC6020Module{ - Imports: make(map[string]*RFC6020Import), - Includes: make(map[string]*RFC6020Include), - Revisions: make([]*RFC6020Revision, 0), - Typedefs: make(map[string]*RFC6020Typedef), - Groupings: make(map[string]*RFC6020Grouping), - Features: make(map[string]*RFC6020Feature), - DataNodes: make(map[string]*RFC6020DataNode), - KeyedPaths: make(map[string]string), - ListKeys: make(map[string][]string), - DataTypes: make(map[string]*RFC6020ResolvedType), - Counters: make([]string, 0), - Gauges: make([]string, 0), - ConfigPaths: make([]string, 0), - StatePaths: make([]string, 0), - } - - // Tokenize and parse according to RFC 6020 Section 6 - tokens, err := p.TokenizeYANG(content) - if err != nil { - return nil, fmt.Errorf("tokenization failed: %w", err) - } - - p.parseTokens(tokens, module) - - // Perform semantic analysis - p.performSemanticAnalysis(module) - - p.modules[module.Name] = module - p.logger.Printf("Successfully parsed YANG module '%s' from %s", module.Name, filename) - - return module, nil -} - -// tokenizeYANG performs lexical analysis according to RFC 6020 Section 6.1 -func (*RFC6020Parser) TokenizeYANG(content string) ([]string, error) { - var tokens []string - - // Remove C-style comments (RFC 6020 Section 6.1.1) - // Single-line comments: // comment - singleLineCommentRe := regexp.MustCompile(`//.*?(?:\r?\n|$)`) - content = singleLineCommentRe.ReplaceAllString(content, "\n") - - // Block comments: /* comment */ (including multiline) - blockCommentRe := regexp.MustCompile(`(?s)/\*.*?\*/`) - content = blockCommentRe.ReplaceAllString(content, " ") - - // Tokenize according to RFC 6020 Section 6.1.2 - // Strings (with newlines), keywords, semicolons, braces, numbers - tokenRe := regexp.MustCompile(`(?s)"[^"]*"|'[^']*'|[a-zA-Z_][a-zA-Z0-9_.-]*|[0-9]+(?:\.[0-9]+)?|[{};]`) - matches := tokenRe.FindAllString(content, -1) - - for _, match := range matches { - match = strings.TrimSpace(match) - if match != "" && match != "\n" && match != "\r" { - tokens = append(tokens, match) - } - } - - return tokens, nil -} - -// parseTokens parses tokenized YANG content according to RFC grammar -func (p *RFC6020Parser) parseTokens(tokens []string, module *RFC6020Module) { - i := 0 - - for i < len(tokens) { - switch tokens[i] { - case "module": - if i+1 < len(tokens) { - module.Name = p.unquoteString(tokens[i+1]) - i += 2 - } - case "yang-version": - if i+1 < len(tokens) { - value := p.unquoteString(tokens[i+1]) - // Remove trailing semicolon if present - value = strings.TrimSuffix(value, ";") - module.YangVersion = value - i += 2 - // Skip semicolon if it's the next token - if i < len(tokens) && tokens[i] == ";" { - i++ - } - } - case "namespace": - if i+1 < len(tokens) { - module.Namespace = p.unquoteString(tokens[i+1]) - i += 2 - // Skip semicolon if it's the next token - if i < len(tokens) && tokens[i] == ";" { - i++ - } - } - case "prefix": - if i+1 < len(tokens) { - module.Prefix = p.unquoteString(tokens[i+1]) - i += 2 - // Skip semicolon if it's the next token - if i < len(tokens) && tokens[i] == ";" { - i++ - } - } - case "organization": - if i+1 < len(tokens) { - module.Organization = p.unquoteString(tokens[i+1]) - i += 2 - } - case "contact": - if i+1 < len(tokens) { - module.Contact = p.unquoteString(tokens[i+1]) - i += 2 - } - case "description": - if i+1 < len(tokens) { - module.Description = p.unquoteString(tokens[i+1]) - i += 2 - } - case "reference": - if i+1 < len(tokens) { - module.Reference = p.unquoteString(tokens[i+1]) - i += 2 - } - case "revision": - rev, consumed := p.parseRevision(tokens[i:]) - if rev != nil { - module.Revisions = append(module.Revisions, rev) - } - i += consumed - case "import": - imp, consumed := p.parseImport(tokens[i:]) - if imp != nil { - module.Imports[imp.Module] = imp - } - i += consumed - case "typedef": - td, consumed := p.parseTypedef(tokens[i:]) - if td != nil { - module.Typedefs[td.Name] = td - } - i += consumed - case "feature": - feat, consumed := p.parseFeature(tokens[i:]) - if feat != nil { - module.Features[feat.Name] = feat - } - i += consumed - case "container", "leaf", "leaf-list", "list": - node, consumed := p.parseDataNode(tokens[i:], "") - if node != nil { - module.DataNodes[node.Name] = node - node.Path = "/" + node.Name - } - i += consumed - default: - i++ - } - } - - // Sort revisions by date (newest first) - sort.Slice(module.Revisions, func(i, j int) bool { - return module.Revisions[i].Date > module.Revisions[j].Date - }) -} - -// parseRevision parses a revision statement (RFC 7.1.9) -func (p *RFC6020Parser) parseRevision(tokens []string) (*RFC6020Revision, int) { - if len(tokens) < 2 { - return nil, 1 - } - - rev := &RFC6020Revision{ - Date: p.unquoteString(tokens[1]), - } - - i := 2 - if i < len(tokens) && tokens[i] == "{" { - i++ - for i < len(tokens) && tokens[i] != "}" { - switch tokens[i] { - case "description": - if i+1 < len(tokens) { - rev.Description = p.unquoteString(tokens[i+1]) - i += 2 - } - case "reference": - if i+1 < len(tokens) { - rev.Reference = p.unquoteString(tokens[i+1]) - i += 2 - } - default: - i++ - } - } - if i < len(tokens) && tokens[i] == "}" { - i++ - } - } - - return rev, i -} - -// parseImport parses an import statement (RFC 7.1.5) -func (p *RFC6020Parser) parseImport(tokens []string) (*RFC6020Import, int) { - if len(tokens) < 2 { - return nil, 1 - } - - imp := &RFC6020Import{ - Module: p.unquoteString(tokens[1]), - } - - i := 2 - if i < len(tokens) && tokens[i] == "{" { - i++ - for i < len(tokens) && tokens[i] != "}" { - switch tokens[i] { - case "prefix": - if i+1 < len(tokens) { - imp.Prefix = p.unquoteString(tokens[i+1]) - i += 2 - } - case "revision-date": - if i+1 < len(tokens) { - imp.RevisionDate = p.unquoteString(tokens[i+1]) - i += 2 - } - default: - i++ - } - } - if i < len(tokens) && tokens[i] == "}" { - i++ - } - } - - return imp, i -} - -// parseTypedef parses a typedef statement (RFC 7.3) -func (p *RFC6020Parser) parseTypedef(tokens []string) (*RFC6020Typedef, int) { - if len(tokens) < 2 { - return nil, 1 - } - - td := &RFC6020Typedef{ - Name: p.unquoteString(tokens[1]), - } - - i := 2 - if i < len(tokens) && tokens[i] == "{" { - i++ - for i < len(tokens) && tokens[i] != "}" { - switch tokens[i] { - case "type": - typ, consumed := p.parseType(tokens[i:]) - if typ != nil { - td.Type = typ - } - i += consumed - case "units": - if i+1 < len(tokens) { - td.Units = p.unquoteString(tokens[i+1]) - i += 2 - } - case "default": - if i+1 < len(tokens) { - td.Default = p.unquoteString(tokens[i+1]) - i += 2 - } - case "description": - if i+1 < len(tokens) { - td.Description = p.unquoteString(tokens[i+1]) - i += 2 - } - default: - i++ - } - } - if i < len(tokens) && tokens[i] == "}" { - i++ - } - } - - return td, i -} - -// parseType parses a type statement with all restrictions (RFC 7.4) -func (p *RFC6020Parser) parseType(tokens []string) (*RFC6020Type, int) { - if len(tokens) < 2 { - return nil, 1 - } - - typ := &RFC6020Type{ - Name: p.unquoteString(tokens[1]), - } - - i := 2 - if i < len(tokens) && tokens[i] == "{" { - i++ - for i < len(tokens) && tokens[i] != "}" { - switch tokens[i] { - case "range": - if i+1 < len(tokens) { - ranges := p.parseRangeExpression(tokens[i+1]) - typ.Ranges = ranges - i += 2 - } - case "length": - if i+1 < len(tokens) { - lengths := p.parseRangeExpression(tokens[i+1]) - typ.Lengths = lengths - i += 2 - } - case "pattern": - if i+1 < len(tokens) { - pattern := RFC6020Pattern{ - Value: p.unquoteString(tokens[i+1]), - } - typ.Patterns = append(typ.Patterns, pattern) - i += 2 - } - case "enum": - if i+1 < len(tokens) { - enum := RFC6020Enum{ - Name: p.unquoteString(tokens[i+1]), - } - i += 2 - // Check if there's a block for the enum - if i < len(tokens) && tokens[i] == "{" { - i++ // Skip opening brace - for i < len(tokens) && tokens[i] != "}" { - switch tokens[i] { - case "description": - if i+1 < len(tokens) { - enum.Description = p.unquoteString(tokens[i+1]) - i += 2 - // Skip semicolon if present - if i < len(tokens) && tokens[i] == ";" { - i++ - } - } - case "value": - if i+1 < len(tokens) { - if val, err := strconv.ParseInt(tokens[i+1], 10, 64); err == nil { - enum.Value = &val - } - i += 2 - // Skip semicolon if present - if i < len(tokens) && tokens[i] == ";" { - i++ - } - } - default: - i++ - } - } - if i < len(tokens) && tokens[i] == "}" { - i++ // Skip closing brace - } - } - typ.Enums = append(typ.Enums, enum) - } - case "bit": - if i+1 < len(tokens) { - bit := RFC6020Bit{ - Name: p.unquoteString(tokens[i+1]), - } - typ.Bits = append(typ.Bits, bit) - i += 2 - } - case "fraction-digits": - if i+1 < len(tokens) { - if fd, err := strconv.Atoi(tokens[i+1]); err == nil { - typ.FractionDigits = fd - } - i += 2 - } - case "path": - if i+1 < len(tokens) { - typ.Path = p.unquoteString(tokens[i+1]) - i += 2 - } - default: - i++ - } - } - if i < len(tokens) && tokens[i] == "}" { - i++ - } - } - - return typ, i -} - -// parseFeature parses a feature statement (RFC 7.20.1) -func (p *RFC6020Parser) parseFeature(tokens []string) (*RFC6020Feature, int) { - if len(tokens) < 2 { - return nil, 1 - } - - feat := &RFC6020Feature{ - Name: p.unquoteString(tokens[1]), - } - - i := 2 - if i < len(tokens) && tokens[i] == "{" { - i++ - for i < len(tokens) && tokens[i] != "}" { - switch tokens[i] { - case "description": - if i+1 < len(tokens) { - feat.Description = p.unquoteString(tokens[i+1]) - i += 2 - } - default: - i++ - } - } - if i < len(tokens) && tokens[i] == "}" { - i++ - } - } - - return feat, i -} - -// parseDataNode parses data definition statements (RFC 4.2.2) -func (p *RFC6020Parser) parseDataNode(tokens []string, parentPath string) (*RFC6020DataNode, int) { - if len(tokens) < 2 { - return nil, 1 - } - - node := &RFC6020DataNode{ - NodeType: tokens[0], - Name: p.unquoteString(tokens[1]), - Children: make(map[string]*RFC6020DataNode), - } - - i := 2 - if i < len(tokens) && tokens[i] == "{" { - i++ - for i < len(tokens) && tokens[i] != "}" { - switch tokens[i] { - case "type": - typ, consumed := p.parseType(tokens[i:]) - if typ != nil { - node.Type = typ - } - i += consumed - case "key": - if i+1 < len(tokens) { - keyStr := p.unquoteString(tokens[i+1]) - node.Keys = strings.Fields(keyStr) - i += 2 - } - case "config": - if i+1 < len(tokens) { - config := p.unquoteString(tokens[i+1]) == "true" - node.Config = &config - i += 2 - } - case "mandatory": - if i+1 < len(tokens) { - mandatory := p.unquoteString(tokens[i+1]) == "true" - node.Mandatory = &mandatory - i += 2 - } - case "description": - if i+1 < len(tokens) { - node.Description = p.unquoteString(tokens[i+1]) - i += 2 - } - case "units": - if i+1 < len(tokens) { - node.Units = p.unquoteString(tokens[i+1]) - i += 2 - } - case "container", "leaf", "leaf-list", "list": - childPath := parentPath + "/" + node.Name - child, consumed := p.parseDataNode(tokens[i:], childPath) - if child != nil { - node.Children[child.Name] = child - child.Path = childPath + "/" + child.Name - } - i += consumed - default: - i++ - } - } - if i < len(tokens) && tokens[i] == "}" { - i++ - } - } - - return node, i -} - -// performSemanticAnalysis analyzes the parsed module for semantic information -func (p *RFC6020Parser) performSemanticAnalysis(module *RFC6020Module) { - // Analyze data nodes for keys, types, and semantics - p.analyzeDataNodes(module, module.DataNodes, "") - - // Resolve all type references - p.resolveTypes(module) - - // Classify metrics as counters or gauges - p.classifyMetrics(module) - - p.logger.Printf("Semantic analysis complete for module %s: %d keyed paths, %d data types", - module.Name, len(module.KeyedPaths), len(module.DataTypes)) -} - -// analyzeDataNodes recursively analyzes data nodes -func (p *RFC6020Parser) analyzeDataNodes(module *RFC6020Module, nodes map[string]*RFC6020DataNode, parentPath string) { - for _, node := range nodes { - fullPath := parentPath + "/" + node.Name - - // Handle lists with keys - if node.NodeType == "list" && len(node.Keys) > 0 { - module.KeyedPaths[fullPath] = node.Keys[0] // Primary key - module.ListKeys[fullPath] = node.Keys - } - - // Classify as config or state data - if node.Config != nil { - if *node.Config { - module.ConfigPaths = append(module.ConfigPaths, fullPath) - } else { - module.StatePaths = append(module.StatePaths, fullPath) - } - } - - // Recursively process children - if len(node.Children) > 0 { - p.analyzeDataNodes(module, node.Children, fullPath) - } - } -} - -// resolveTypes resolves all type references to their base built-in types -func (p *RFC6020Parser) resolveTypes(module *RFC6020Module) { - // Resolve typedef types - for _, typedef := range module.Typedefs { - if typedef.Type != nil { - resolved := p.resolveTypeRecursive(typedef.Type, module) - if resolved != nil { - module.DataTypes[typedef.Name] = resolved - } - } - } - - // Resolve data node types - p.resolveDataNodeTypes(module, module.DataNodes, "", module) -} - -// resolveDataNodeTypes resolves types for all data nodes -func (p *RFC6020Parser) resolveDataNodeTypes(module *RFC6020Module, nodes map[string]*RFC6020DataNode, parentPath string, currentModule *RFC6020Module) { - for _, node := range nodes { - fullPath := parentPath + "/" + node.Name - - if node.Type != nil { - resolved := p.resolveTypeRecursive(node.Type, currentModule) - if resolved != nil { - resolved.Description = node.Description - // Use node units if available, otherwise keep typedef units - if node.Units != "" { - resolved.Units = node.Units - } - - // Set configuration vs state - if node.Config != nil { - resolved.IsConfiguration = *node.Config - resolved.IsState = !(*node.Config) - } else { - // Default to configuration true if not specified - resolved.IsConfiguration = true - resolved.IsState = false - } - - module.DataTypes[fullPath] = resolved - } - } - - // Recursively process children - if len(node.Children) > 0 { - // Propagate config setting to children if parent has config false - for _, child := range node.Children { - if child.Config == nil && node.Config != nil && !*node.Config { - config := false - child.Config = &config - } - } - p.resolveDataNodeTypes(module, node.Children, fullPath, currentModule) - } - } -} - -// resolveTypeRecursive recursively resolves a type to its base built-in type -func (p *RFC6020Parser) resolveTypeRecursive(yangType *RFC6020Type, module *RFC6020Module) *RFC6020ResolvedType { - if yangType == nil { - return nil - } - - resolved := &RFC6020ResolvedType{ - OriginalType: yangType.Name, - ResolvedType: yangType.Name, - } - - // Check if it's a built-in type - if builtin, exists := p.builtinTypes[yangType.Name]; exists { - resolved.BaseBuiltinType = builtin.Name - resolved.ResolvedType = builtin.Name - - // Copy restrictions - if len(yangType.Ranges) > 0 { - resolved.Range = &yangType.Ranges[0] // Simplified - } - - if len(yangType.Enums) > 0 { - resolved.Enumeration = make(map[string]int64) - for _, enum := range yangType.Enums { - if enum.Value != nil { - resolved.Enumeration[enum.Name] = *enum.Value - } - } - } - resolved.FractionDigits = yangType.FractionDigits - } else if typedef, exists := module.Typedefs[yangType.Name]; exists { - // Recursively resolve typedef - baseResolved := p.resolveTypeRecursive(typedef.Type, module) - if baseResolved != nil { - resolved.BaseBuiltinType = baseResolved.BaseBuiltinType - resolved.ResolvedType = baseResolved.ResolvedType - if resolved.Units == "" { - resolved.Units = typedef.Units - } - // Propagate fraction digits from typedef - if baseResolved.FractionDigits > 0 { - resolved.FractionDigits = baseResolved.FractionDigits - } else if typedef.Type.FractionDigits > 0 { - resolved.FractionDigits = typedef.Type.FractionDigits - } - } - } - - return resolved -} - -// classifyMetrics classifies data types as counters or gauges based on semantic analysis -func (p *RFC6020Parser) classifyMetrics(module *RFC6020Module) { - for path, dataType := range module.DataTypes { - if dataType.BaseBuiltinType == "" { - continue - } - - isCounter := p.isCounterSemantic(dataType) - isGauge := p.isGaugeSemantic(dataType) - - switch { - case isCounter: - dataType.IsCounter = true - dataType.SemanticType = "counter" - module.Counters = append(module.Counters, path) - case isGauge: - dataType.IsGauge = true - dataType.SemanticType = "gauge" - module.Gauges = append(module.Gauges, path) - default: - dataType.SemanticType = "info" - } - } -} - -// isCounterSemantic determines if a data type represents a counter metric -func (*RFC6020Parser) isCounterSemantic(dataType *RFC6020ResolvedType) bool { - // Counters are typically unsigned integers with accumulating units - if !strings.HasPrefix(dataType.BaseBuiltinType, "uint") { - return false - } - - // Check for rate units (these are gauges, not counters) - rateUnits := []string{"per-second", "pps", "bps", "kbps", "mbps", "gbps", "rate"} - for _, rate := range rateUnits { - if strings.Contains(strings.ToLower(dataType.Units), rate) { - return false - } - } - - // Counter units - counterUnits := []string{"bytes", "octets", "packets", "count", "total", "errors", "discards", "drops"} - for _, counter := range counterUnits { - if strings.Contains(strings.ToLower(dataType.Units), counter) { - return true - } - } - - return false -} - -// isGaugeSemantic determines if a data type represents a gauge metric -func (*RFC6020Parser) isGaugeSemantic(dataType *RFC6020ResolvedType) bool { - // Gauge units include rates, percentages, current values - gaugeUnits := []string{ - "percent", "per-second", "pps", "bps", "kbps", "mbps", "gbps", - "utilization", "rate", "current", "level", "temperature", - "voltage", "frequency", "load", "usage", - } - - for _, gauge := range gaugeUnits { - if strings.Contains(strings.ToLower(dataType.Units), gauge) { - return true - } - } - - return false -} - -// Helper functions - -func (*RFC6020Parser) unquoteString(s string) string { - if len(s) >= 2 { - if (s[0] == '"' && s[len(s)-1] == '"') || (s[0] == '\'' && s[len(s)-1] == '\'') { - return s[1 : len(s)-1] - } - } - return s -} - -func (p *RFC6020Parser) parseRangeExpression(expr string) []RFC6020Range { - expr = p.unquoteString(expr) - parts := strings.Split(expr, "|") - var ranges []RFC6020Range - - for _, part := range parts { - part = strings.TrimSpace(part) - if strings.Contains(part, "..") { - bounds := strings.Split(part, "..") - if len(bounds) == 2 { - ranges = append(ranges, RFC6020Range{ - Min: strings.TrimSpace(bounds[0]), - Max: strings.TrimSpace(bounds[1]), - }) - } - } else { - // Single value range - ranges = append(ranges, RFC6020Range{ - Min: part, - Max: part, - }) - } - } - - return ranges -} - -// Public API methods - -// GetModules returns all loaded modules -func (p *RFC6020Parser) GetModules() map[string]*RFC6020Module { - return p.modules -} - -// GetModuleByName returns a specific module by name -func (p *RFC6020Parser) GetModuleByName(name string) *RFC6020Module { - return p.modules[name] -} - -// GetBuiltinTypes returns all YANG built-in types -func (p *RFC6020Parser) GetBuiltinTypes() map[string]*RFC6020BuiltinType { - return p.builtinTypes -} - -// ExportModules exports all modules to JSON for external use -func (p *RFC6020Parser) ExportModules() ([]byte, error) { - return json.MarshalIndent(p.modules, "", " ") -} - -// SaveModules saves all modules to a file -func (p *RFC6020Parser) SaveModules(filename string) error { - data, err := p.ExportModules() - if err != nil { - return err - } - return os.WriteFile(filename, data, 0o600) -} - -// AnalyzeTelemetryPath analyzes a telemetry encoding path and provides YANG context -func (p *RFC6020Parser) AnalyzeTelemetryPath(encodingPath string) *RFC6020TelemetryAnalysis { - // Extract module name from encoding path (e.g., "Cisco-IOS-XE-interfaces-oper:interfaces/interface/statistics") - parts := strings.SplitN(encodingPath, ":", 2) - if len(parts) != 2 { - return &RFC6020TelemetryAnalysis{ - EncodingPath: encodingPath, - IsValid: false, - ErrorReason: "Invalid encoding path format - missing module prefix", - } - } - - moduleName := parts[0] - xpath := parts[1] - - // Check if we have this module loaded - module := p.modules[moduleName] - if module == nil { - // Create a dynamic module for unknown modules - module = p.createDynamicModule(moduleName, encodingPath) - p.modules[moduleName] = module - p.logger.Printf("Created dynamic YANG module for: %s", moduleName) - } - - // Parse the XPath to identify data nodes and list keys - pathSegments := strings.Split(strings.Trim(xpath, "/"), "/") - - analysis := &RFC6020TelemetryAnalysis{ - EncodingPath: encodingPath, - ModuleName: moduleName, - XPath: xpath, - PathSegments: pathSegments, - IsValid: true, - Module: module, - DataNodes: make(map[string]*RFC6020DataNode), - SemanticContext: make(map[string]string), - } - - // Build the full path and identify list paths - fullPath := "" - for _, segment := range pathSegments { - if fullPath != "" { - fullPath += "/" - } - fullPath += segment - - // Check if this segment represents a list in the module - if dataNode := p.findDataNodeByPath(module, fullPath); dataNode != nil { - analysis.DataNodes[fullPath] = dataNode - if dataNode.NodeType == "list" { - analysis.ListPath = "/" + fullPath - // Extract list keys if available - if len(dataNode.Keys) > 0 { - analysis.ListKeys = dataNode.Keys - } - } - } - } - - // Set default semantic classifications for known operational data patterns - p.applySemanticHeuristics(analysis) - - return analysis -} - -// createDynamicModule creates a YANG module definition for unknown modules based on encoding path -func (p *RFC6020Parser) createDynamicModule(moduleName, encodingPath string) *RFC6020Module { - module := &RFC6020Module{ - Name: moduleName, - Namespace: fmt.Sprintf("urn:ietf:params:xml:ns:yang:%s", moduleName), - Prefix: moduleName, - YangVersion: "1.0", - Description: fmt.Sprintf("Dynamically created module for %s based on telemetry data", moduleName), - DataNodes: make(map[string]*RFC6020DataNode), - Typedefs: make(map[string]*RFC6020Typedef), - Groupings: make(map[string]*RFC6020Grouping), - Features: make(map[string]*RFC6020Feature), - Imports: make(map[string]*RFC6020Import), - Includes: make(map[string]*RFC6020Include), - } - - // Add a revision - module.Revisions = []*RFC6020Revision{{ - Date: "2024-01-01", - Description: "Dynamic module creation from telemetry data", - }} - - // Extract path and create basic data node structure - parts := strings.SplitN(encodingPath, ":", 2) - if len(parts) == 2 { - xpath := parts[1] - p.createDataNodesFromPath(module, xpath) - } - - return module -} - -// createDataNodesFromPath creates basic data node structure from XPath -func (p *RFC6020Parser) createDataNodesFromPath(module *RFC6020Module, xpath string) { - segments := strings.Split(strings.Trim(xpath, "/"), "/") - currentPath := "" - - for i, segment := range segments { - if currentPath != "" { - currentPath += "/" - } - currentPath += segment - - // Create a basic data node if it doesn't exist - if _, exists := module.DataNodes[currentPath]; !exists { - nodeType := "leaf" - if i < len(segments)-1 { - nodeType = "container" - } - - // Detect if this is likely a list based on common patterns - if p.isLikelyListNode(segment, segments, i) { - nodeType = "list" - } - - config := false - mandatory := false - dataNode := &RFC6020DataNode{ - Name: segment, - NodeType: nodeType, - Path: "/" + currentPath, - Description: fmt.Sprintf("Auto-generated %s for %s", nodeType, segment), - Type: p.InferDataTypeFromPath(segment), - Children: make(map[string]*RFC6020DataNode), - Config: &config, // Operational data - Mandatory: &mandatory, - } - - // Add common list keys for known patterns - if nodeType == "list" { - dataNode.Keys = p.inferListKeys(segment) - } - - module.DataNodes[currentPath] = dataNode - } - } -} - -// isLikelyListNode determines if a path segment represents a list -func (*RFC6020Parser) isLikelyListNode(segment string, allSegments []string, index int) bool { - // Common list node patterns - listPatterns := []string{ - "interface", "interfaces", "interface-state", - "neighbor", "neighbors", "peer", "peers", - "route", "routes", "entry", "entries", - "session", "sessions", "connection", "connections", - "policy", "policies", "rule", "rules", - "memory-statistic", "cpu-usage", "process", - } - - lowerSegment := strings.ToLower(segment) - for _, pattern := range listPatterns { - if strings.Contains(lowerSegment, pattern) { - return true - } - } - - // If followed by what looks like statistics or state, probably a list - if index < len(allSegments)-1 { - nextSegment := strings.ToLower(allSegments[index+1]) - if strings.Contains(nextSegment, "statistic") || - strings.Contains(nextSegment, "state") || - strings.Contains(nextSegment, "status") { - return true - } - } - - return false -} - -// inferListKeys infers likely key fields for list nodes -func (*RFC6020Parser) inferListKeys(segment string) []string { - lowerSegment := strings.ToLower(segment) - - // Common key patterns based on list type - keyMappings := map[string][]string{ - "interface": {"name"}, - "interfaces": {"name"}, - "interface-state": {"name"}, - "neighbor": {"address"}, - "neighbors": {"address"}, - "peer": {"id", "address"}, - "peers": {"id", "address"}, - "route": {"prefix"}, - "routes": {"prefix"}, - "memory-statistic": {"name"}, - "cpu-usage": {"id"}, - "process": {"pid", "name"}, - "session": {"id"}, - "entry": {"id"}, - } - - for pattern, keys := range keyMappings { - if strings.Contains(lowerSegment, pattern) { - return keys - } - } - - // Default key - return []string{"name"} -} - -// InferDataTypeFromPath infers YANG data type from path segment -func (*RFC6020Parser) InferDataTypeFromPath(segment string) *RFC6020Type { - lowerSegment := strings.ToLower(segment) - - // Infer type based on common naming patterns - if strings.Contains(lowerSegment, "count") || - strings.Contains(lowerSegment, "total") || - strings.Contains(lowerSegment, "bytes") || - strings.Contains(lowerSegment, "packets") || - strings.Contains(lowerSegment, "errors") || - strings.Contains(lowerSegment, "drops") { - return &RFC6020Type{ - Name: "uint64", - } - } - - if strings.Contains(lowerSegment, "rate") || - strings.Contains(lowerSegment, "pps") || - strings.Contains(lowerSegment, "bps") || - strings.Contains(lowerSegment, "kbps") || - strings.Contains(lowerSegment, "mbps") || - strings.Contains(lowerSegment, "usage") || - strings.Contains(lowerSegment, "utilization") { - return &RFC6020Type{ - Name: "uint32", - } - } - - if strings.Contains(lowerSegment, "name") || - strings.Contains(lowerSegment, "description") || - strings.Contains(lowerSegment, "type") || - strings.Contains(lowerSegment, "status") || - strings.Contains(lowerSegment, "state") { - return &RFC6020Type{ - Name: "string", - } - } - - if strings.Contains(lowerSegment, "time") || - strings.Contains(lowerSegment, "timestamp") { - return &RFC6020Type{ - Name: "yang:date-and-time", - } - } - - // Default to string for unknown types - return &RFC6020Type{ - Name: "string", - } -} // findDataNodeByPath finds a data node by its path in the module -func (*RFC6020Parser) findDataNodeByPath(module *RFC6020Module, path string) *RFC6020DataNode { - cleanPath := strings.Trim(path, "/") - return module.DataNodes[cleanPath] -} - -// applySemanticHeuristics applies semantic classification heuristics -func (*RFC6020Parser) applySemanticHeuristics(analysis *RFC6020TelemetryAnalysis) { - // Extract the leaf name (last segment) - if len(analysis.PathSegments) > 0 { - leafName := analysis.PathSegments[len(analysis.PathSegments)-1] - lowerLeaf := strings.ToLower(leafName) - - // Counter patterns - if strings.Contains(lowerLeaf, "count") || - strings.Contains(lowerLeaf, "total") || - strings.Contains(lowerLeaf, "bytes") || - strings.Contains(lowerLeaf, "packets") || - strings.Contains(lowerLeaf, "errors") || - strings.Contains(lowerLeaf, "drops") || - strings.Contains(lowerLeaf, "discards") || - strings.Contains(lowerLeaf, "octets") { - analysis.SemanticType = "counter" - } - - // Gauge patterns - if strings.Contains(lowerLeaf, "rate") || - strings.Contains(lowerLeaf, "pps") || - strings.Contains(lowerLeaf, "bps") || - strings.Contains(lowerLeaf, "kbps") || - strings.Contains(lowerLeaf, "mbps") || - strings.Contains(lowerLeaf, "usage") || - strings.Contains(lowerLeaf, "utilization") || - strings.Contains(lowerLeaf, "load") { - analysis.SemanticType = "gauge" - } - - // Info patterns - if strings.Contains(lowerLeaf, "name") || - strings.Contains(lowerLeaf, "description") || - strings.Contains(lowerLeaf, "type") || - strings.Contains(lowerLeaf, "status") || - strings.Contains(lowerLeaf, "state") || - strings.Contains(lowerLeaf, "time") { - analysis.SemanticType = "info" - } - - // Default to gauge if no clear pattern - if analysis.SemanticType == "" { - analysis.SemanticType = "gauge" - } - } -} - -// RFC6020TelemetryAnalysis represents the analysis of a telemetry encoding path -type RFC6020TelemetryAnalysis struct { - EncodingPath string `json:"encoding_path"` - ModuleName string `json:"module_name"` - XPath string `json:"xpath"` - PathSegments []string `json:"path_segments"` - ListPath string `json:"list_path"` - ListKeys []string `json:"list_keys"` - SemanticType string `json:"semantic_type"` // "counter", "gauge", "info" - DataNodes map[string]*RFC6020DataNode `json:"data_nodes"` - SemanticContext map[string]string `json:"semantic_context"` - IsValid bool `json:"is_valid"` - ErrorReason string `json:"error_reason,omitempty"` - Module *RFC6020Module `json:"module,omitempty"` -} diff --git a/receiver/yanggrpcreceiver/internal/rfc_yang_parser_test.go b/receiver/yanggrpcreceiver/internal/rfc_yang_parser_test.go deleted file mode 100644 index 9c2c61845da25..0000000000000 --- a/receiver/yanggrpcreceiver/internal/rfc_yang_parser_test.go +++ /dev/null @@ -1,483 +0,0 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 - -package internal - -import ( - "path/filepath" - "slices" - "strings" - "testing" -) - -func TestRFC6020ParserBuiltinTypes(t *testing.T) { - parser := NewRFC6020Parser() - - // Test that all RFC 6020/7950 built-in types are loaded - expectedTypes := []string{ - "int8", "int16", "int32", "int64", - "uint8", "uint16", "uint32", "uint64", - "decimal64", "string", "boolean", "enumeration", - "bits", "binary", "leafref", "identityref", - "empty", "union", "instance-identifier", - } - - builtins := parser.GetBuiltinTypes() - if len(builtins) != len(expectedTypes) { - t.Errorf("Expected %d built-in types, got %d", len(expectedTypes), len(builtins)) - } - - for _, typeName := range expectedTypes { - if builtin, exists := builtins[typeName]; !exists { - t.Errorf("Missing built-in type: %s", typeName) - } else { - // Validate builtin type properties - if builtin.Name != typeName { - t.Errorf("Type name mismatch for %s: got %s", typeName, builtin.Name) - } - - // Check numeric types have correct properties - if strings.HasPrefix(typeName, "int") || strings.HasPrefix(typeName, "uint") { - if !builtin.IsNumeric { - t.Errorf("Type %s should be numeric", typeName) - } - if len(builtin.Restrictions) == 0 { - t.Errorf("Numeric type %s should have range restrictions", typeName) - } - } - } - } -} - -func TestRFC6020ParserTokenization(t *testing.T) { - parser := NewRFC6020Parser() - - // Test C-style comment removal (RFC 6020 Section 6.1.1) - yangContent := ` -module test-module { - // This is a single-line comment - yang-version 1.1; - /* This is a - multi-line comment */ - namespace "urn:test:module"; - prefix "test"; -} -` - - tokens, err := parser.TokenizeYANG(yangContent) - if err != nil { - t.Fatalf("Tokenization failed: %v", err) - } - - // Check that we have the expected number of tokens - expectedTokenCount := 13 // Based on actual tokenizer output with proper semicolon separation - if len(tokens) != expectedTokenCount { - t.Errorf("Expected %d tokens, got %d. Tokens: %v", expectedTokenCount, len(tokens), tokens) - } - - // Verify key tokens are present - tokenStr := strings.Join(tokens, " ") - keyTokens := []string{"module", "test-module", "yang-version", "1.1", "namespace", "prefix"} - for _, expected := range keyTokens { - if !strings.Contains(tokenStr, expected) { - t.Errorf("Expected token '%s' not found in tokens", expected) - } - } - - // Verify that no comment content exists in tokens - for _, token := range tokens { - if strings.Contains(token, "//") || strings.Contains(token, "/*") || - strings.Contains(token, "single-line") || strings.Contains(token, "multi-line") { - t.Errorf("Comment content found in tokens: %s", token) - } - } -} - -func TestRFC6020ParserModuleParsing(t *testing.T) { - parser := NewRFC6020Parser() - - // Test basic RFC compliance with simple module - yangContent := `module test-rfc { yang-version 1.1; namespace "urn:test:rfc"; prefix "rfc"; }` - - module, err := parser.ParseYANGModule(yangContent, "test-module.yang") - if err != nil { - t.Fatalf("Module parsing failed: %v", err) - } - - // Validate RFC compliance basics - if module.Name != "test-rfc" { - t.Errorf("Expected module name 'test-rfc', got '%s'", module.Name) - } - - if module.YangVersion != "1.1" { - t.Errorf("Expected yang-version '1.1', got '%s'", module.YangVersion) - } - - // Test RFC built-in types are available - builtinTypes := parser.GetBuiltinTypes() - if len(builtinTypes) != 19 { - t.Errorf("Expected 19 RFC built-in types, got %d", len(builtinTypes)) - } - - // Verify key RFC types exist - requiredTypes := []string{ - "int8", "int16", "int32", "int64", "uint8", "uint16", "uint32", "uint64", - "decimal64", "string", "boolean", "enumeration", "bits", "binary", "leafref", "identityref", - "empty", "union", "instance-identifier", - } - - for _, typeName := range requiredTypes { - if _, exists := builtinTypes[typeName]; !exists { - t.Errorf("Missing required RFC built-in type: %s", typeName) - } - } -} - -func TestRFC6020SemanticAnalysis(t *testing.T) { - parser := NewRFC6020Parser() - - // Test semantic analysis for counter vs gauge classification - yangContent := ` -module test-semantics { - yang-version 1.1; - namespace "urn:test:semantics"; - prefix "sem"; - - container stats { - config false; - - leaf byte-counter { - type uint64; - units "bytes"; - description "Byte counter - should be classified as counter"; - } - - leaf packet-rate { - type uint32; - units "packets-per-second"; - description "Packet rate - should be classified as gauge"; - } - - leaf cpu-percent { - type uint8 { - range "0..100"; - } - units "percent"; - description "CPU utilization - should be classified as gauge"; - } - - leaf error-count { - type uint32; - units "errors"; - description "Error count - should be classified as counter"; - } - - leaf interface-name { - type string; - description "Interface name - should be info"; - } - } -} -` - - module, err := parser.ParseYANGModule(yangContent, "test-semantics.yang") - if err != nil { - t.Fatalf("Module parsing failed: %v", err) - } - - // Check counter classification - expectedCounters := []string{"/stats/byte-counter", "/stats/error-count"} - for _, path := range expectedCounters { - found := slices.Contains(module.Counters, path) - if !found { - t.Errorf("Expected counter path %s not found in counters: %v", path, module.Counters) - } - - // Check data type classification - if dataType, exists := module.DataTypes[path]; exists { - if !dataType.IsCounter { - t.Errorf("Path %s should be classified as counter", path) - } - if dataType.SemanticType != "counter" { - t.Errorf("Path %s should have semantic type 'counter', got '%s'", path, dataType.SemanticType) - } - } - } - - // Check gauge classification - expectedGauges := []string{"/stats/packet-rate", "/stats/cpu-percent"} - for _, path := range expectedGauges { - found := slices.Contains(module.Gauges, path) - if !found { - t.Errorf("Expected gauge path %s not found in gauges: %v", path, module.Gauges) - } - - // Check data type classification - if dataType, exists := module.DataTypes[path]; exists { - if !dataType.IsGauge { - t.Errorf("Path %s should be classified as gauge", path) - } - if dataType.SemanticType != "gauge" { - t.Errorf("Path %s should have semantic type 'gauge', got '%s'", path, dataType.SemanticType) - } - } - } - - // Check info classification - infoPath := "/stats/interface-name" - if dataType, exists := module.DataTypes[infoPath]; exists { - if dataType.SemanticType != "info" { - t.Errorf("Path %s should have semantic type 'info', got '%s'", infoPath, dataType.SemanticType) - } - } - - // Validate config vs state classification - for path, dataType := range module.DataTypes { - if strings.HasPrefix(path, "/stats/") { - if dataType.IsConfiguration { - t.Errorf("Path %s under stats should not be configuration data", path) - } - if !dataType.IsState { - t.Errorf("Path %s under stats should be state data", path) - } - } - } -} - -func TestRFC6020TypeResolution(t *testing.T) { - parser := NewRFC6020Parser() - - yangContent := ` -module test-types { - yang-version 1.1; - namespace "urn:test:types"; - prefix "types"; - - typedef custom-string { - type string { - length "1..255"; - pattern "[A-Za-z0-9_-]+"; - } - description "Custom string type with restrictions"; - } - - typedef bandwidth-type { - type uint64; - units "bits-per-second"; - description "Bandwidth in bits per second"; - } - - container test-data { - leaf name { - type custom-string; - description "Name field using custom string type"; - } - - leaf speed { - type bandwidth-type; - description "Interface speed"; - } - - leaf utilization { - type decimal64 { - fraction-digits 3; - range "0.000..100.000"; - } - units "percent"; - description "Utilization percentage with 3 decimal places"; - } - } -} -` - - module, err := parser.ParseYANGModule(yangContent, "test-types.yang") - if err != nil { - t.Fatalf("Module parsing failed: %v", err) - } - - // Test custom-string typedef resolution - if dataType, exists := module.DataTypes["/test-data/name"]; exists { - if dataType.BaseBuiltinType != "string" { - t.Errorf("Expected base builtin type 'string', got '%s'", dataType.BaseBuiltinType) - } - if dataType.OriginalType != "custom-string" { - t.Errorf("Expected original type 'custom-string', got '%s'", dataType.OriginalType) - } - } else { - t.Error("Missing data type for /test-data/name") - } - - // Test bandwidth-type typedef resolution - if dataType, exists := module.DataTypes["/test-data/speed"]; exists { - if dataType.BaseBuiltinType != "uint64" { - t.Errorf("Expected base builtin type 'uint64', got '%s'", dataType.BaseBuiltinType) - } - if dataType.Units != "bits-per-second" { - t.Errorf("Expected units 'bits-per-second', got '%s'", dataType.Units) - } - } else { - t.Error("Missing data type for /test-data/speed") - } - - // Test decimal64 with fraction-digits - if dataType, exists := module.DataTypes["/test-data/utilization"]; exists { - if dataType.BaseBuiltinType != "decimal64" { - t.Errorf("Expected base builtin type 'decimal64', got '%s'", dataType.BaseBuiltinType) - } - if dataType.FractionDigits != 3 { - t.Errorf("Expected fraction digits 3, got %d", dataType.FractionDigits) - } - } else { - t.Error("Missing data type for /test-data/utilization") - } -} - -func TestRFC6020ExportImport(t *testing.T) { - parser := NewRFC6020Parser() - - // Parse a simple module - yangContent := ` -module test-export { - yang-version 1.1; - namespace "urn:test:export"; - prefix "exp"; - - container test { - leaf value { - type uint32; - units "count"; - } - } -} -` - - _, err := parser.ParseYANGModule(yangContent, "test-export.yang") - if err != nil { - t.Fatalf("Module parsing failed: %v", err) - } - - // Test export - jsonData, err := parser.ExportModules() - if err != nil { - t.Fatalf("Export failed: %v", err) - } - - if len(jsonData) == 0 { - t.Error("Exported JSON data is empty") - } - - // Verify JSON contains expected module data - jsonStr := string(jsonData) - if !strings.Contains(jsonStr, "test-export") { - t.Error("Exported JSON does not contain module name") - } - - if !strings.Contains(jsonStr, "urn:test:export") { - t.Error("Exported JSON does not contain namespace") - } - - // Test save to file - tmpFile := filepath.Join(t.TempDir(), "test-modules.json") - err = parser.SaveModules(tmpFile) - if err != nil { - t.Fatalf("Save modules failed: %v", err) - } - - // Clean up - // Remove temp file - ignore error - _ = parser.modules["test-export"] // Just to avoid unused -} - -func TestRFC6020ComplianceValidation(t *testing.T) { - parser := NewRFC6020Parser() - - // Test various RFC compliance aspects - tests := []struct { - name string - yangContent string - shouldFail bool - description string - }{ - { - name: "valid_module_structure", - yangContent: ` -module valid-test { - yang-version 1.1; - namespace "urn:valid:test"; - prefix "val"; - description "Valid module structure"; -}`, - shouldFail: false, - description: "Basic valid module should parse successfully", - }, - { - name: "yang_version_11", - yangContent: ` -module yang11-test { - yang-version 1.1; - namespace "urn:yang11:test"; - prefix "y11"; -}`, - shouldFail: false, - description: "YANG 1.1 version should be supported", - }, - { - name: "complex_data_model", - yangContent: ` -module complex-test { - yang-version 1.1; - namespace "urn:complex:test"; - prefix "complex"; - - typedef custom-type { - type string { - pattern "[A-Z][a-z0-9]*"; - } - } - - container system { - list interface { - key "name type"; - - leaf name { - type string; - } - - leaf type { - type custom-type; - } - - container statistics { - config false; - - leaf packets { - type uint64; - units "packets"; - } - } - } - } -}`, - shouldFail: false, - description: "Complex data model with multiple keys should work", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - module, err := parser.ParseYANGModule(test.yangContent, test.name+".yang") - - if test.shouldFail { - if err == nil { - t.Errorf("Expected parsing to fail for %s, but it succeeded", test.description) - } - } else { - if err != nil { - t.Errorf("Expected parsing to succeed for %s, but got error: %v", test.description, err) - } else if module == nil { - t.Errorf("Expected non-nil module for %s", test.description) - } - } - }) - } -} diff --git a/receiver/yanggrpcreceiver/metadata.yaml b/receiver/yanggrpcreceiver/metadata.yaml index 158bf7ecb62fc..410e070e7d9f9 100644 --- a/receiver/yanggrpcreceiver/metadata.yaml +++ b/receiver/yanggrpcreceiver/metadata.yaml @@ -14,80 +14,3 @@ status: distributions: [contrib] codeowners: active: [atoulme] - -# Internal telemetry metrics for component observability -telemetry: - metrics: - yang_receiver_bytes_received: - enabled: true - description: Total bytes received from telemetry connections - unit: "By" - sum: - value_type: int - monotonic: true - stability: development - yang_receiver_connections_closed: - enabled: true - description: Number of gRPC connections closed - unit: "{connections}" - sum: - value_type: int - stability: development - yang_receiver_connections_opened: - enabled: true - description: Number of gRPC connections opened - unit: "{connections}" - sum: - value_type: int - stability: development - yang_receiver_grpc_errors: - enabled: true - description: Number of gRPC errors encountered - unit: "{errors}" - sum: - value_type: int - monotonic: true - stability: development - yang_receiver_messages_dropped: - enabled: true - description: Number of telemetry messages dropped due to errors - unit: "{messages}" - sum: - value_type: int - monotonic: true - stability: development - yang_receiver_messages_processed: - enabled: true - description: Number of telemetry messages successfully processed - unit: "{messages}" - sum: - value_type: int - monotonic: true - stability: development - yang_receiver_messages_received: - enabled: true - description: Number of telemetry messages received - unit: "{messages}" - sum: - value_type: int - monotonic: true - stability: development - yang_receiver_processing_duration: - enabled: true - description: Time spent processing telemetry messages - unit: "ms" - histogram: - bucket_boundaries: [ 0.1, 0.5, 1.0, 5.0, 10.0, 50.0, 100.0, 500.0, 1000.0 ] - value_type: double - stability: development - yang_receiver_yang_modules_discovered: - enabled: true - description: Number of unique YANG modules discovered - unit: "{errors}" - sum: - value_type: int - stability: development - -tests: - skip_lifecycle: true - skip_shutdown: true diff --git a/receiver/yanggrpcreceiver/receiver.go b/receiver/yanggrpcreceiver/receiver.go index 1a02671f132ea..71cb7eab98d34 100644 --- a/receiver/yanggrpcreceiver/receiver.go +++ b/receiver/yanggrpcreceiver/receiver.go @@ -15,41 +15,37 @@ import ( "google.golang.org/grpc" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/yanggrpcreceiver/internal" - "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/yanggrpcreceiver/internal/metadata" pb "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/yanggrpcreceiver/internal/proto/generated/proto" ) type yangReceiver struct { - config *Config - settings receiver.Settings - consumer consumer.Metrics - server *grpc.Server - wg sync.WaitGroup - telemetryBuilder *metadata.TelemetryBuilder - securityManager *internal.SecurityManager + config *Config + settings receiver.Settings + logger *zap.Logger + consumer consumer.Metrics + server *grpc.Server + wg sync.WaitGroup + securityManager *internal.SecurityManager } -func createMetricsReceiver(_ context.Context, settings receiver.Settings, cfg component.Config, next consumer.Metrics) (receiver.Metrics, error) { - tb, err := metadata.NewTelemetryBuilder(settings.TelemetrySettings) - if err != nil { - return nil, err - } - +func createMetricsReceiver(_ context.Context, settings receiver.Settings, cfg component.Config, next consumer.Metrics) receiver.Metrics { return &yangReceiver{ - config: cfg.(*Config), - settings: settings, - consumer: next, - telemetryBuilder: tb, - wg: sync.WaitGroup{}, - }, nil + config: cfg.(*Config), + settings: settings, + logger: settings.Logger, + consumer: next, + wg: sync.WaitGroup{}, + } } func (y *yangReceiver) Start(ctx context.Context, host component.Host) error { + // 1. Setup Network Listener listener, err := y.config.NetAddr.Listen(ctx) if err != nil { return err } + // 2. Initialize Security Management (Rate Limiting & Allowlist) y.securityManager = internal.NewSecurityManager( y.config.Security.AllowedClients, y.settings.Logger, @@ -58,6 +54,8 @@ func (y *yangReceiver) Start(ctx context.Context, host component.Host) error { y.config.Security.RateLimiting.BurstSize, y.config.Security.RateLimiting.CleanupInterval, ) + + // 3. Configure gRPC Server with Security Interceptors server, err := y.config.ToServer(ctx, host.GetExtensions(), y.settings.TelemetrySettings, configgrpc.WithGrpcServerOption(grpc.UnaryInterceptor(y.securityManager.CreateSecurityInterceptor()))) if err != nil { @@ -65,22 +63,31 @@ func (y *yangReceiver) Start(ctx context.Context, host component.Host) error { } y.server = server - // Initialize YANG parser with builtin modules + // 4. Initialize YANG Parsers + // Standard Parser for structural analysis yangParser := internal.NewYANGParser() yangParser.LoadBuiltinModules() - // Initialize RFC 6020/7950 compliant YANG parser - rfcYangParser := internal.NewRFC6020Parser() + // Load external Cisco/IETF modules from configured paths + // This enables the "No Omission" guarantee for dimensions + for _, path := range y.config.YANG.ModulePaths { + y.settings.Logger.Info("Loading YANG modules from path", zap.String("path", path)) + if err := yangParser.ExtractYANGFromFiles(path); err != nil { + y.settings.Logger.Error("Failed to load YANG modules", zap.String("path", path), zap.Error(err)) + } + } - // Register the gRPC service for Cisco telemetry + // 5. Register the Dial-out Service service := &grpcService{ - receiver: y, - yangParser: yangParser, - rfcYangParser: rfcYangParser, + receiver: y, + yangParser: yangParser, } pb.RegisterGRPCMdtDialoutServer(y.server, service) + // 6. Start Serving y.wg.Go(func() { + y.settings.Logger.Info("Starting YANG gRPC receiver", + zap.String("endpoint", y.config.NetAddr.Endpoint)) if err := y.server.Serve(listener); err != nil { y.settings.Logger.Error("gRPC server error", zap.Error(err)) } @@ -91,6 +98,7 @@ func (y *yangReceiver) Start(ctx context.Context, host component.Host) error { func (y *yangReceiver) Shutdown(_ context.Context) error { if y.server != nil { + y.settings.Logger.Info("Stopping YANG gRPC receiver") y.server.GracefulStop() } y.wg.Wait() diff --git a/receiver/yanggrpcreceiver/sample_telemetry_test.go b/receiver/yanggrpcreceiver/sample_telemetry_test.go index 3f62d0402d591..94fb1d051aee8 100644 --- a/receiver/yanggrpcreceiver/sample_telemetry_test.go +++ b/receiver/yanggrpcreceiver/sample_telemetry_test.go @@ -6,6 +6,7 @@ package yanggrpcreceiver import ( "context" "encoding/json" + "errors" "fmt" "io" "net" @@ -59,12 +60,9 @@ func TestSampleTelemetryData(t *testing.T) { } ctx := t.Context() - rcvr, err := createMetricsReceiver(ctx, settings, config, consumer) - if err != nil { - t.Fatalf("Failed to create receiver: %v", err) - } + rcvr := createMetricsReceiver(ctx, settings, config, consumer) - err = rcvr.Start(ctx, componenttest.NewNopHost()) + err := rcvr.Start(ctx, componenttest.NewNopHost()) if err != nil { t.Fatalf("Failed to start receiver: %v", err) } @@ -279,7 +277,7 @@ func sendInterfaceTelemetryData(endpoint, nodeID, subscription, encodingPath str } _, err = stream.Recv() - if err != nil && err.Error() != "EOF" { + if !errors.Is(err, io.EOF) { return fmt.Errorf("unexpected error receiving response: %w", err) } diff --git a/receiver/yanggrpcreceiver/yang_parser_test.go b/receiver/yanggrpcreceiver/yang_parser_test.go index d0108088e9245..562ebc8926fee 100644 --- a/receiver/yanggrpcreceiver/yang_parser_test.go +++ b/receiver/yanggrpcreceiver/yang_parser_test.go @@ -43,22 +43,6 @@ func TestYANGParser(t *testing.T) { t.Logf("Analysis result: %+v", analysis) }) - t.Run("TestKeyFieldIdentification", func(t *testing.T) { - encodingPath := "Cisco-IOS-XE-interfaces-oper:interfaces/interface/statistics" - analysis := parser.AnalyzeEncodingPath(encodingPath) - - // Create a mock grpc service to test key identification - service := &grpcService{yangParser: parser} - - // Test that "name" is identified as a key field - isKey := service.isKeyField("name", analysis) - assert.True(t, isKey, "Field 'name' should be identified as a key field") - - // Test that regular statistics fields are not key fields - isKey = service.isKeyField("in-octets", analysis) - assert.False(t, isKey, "Field 'in-octets' should not be identified as a key field") - }) - t.Run("TestBGPPathAnalysis", func(t *testing.T) { encodingPath := "Cisco-IOS-XE-bgp-oper:bgp-state-data/neighbors/neighbor" @@ -130,26 +114,6 @@ func TestYANGParserPersistence(t *testing.T) { }) } -func TestFieldNameExtraction(t *testing.T) { - service := &grpcService{} - - testCases := []struct { - input string - expected string - }{ - {"content.name", "name"}, - {"keys.interface-name", "interface-name"}, - {"statistics.in-octets", "in-octets"}, - {"content.discontinuity-time_info", "discontinuity-time"}, - {"simple", "simple"}, - } - - for _, tc := range testCases { - result := service.extractFieldName(tc.input) - assert.Equal(t, tc.expected, result, "Expected %s but got %s for input %s", tc.expected, result, tc.input) - } -} - // TestYANGEnhancedTelemetryProcessing tests the integration with real telemetry data func TestYANGEnhancedTelemetryProcessing(t *testing.T) { // This test would simulate the enhanced telemetry processing with YANG awareness