Detect yaml groups that are not used in md files, deprecate otel.scope entity

.chloggen/2310.yaml

@@ -0,0 +1,4 @@
+change_type: bug_fix
+component: otel
+note: Removes `otel.scope` entity.
+issues: [2310]

.github/workflows/checks.yml

@@ -126,3 +126,11 @@ jobs:
     - name: verify semantic conventions yaml definitions
       run: make test-policies
 
+  dead-yaml-check:
+    runs-on: ubuntu-latest
+    steps:
+    - name: check out code
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+    - name: find signals defined in yaml files that are not used in the markdown files
+      run: make check-dead-yaml
+

Makefile

@@ -334,3 +334,18 @@ test-policies:
 	/policies \
 	/policies_test
 
+.PHONY: check-dead-yaml
+check-dead-yaml:
+	mkdir -p $(TOOLS_DIR)/bin
+	$(DOCKER_RUN) --rm \
+	$(DOCKER_USER_IS_HOST_USER_ARG) \
+	--mount 'type=bind,source=$(PWD)/internal/tools/scripts,target=/home/weaver/templates,readonly' \
+	--mount 'type=bind,source=$(PWD)/model,target=/home/weaver/source,readonly' \
+	--mount 'type=bind,source=$(TOOLS_DIR)/bin,target=/home/weaver/target' \
+	$(WEAVER_CONTAINER) registry generate \
+		--registry=/home/weaver/source \
+		--templates=/home/weaver/templates \
+		--config=/home/weaver/templates/registry/signal-groups-weaver.yaml \
+		. \
+		/home/weaver/target
+	$(TOOLS_DIR)/scripts/find-dead-yaml.sh $(PWD)/internal/tools/bin/signal-groups.txt $(PWD)/docs

docs/system/system-metrics.md

@@ -25,6 +25,7 @@ Resource attributes related to a host, SHOULD be reported under the `host.*` nam
   - [Metric: `system.cpu.physical.count`](#metric-systemcpuphysicalcount)
   - [Metric: `system.cpu.logical.count`](#metric-systemcpulogicalcount)
   - [Metric: `system.cpu.time`](#metric-systemcputime)
+  - [Metric: `system.cpu.frequency`](#metric-systemcpufrequency)
   - [Metric: `system.cpu.utilization`](#metric-systemcpuutilization)
 - [Memory metrics](#memory-metrics)
   - [Metric: `system.memory.usage`](#metric-systemmemoryusage)
@@ -187,6 +188,30 @@ This metric is [recommended][MetricRecommended].
 <!-- END AUTOGENERATED TEXT -->
 <!-- endsemconv -->
 
+### Metric: `system.cpu.frequency`
+
+This metric is [recommended][MetricRecommended].
+
+<!-- semconv metric.system.cpu.frequency -->
+<!-- NOTE: THIS TEXT IS AUTOGENERATED. DO NOT EDIT BY HAND. -->
+<!-- see templates/registry/markdown/snippet.md.j2 -->
+<!-- prettier-ignore-start -->
+<!-- markdownlint-capture -->
+<!-- markdownlint-disable -->
+
+| Name     | Instrument Type | Unit (UCUM) | Description    | Stability | Entity Associations |
+| -------- | --------------- | ----------- | -------------- | --------- | ------ |
+| `system.cpu.frequency` | Gauge | `Hz` | Operating frequency of the logical CPU in Hertz. | ![Development](https://img.shields.io/badge/-development-blue) | `host` |
+
+| Attribute  | Type | Description  | Examples  | [Requirement Level](https://opentelemetry.io/docs/specs/semconv/general/attribute-requirement-level/) | Stability |
+|---|---|---|---|---|---|
+| [`cpu.logical_number`](/docs/registry/attributes/cpu.md) | int | The logical CPU number [0..n-1] | `1` | `Recommended` | ![Development](https://img.shields.io/badge/-development-blue) |
+
+<!-- markdownlint-restore -->
+<!-- prettier-ignore-end -->
+<!-- END AUTOGENERATED TEXT -->
+<!-- endsemconv -->
+
 ### Metric: `system.cpu.utilization`
 
 This metric is [opt-in][MetricOptIn].

internal/tools/scripts/find-dead-yaml.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Description: This script checks if all signal groups declared in a YAML file are present in the markdown files under a specified directory.
+# Usage: ./check_lines.sh <groups_list> <md_directory_path>
+
+set -euo pipefail
+
+groups="$1"
+docs_folder="$2"
+
+if [[ ! -f "$groups" ]]; then
+  echo "File with groups does not exist: $groups"
+  exit 1
+fi
+
+if [[ ! -d "$docs_folder" ]]; then
+  echo "Docs folder does not exist: $docs_folder"
+  exit 1
+fi
+
+declare -A semconv_snippets
+# Extract only lines of the form <!-- semconv ... --> from .md files
+echo "Indexing semconv lines in .md files under: $docs_folder ..."
+while IFS= read -r LINE; do
+  semconv_snippets["$LINE"]=1
+done < <(
+  grep -rhoP '<!--\s*semconv\s+\K[a-z0-9._]+(?=\s*-->)' "$docs_folder" --include="*.md"
+)
+
+not_found_groups=()
+
+while IFS= read -r LINE || [[ -n "$LINE" ]]; do
+  if [[ -z "${semconv_snippets[$LINE]+_}" ]]; then
+    not_found_groups+=("$LINE")
+  fi
+done < "$groups"
+
+echo
+
+if [[ ${#not_found_groups[@]} -gt 0 ]]; then
+  echo "❌ The following signals were declared in yaml and NOT found in markdown:"
+  for line in "${not_found_groups[@]}"; do
+    echo " - $line"
+  done
+  exit 1
+fi
+
+echo "All groups were found."

internal/tools/scripts/registry/signal-groups-weaver.yaml

@@ -0,0 +1,14 @@
+templates:
+  - pattern: signal-groups.j2
+    # attribute groups and entities are auto-generated and don't need manual code
+    # snippets, let's filter them out from the check.
+    filter: >
+      .groups
+      | map(select(.type != "attribute_group" and .type != "entity"))
+      | map(select(has("deprecated") | not))
+      | .[].id
+    application_mode: single
+whitespace_control:
+  trim_blocks: true
+  lstrip_blocks: true
+  keep_trailing_newline: true

internal/tools/scripts/registry/signal-groups.j2

@@ -0,0 +1,5 @@
+{{- template.set_file_name("signal-groups.txt") -}}
+
+{% for item in ctx | sort %}
+{{ item }}
+{% endfor %}

model/otel/entities.yaml → model/otel/deprecated/entities-deprecated.yaml

@@ -4,6 +4,8 @@ groups:
     type: entity
     stability: development
     name: otel.scope
+    deprecated:
+      reason: obsoleted
     brief: Attributes used by non-OTLP exporters to represent OpenTelemetry Scope's concepts.
     attributes:
       - ref: otel.scope.name