Detect yaml groups that are not used in md files, deprecate otel.scope entity

.chloggen/2310.yaml

@@ -0,0 +1,4 @@
+change_type: bug_fix
+component: otel
+note: Removes `otel.scope` entity.
+issues: [2310]

.github/workflows/checks.yml

@@ -126,3 +126,11 @@ jobs:
     - name: verify semantic conventions yaml definitions
       run: make test-policies
 
+  dead-yaml-check:
+    runs-on: ubuntu-latest
+    steps:
+    - name: check out code
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+    - name: find signals defined in yaml files that are not used in the markdown files
+      run: make check-dead-yaml
+

Makefile

@@ -334,3 +334,18 @@ test-policies:
 	/policies \
 	/policies_test
 
+.PHONY: check-dead-yaml
+check-dead-yaml:
+	mkdir -p $(TOOLS_DIR)/bin
+	$(DOCKER_RUN) --rm \
+	$(DOCKER_USER_IS_HOST_USER_ARG) \
+	--mount 'type=bind,source=$(PWD)/internal/tools/scripts,target=/home/weaver/templates,readonly' \
+	--mount 'type=bind,source=$(PWD)/model,target=/home/weaver/source,readonly' \
+	--mount 'type=bind,source=$(TOOLS_DIR)/bin,target=/home/weaver/target' \
+	$(WEAVER_CONTAINER) registry generate \
+		--registry=/home/weaver/source \
+		--templates=/home/weaver/templates \
+		--config=/home/weaver/templates/registry/signal-groups-weaver.yaml \
+		. \
+		/home/weaver/target
+	$(TOOLS_DIR)/scripts/find-dead-yaml.sh $(PWD)/internal/tools/bin/signal-groups.txt $(PWD)/docs

docs/resource/app.md

@@ -0,0 +1,42 @@
+# App
+
+<!-- semconv entity.app -->
+<!-- NOTE: THIS TEXT IS AUTOGENERATED. DO NOT EDIT BY HAND. -->
+<!-- see templates/registry/markdown/snippet.md.j2 -->
+<!-- prettier-ignore-start -->
+<!-- markdownlint-capture -->
+<!-- markdownlint-disable -->
+
+
+**Status:** ![Development](https://img.shields.io/badge/-development-blue)
+
+**type:** `app`
+
+**Description:** An app used directly by end users — like mobile, web, or desktop.
+
+| Attribute  | Type | Description  | Examples  | [Requirement Level](https://opentelemetry.io/docs/specs/semconv/general/attribute-requirement-level/) | Stability |
+|---|---|---|---|---|---|
+| [`app.installation.id`](/docs/registry/attributes/app.md) | string | A unique identifier representing the installation of an application on a specific device [1] | `2ab2916d-a51f-4ac8-80ee-45ac31a28092` | `Recommended` | ![Development](https://img.shields.io/badge/-development-blue) |
+
+**[1] `app.installation.id`:** Its value SHOULD persist across launches of the same application installation, including through application upgrades.
+It SHOULD change if the application is uninstalled or if all applications of the vendor are uninstalled.
+Additionally, users might be able to reset this value (e.g. by clearing application data).
+If an app is installed multiple times on the same device (e.g. in different accounts on Android), each `app.installation.id` SHOULD have a different value.
+If multiple OpenTelemetry SDKs are used within the same application, they SHOULD use the same value for `app.installation.id`.
+Hardware IDs (e.g. serial number, IMEI, MAC address) MUST NOT be used as the `app.installation.id`.
+
+For iOS, this value SHOULD be equal to the [vendor identifier](https://developer.apple.com/documentation/uikit/uidevice/identifierforvendor).
+
+For Android, examples of `app.installation.id` implementations include:
+
+- [Firebase Installation ID](https://firebase.google.com/docs/projects/manage-installations).
+- A globally unique UUID which is persisted across sessions in your application.
+- [App set ID](https://developer.android.com/identity/app-set-id).
+- [`Settings.getString(Settings.Secure.ANDROID_ID)`](https://developer.android.com/reference/android/provider/Settings.Secure#ANDROID_ID).
+
+More information about Android identifier best practices can be found [here](https://developer.android.com/training/articles/user-data-ids).
+
+<!-- markdownlint-restore -->
+<!-- prettier-ignore-end -->
+<!-- END AUTOGENERATED TEXT -->
+<!-- endsemconv -->

docs/system/system-metrics.md

@@ -25,6 +25,7 @@ Resource attributes related to a host, SHOULD be reported under the `host.*` nam
   - [Metric: `system.cpu.physical.count`](#metric-systemcpuphysicalcount)
   - [Metric: `system.cpu.logical.count`](#metric-systemcpulogicalcount)
   - [Metric: `system.cpu.time`](#metric-systemcputime)
+  - [Metric: `system.cpu.frequency`](#metric-systemcpufrequency)
   - [Metric: `system.cpu.utilization`](#metric-systemcpuutilization)
 - [Memory metrics](#memory-metrics)
   - [Metric: `system.memory.usage`](#metric-systemmemoryusage)
@@ -187,6 +188,30 @@ This metric is [recommended][MetricRecommended].
 <!-- END AUTOGENERATED TEXT -->
 <!-- endsemconv -->
 
+### Metric: `system.cpu.frequency`
+
+This metric is [recommended][MetricRecommended].
+
+<!-- semconv metric.system.cpu.frequency -->
+<!-- NOTE: THIS TEXT IS AUTOGENERATED. DO NOT EDIT BY HAND. -->
+<!-- see templates/registry/markdown/snippet.md.j2 -->
+<!-- prettier-ignore-start -->
+<!-- markdownlint-capture -->
+<!-- markdownlint-disable -->
+
+| Name     | Instrument Type | Unit (UCUM) | Description    | Stability | Entity Associations |
+| -------- | --------------- | ----------- | -------------- | --------- | ------ |
+| `system.cpu.frequency` | Gauge | `Hz` | Operating frequency of the logical CPU in Hertz. | ![Development](https://img.shields.io/badge/-development-blue) | `host` |
+
+| Attribute  | Type | Description  | Examples  | [Requirement Level](https://opentelemetry.io/docs/specs/semconv/general/attribute-requirement-level/) | Stability |
+|---|---|---|---|---|---|
+| [`cpu.logical_number`](/docs/registry/attributes/cpu.md) | int | The logical CPU number [0..n-1] | `1` | `Recommended` | ![Development](https://img.shields.io/badge/-development-blue) |
+
+<!-- markdownlint-restore -->
+<!-- prettier-ignore-end -->
+<!-- END AUTOGENERATED TEXT -->
+<!-- endsemconv -->
+
 ### Metric: `system.cpu.utilization`
 
 This metric is [opt-in][MetricOptIn].

internal/tools/scripts/find-dead-yaml.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Description: This script checks if all signal groups declared in a YAML file are present in the markdown files under a specified directory.
+# Usage: ./check_lines.sh <groups_list> <md_directory_path>
+
+set -euo pipefail
+
+groups="$1"
+docs_folder="$2"
+
+if [[ ! -f "$groups" ]]; then
+  echo "File with groups does not exist: $groups"
+  exit 1
+fi
+
+if [[ ! -d "$docs_folder" ]]; then
+  echo "Docs folder does not exist: $docs_folder"
+  exit 1
+fi
+
+declare -A semconv_snippets
+# Extract only lines of the form <!-- semconv ... --> from .md files
+echo "Indexing semconv lines in .md files under: $docs_folder ..."
+while IFS= read -r LINE; do
+  semconv_snippets["$LINE"]=1
+done < <(
+  grep -rhoP '<!--\s*semconv\s+\K[a-z0-9._]+(?=\s*-->)' "$docs_folder" --include="*.md"
+)
+
+not_found_groups=()
+
+while IFS= read -r LINE || [[ -n "$LINE" ]]; do
+  if [[ -z "${semconv_snippets[$LINE]+_}" ]]; then
+    not_found_groups+=("$LINE")
+  fi
+done < "$groups"
+
+echo
+
+if [[ ${#not_found_groups[@]} -gt 0 ]]; then
+  echo "❌ The following signals were declared in yaml and NOT found in markdown:"
+  for line in "${not_found_groups[@]}"; do
+    echo " - $line"
+  done
+  exit 1
+fi
+
+echo "All groups were found."

internal/tools/scripts/registry/signal-groups-weaver.yaml

@@ -0,0 +1,12 @@
+templates:
+  - pattern: signal-groups.j2
+    filter: >
+      .groups
+      | map(select(.type != "attribute_group"))
+      | map(select(has("deprecated") | not))
+      | .[].id
+    application_mode: single
+whitespace_control:
+  trim_blocks: true
+  lstrip_blocks: true
+  keep_trailing_newline: true

internal/tools/scripts/registry/signal-groups.j2

@@ -0,0 +1,5 @@
+{{- template.set_file_name("signal-groups.txt") -}}
+
+{% for item in ctx | sort %}
+{{ item }}
+{% endfor %}

model/otel/entities.yaml → model/otel/deprecated/entities-deprecated.yaml

@@ -4,6 +4,8 @@ groups:
     type: entity
     stability: development
     name: otel.scope
+    deprecated:
+      reason: obsoleted
     brief: Attributes used by non-OTLP exporters to represent OpenTelemetry Scope's concepts.
     attributes:
       - ref: otel.scope.name