Skip to content

[8.19] [APM][Infra] Fix OTel metrics mapping in infrastructure tab (#259552) #261020

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rmyz merged 2 commits into from
Apr 3, 2026
Merged

[8.19] [APM][Infra] Fix OTel metrics mapping in infrastructure tab (#259552) #261020

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1ad45f5
[APM][Infra] Fix OTel metrics mapping in infrastructure tab (#259552)
rmyz Apr 2, 2026
a4adb00
Changes from node scripts/eslint_all_files --no-cache --fix
kibanamachine Apr 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 16 additions & 21 deletions ...components/infrastructure_node_metrics_tables/container/container_metrics_configs.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,11 @@
import {
ECS_CONTAINER_CPU_USAGE_LIMIT_PCT,
ECS_CONTAINER_MEMORY_USAGE_BYTES,
otelDatasetFilterDsl,
SEMCONV_DOCKER_CONTAINER_CPU_UTILIZATION,
SEMCONV_DOCKER_CONTAINER_MEMORY_PERCENT,
SEMCONV_K8S_CONTAINER_CPU_LIMIT_UTILIZATION,
SEMCONV_K8S_CONTAINER_MEMORY_LIMIT_UTILIZATION,
SEMCONV_CONTAINER_CPU_USAGE,
SEMCONV_CONTAINER_MEMORY_WORKING_SET,
} from '../shared/constants';
import { getOptionsForSchema } from './container_metrics_configs';

Expand Down Expand Up @@ -39,14 +40,14 @@ describe('container_metrics_configs', () => {

it('returns SemConv Docker options when isOtel is true and isK8sContainer is false', () => {
const { options } = getOptionsForSchema(true, false);
const filterClauseDsl = {
const expectedFilter = {
bool: {
filter: [{ term: { 'event.dataset': 'dockerstatsreceiver.otel' } }],
filter: [otelDatasetFilterDsl('dockerstatsreceiver.otel')],
},
};

expect(options.groupBy).toBe('container.id');
expect(options.filterQuery).toBe(JSON.stringify(filterClauseDsl));
expect(options.filterQuery).toBe(JSON.stringify(expectedFilter));
expect(options.metrics).toEqual(
expect.arrayContaining([
{ field: SEMCONV_DOCKER_CONTAINER_CPU_UTILIZATION, aggregation: 'avg' },
Expand All @@ -58,13 +59,13 @@ describe('container_metrics_configs', () => {

it('returns SemConv Docker options when isOtel is true and isK8sContainer is undefined', () => {
const { options } = getOptionsForSchema(true);
const filterClauseDsl = {
const expectedFilter = {
bool: {
filter: [{ term: { 'event.dataset': 'dockerstatsreceiver.otel' } }],
filter: [otelDatasetFilterDsl('dockerstatsreceiver.otel')],
},
};
expect(options.groupBy).toBe('container.id');
expect(options.filterQuery).toBe(JSON.stringify(filterClauseDsl));
expect(options.filterQuery).toBe(JSON.stringify(expectedFilter));
expect(options.metrics).toEqual(
expect.arrayContaining([
{ field: SEMCONV_DOCKER_CONTAINER_CPU_UTILIZATION, aggregation: 'avg' },
Expand All @@ -76,17 +77,17 @@ describe('container_metrics_configs', () => {

it('returns SemConv K8s options when isOtel is true and isK8sContainer is true', () => {
const { options } = getOptionsForSchema(true, true);
const filterClauseDsl = {
const expectedFilter = {
bool: {
filter: [{ term: { 'event.dataset': 'kubeletstatsreceiver.otel' } }],
filter: [otelDatasetFilterDsl('kubeletstatsreceiver.otel')],
},
};
expect(options.groupBy).toBe('container.id');
expect(options.filterQuery).toBe(JSON.stringify(filterClauseDsl));
expect(options.filterQuery).toBe(JSON.stringify(expectedFilter));
expect(options.metrics).toEqual(
expect.arrayContaining([
{ field: SEMCONV_K8S_CONTAINER_CPU_LIMIT_UTILIZATION, aggregation: 'avg' },
{ field: SEMCONV_K8S_CONTAINER_MEMORY_LIMIT_UTILIZATION, aggregation: 'avg' },
{ field: SEMCONV_CONTAINER_CPU_USAGE, aggregation: 'avg' },
{ field: SEMCONV_CONTAINER_MEMORY_WORKING_SET, aggregation: 'avg' },
])
);
expect(options.metrics).toHaveLength(2);
Expand Down Expand Up @@ -119,10 +120,7 @@ describe('container_metrics_configs', () => {

const filterClauseWithEventModuleFilter = {
bool: {
filter: [
{ term: { 'event.dataset': 'dockerstatsreceiver.otel' } },
{ ...filterClauseDsl },
],
filter: [otelDatasetFilterDsl('dockerstatsreceiver.otel'), { ...filterClauseDsl }],
},
};

Expand All @@ -139,10 +137,7 @@ describe('container_metrics_configs', () => {
};
const filterClauseWithEventModuleFilter = {
bool: {
filter: [
{ term: { 'event.dataset': 'kubeletstatsreceiver.otel' } },
{ ...filterClauseDsl },
],
filter: [otelDatasetFilterDsl('kubeletstatsreceiver.otel'), { ...filterClauseDsl }],
},
};

Expand Down
33 changes: 14 additions & 19 deletions ...blic/components/infrastructure_node_metrics_tables/container/container_metrics_configs.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,11 @@ import { createMetricByFieldLookup, makeUnpackMetric, metricsToApiOptions } from
import {
ECS_CONTAINER_CPU_USAGE_LIMIT_PCT,
ECS_CONTAINER_MEMORY_USAGE_BYTES,
otelDatasetFilterDsl,
SEMCONV_DOCKER_CONTAINER_MEMORY_PERCENT,
SEMCONV_DOCKER_CONTAINER_CPU_UTILIZATION,
SEMCONV_K8S_CONTAINER_CPU_LIMIT_UTILIZATION,
SEMCONV_K8S_CONTAINER_MEMORY_LIMIT_UTILIZATION,
SEMCONV_CONTAINER_CPU_USAGE,
SEMCONV_CONTAINER_MEMORY_WORKING_SET,
} from '../shared/constants';

// --- ECS (Elastic Common Schema) ---
Expand Down Expand Up @@ -48,11 +49,7 @@ type ContainerMetricsFieldSemconvDocker =

const containerMetricsQueryConfigSemconvDocker: MetricsQueryOptions<ContainerMetricsFieldSemconvDocker> =
{
sourceFilter: {
term: {
'event.dataset': 'dockerstatsreceiver.otel',
},
},
sourceFilter: otelDatasetFilterDsl('dockerstatsreceiver.otel'),
groupByField: 'container.id',
metricsMap: {
[SEMCONV_DOCKER_CONTAINER_CPU_UTILIZATION]: {
Expand All @@ -67,26 +64,24 @@ const containerMetricsQueryConfigSemconvDocker: MetricsQueryOptions<ContainerMet
};

// --- SemConv K8s (Kubernetes container metrics) ---
// Uses generic container metrics always emitted by kubeletstats,
// regardless of whether k8s resource limits are configured.
type ContainerMetricsFieldSemconvK8s =
| typeof SEMCONV_K8S_CONTAINER_CPU_LIMIT_UTILIZATION
| typeof SEMCONV_K8S_CONTAINER_MEMORY_LIMIT_UTILIZATION;
| typeof SEMCONV_CONTAINER_CPU_USAGE
| typeof SEMCONV_CONTAINER_MEMORY_WORKING_SET;

const containerMetricsQueryConfigSemconvK8s: MetricsQueryOptions<ContainerMetricsFieldSemconvK8s> =
{
sourceFilter: {
term: {
'event.dataset': 'kubeletstatsreceiver.otel',
},
},
sourceFilter: otelDatasetFilterDsl('kubeletstatsreceiver.otel'),
groupByField: 'container.id',
metricsMap: {
[SEMCONV_K8S_CONTAINER_CPU_LIMIT_UTILIZATION]: {
[SEMCONV_CONTAINER_CPU_USAGE]: {
aggregation: 'avg',
field: SEMCONV_K8S_CONTAINER_CPU_LIMIT_UTILIZATION,
field: SEMCONV_CONTAINER_CPU_USAGE,
},
[SEMCONV_K8S_CONTAINER_MEMORY_LIMIT_UTILIZATION]: {
[SEMCONV_CONTAINER_MEMORY_WORKING_SET]: {
aggregation: 'avg',
field: SEMCONV_K8S_CONTAINER_MEMORY_LIMIT_UTILIZATION,
field: SEMCONV_CONTAINER_MEMORY_WORKING_SET,
},
},
};
Expand All @@ -101,7 +96,7 @@ const metricByFieldSemconvDocker = createMetricByFieldLookup(
);
export const unpackMetricSemconvDocker = makeUnpackMetric(metricByFieldSemconvDocker);

const metricByFieldSemconvK8s = createMetricByFieldLookup(
export const metricByFieldSemconvK8s = createMetricByFieldLookup(
containerMetricsQueryConfigSemconvK8s.metricsMap
);
export const unpackMetricSemconvK8s = makeUnpackMetric(metricByFieldSemconvK8s);
Expand Down
2 changes: 1 addition & 1 deletion ...ublic/components/infrastructure_node_metrics_tables/container/container_metrics_table.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ function containerNodeColumns({
ContainerMetricsTableProps,
'timerange' | 'isOtel' | 'metricsIndices' | 'isK8sContainer'
>): Array<EuiBasicTableColumn<ContainerNodeMetricsRow>> {
const memoryUnit = isOtel ? '%' : ' MB';
const memoryUnit = isOtel && !isK8sContainer ? '%' : ' MB';
return [
{
name: i18n.translate('xpack.metricsData.metricsTable.container.idColumnHeader', {
Expand Down
16 changes: 7 additions & 9 deletions ...mponents/infrastructure_node_metrics_tables/container/use_container_metrics_table.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,11 @@
import {
ECS_CONTAINER_CPU_USAGE_LIMIT_PCT,
ECS_CONTAINER_MEMORY_USAGE_BYTES,
otelDatasetFilterDsl,
SEMCONV_DOCKER_CONTAINER_CPU_UTILIZATION,
SEMCONV_DOCKER_CONTAINER_MEMORY_PERCENT,
SEMCONV_K8S_CONTAINER_CPU_LIMIT_UTILIZATION,
SEMCONV_K8S_CONTAINER_MEMORY_LIMIT_UTILIZATION,
SEMCONV_CONTAINER_CPU_USAGE,
SEMCONV_CONTAINER_MEMORY_WORKING_SET,
} from '../shared/constants';
import { useContainerMetricsTable } from './use_container_metrics_table';
import { useInfrastructureNodeMetrics } from '../shared';
Expand Down Expand Up @@ -74,7 +75,7 @@ describe('useContainerMetricsTable hook', () => {

const filterClauseWithEventModuleFilter = {
bool: {
filter: [{ term: { 'event.dataset': 'dockerstatsreceiver.otel' } }, { ...filterClauseDsl }],
filter: [otelDatasetFilterDsl('dockerstatsreceiver.otel'), { ...filterClauseDsl }],
},
};
useInfrastructureNodeMetricsMock.mockReturnValue({
Expand Down Expand Up @@ -120,10 +121,7 @@ describe('useContainerMetricsTable hook', () => {

const filterClauseWithEventModuleFilter = {
bool: {
filter: [
{ term: { 'event.dataset': 'kubeletstatsreceiver.otel' } },
{ ...filterClauseDsl },
],
filter: [otelDatasetFilterDsl('kubeletstatsreceiver.otel'), { ...filterClauseDsl }],
},
};

Expand All @@ -143,10 +141,10 @@ describe('useContainerMetricsTable hook', () => {
filterQuery: JSON.stringify(filterClauseWithEventModuleFilter),
metrics: expect.arrayContaining([
expect.objectContaining({
field: SEMCONV_K8S_CONTAINER_CPU_LIMIT_UTILIZATION,
field: SEMCONV_CONTAINER_CPU_USAGE,
}),
expect.objectContaining({
field: SEMCONV_K8S_CONTAINER_MEMORY_LIMIT_UTILIZATION,
field: SEMCONV_CONTAINER_MEMORY_WORKING_SET,
}),
]),
}),
Expand Down
13 changes: 6 additions & 7 deletions ...ic/components/infrastructure_node_metrics_tables/container/use_container_metrics_table.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ import {
ECS_CONTAINER_MEMORY_USAGE_BYTES,
SEMCONV_DOCKER_CONTAINER_CPU_UTILIZATION,
SEMCONV_DOCKER_CONTAINER_MEMORY_PERCENT,
SEMCONV_K8S_CONTAINER_CPU_LIMIT_UTILIZATION,
SEMCONV_K8S_CONTAINER_MEMORY_LIMIT_UTILIZATION,
SEMCONV_CONTAINER_CPU_USAGE,
SEMCONV_CONTAINER_MEMORY_WORKING_SET,
} from '../shared/constants';

export { metricByFieldEcs, metricByField } from './container_metrics_configs';
Expand Down Expand Up @@ -52,12 +52,11 @@ const semconvDockerUnpackMetrics = (row: MetricsExplorerRow) => {
};

const semconvK8sUnpackMetrics = (row: MetricsExplorerRow) => {
// semconv k8s unpack metrics
const cpuUtilization = unpackMetricSemconvK8s(row, SEMCONV_K8S_CONTAINER_CPU_LIMIT_UTILIZATION);
const memoryUsage = unpackMetricSemconvK8s(row, SEMCONV_K8S_CONTAINER_MEMORY_LIMIT_UTILIZATION);
const cpuUsage = unpackMetricSemconvK8s(row, SEMCONV_CONTAINER_CPU_USAGE);
const memoryBytes = unpackMetricSemconvK8s(row, SEMCONV_CONTAINER_MEMORY_WORKING_SET);
return {
averageCpuUsage: cpuUtilization !== null ? scaleUpPercentage(cpuUtilization) : null,
averageMemoryUsage: memoryUsage !== null ? scaleUpPercentage(memoryUsage) : null,
averageCpuUsage: cpuUsage !== null ? scaleUpPercentage(cpuUsage) : null,
averageMemoryUsage: memoryBytes !== null ? Math.floor(memoryBytes / 1_000_000) : null,
};
};

Expand Down
98 changes: 95 additions & 3 deletions .../public/components/infrastructure_node_metrics_tables/host/use_host_metrics_table.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,10 @@
*/

import {
otelDatasetFilterDsl,
SEMCONV_SYSTEM_CPU_LOGICAL_COUNT,
SEMCONV_SYSTEM_CPU_UTILIZATION,
SEMCONV_SYSTEM_MEMORY_LIMIT,
SEMCONV_SYSTEM_MEMORY_TOTAL,
SEMCONV_SYSTEM_MEMORY_UTILIZATION,
SYSTEM_CPU_CORES,
SYSTEM_CPU_TOTAL_NORM_PCT,
Expand Down Expand Up @@ -83,7 +84,7 @@ describe('useHostMetricsTable hook', () => {

const filterClauseWithEventModuleFilter = {
bool: {
filter: [{ term: { 'event.dataset': 'hostmetricsreceiver.otel' } }, { ...filterClauseDsl }],
filter: [otelDatasetFilterDsl('hostmetricsreceiver.otel'), { ...filterClauseDsl }],
},
};

Expand All @@ -110,14 +111,105 @@ describe('useHostMetricsTable hook', () => {
metrics: expect.arrayContaining([
expect.objectContaining({ field: SEMCONV_SYSTEM_CPU_LOGICAL_COUNT }),
expect.objectContaining({ field: SEMCONV_SYSTEM_CPU_UTILIZATION }),
expect.objectContaining({ field: SEMCONV_SYSTEM_MEMORY_LIMIT }),
expect.objectContaining({ field: SEMCONV_SYSTEM_MEMORY_TOTAL }),
expect.objectContaining({ field: SEMCONV_SYSTEM_MEMORY_UTILIZATION }),
]),
}),
})
);
});

it('should transform OTel rows into populated host metrics', () => {
useInfrastructureNodeMetricsMock.mockClear();
useInfrastructureNodeMetricsMock.mockReturnValue({
isLoading: true,
data: { state: 'empty-indices' },
metricIndices: 'test-index',
});

renderHook(() =>
useHostMetricsTable({
timerange: { from: 'now-30d', to: 'now' },
metricsClient: createMetricsClientMock({}),
isOtel: true,
})
);

const lastCallArgs = useInfrastructureNodeMetricsMock.mock.calls.at(-1);
const { transform, metricsExplorerOptions } = lastCallArgs?.[0] ?? {};
expect(transform).toBeDefined();

const metrics = metricsExplorerOptions?.metrics ?? [];
const metricIndexByField = new Map(metrics.map((metric, index) => [metric.field, index]));

const row = transform!({
id: 'otel-host-1',
columns: [],
rows: [
{
timestamp: Date.now(),
[`metric_${metricIndexByField.get(SEMCONV_SYSTEM_CPU_LOGICAL_COUNT)}`]: 8,
[`metric_${metricIndexByField.get(SEMCONV_SYSTEM_CPU_UTILIZATION)}`]: 0.42,
[`metric_${metricIndexByField.get(SEMCONV_SYSTEM_MEMORY_TOTAL)}`]: 16_000_000_000,
[`metric_${metricIndexByField.get(SEMCONV_SYSTEM_MEMORY_UTILIZATION)}`]: 0.25,
},
],
});

expect(row).toEqual({
name: 'otel-host-1',
cpuCount: 8,
averageCpuUsagePercent: 42,
totalMemoryMegabytes: 16000,
averageMemoryUsagePercent: 25,
});
});

it('should return null totalMemoryMegabytes when memory total metric is absent (B=0 in equation)', () => {
useInfrastructureNodeMetricsMock.mockClear();
useInfrastructureNodeMetricsMock.mockReturnValue({
isLoading: true,
data: { state: 'empty-indices' },
metricIndices: 'test-index',
});

renderHook(() =>
useHostMetricsTable({
timerange: { from: 'now-30d', to: 'now' },
metricsClient: createMetricsClientMock({}),
isOtel: true,
})
);

const lastCallArgs = useInfrastructureNodeMetricsMock.mock.calls.at(-1);
const { transform, metricsExplorerOptions } = lastCallArgs?.[0] ?? {};
expect(transform).toBeDefined();

const metrics = metricsExplorerOptions?.metrics ?? [];
const metricIndexByField = new Map(metrics.map((metric, index) => [metric.field, index]));

const row = transform!({
id: 'otel-host-2',
columns: [],
rows: [
{
timestamp: Date.now(),
[`metric_${metricIndexByField.get(SEMCONV_SYSTEM_CPU_LOGICAL_COUNT)}`]: 4,
[`metric_${metricIndexByField.get(SEMCONV_SYSTEM_CPU_UTILIZATION)}`]: 0.1,
[`metric_${metricIndexByField.get(SEMCONV_SYSTEM_MEMORY_UTILIZATION)}`]: 0.5,
},
],
});

expect(row).toEqual({
name: 'otel-host-2',
cpuCount: 4,
averageCpuUsagePercent: 10,
totalMemoryMegabytes: null,
averageMemoryUsagePercent: 50,
});
});

it('should call useInfrastructureNodeMetrics with ECS metrics when isOtel is false', () => {
const filterClauseDsl = {
bool: {
Expand Down
Loading
Loading