Skip to content

[APM] Add kibana.alert.grouping to latency threshold alerts #254904

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fkanout merged 4 commits into from
Mar 25, 2026
Merged

[APM] Add kibana.alert.grouping to latency threshold alerts #254904

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e9dd000
[APM Latency threshold rule] Add kibana.alert.grouping field
fkanout Feb 25, 2026
2f5a905
Changes from node scripts/eslint_all_files --no-cache --fix
kibanamachine Feb 25, 2026
1f162ac
code review add MappingDynamicTemplate
fkanout Mar 5, 2026
55b32cb
codereview update tests
fkanout Mar 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion x-pack/solutions/observability/plugins/apm/server/routes/alerts/register_apm_rule_types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,6 @@ export const apmRuleTypeAlertFieldMap = {
},
};

// Defines which alerts-as-data index alerts will use
export const ApmRuleTypeAlertDefinition: IRuleTypeAlerts<ObservabilityApmAlert> = {
context: APM_RULE_TYPE_ALERT_CONTEXT,
mappings: {
Expand Down
17 changes: 17 additions & 0 deletions ...es/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ describe('registerTransactionDurationRuleType', () => {
payload: {
'kibana.alert.evaluation.threshold': 3000000,
'kibana.alert.evaluation.value': 5500000,
'kibana.alert.grouping': expect.anything(),
'kibana.alert.reason':
'Avg. latency is 5.5 s in the last 5 mins for service: opbeans-java, env: development, type: request. Alert when > 3.0 s.',
'processor.event': 'transaction',
Expand Down Expand Up @@ -176,6 +177,7 @@ describe('registerTransactionDurationRuleType', () => {
payload: {
'kibana.alert.evaluation.threshold': 3000000,
'kibana.alert.evaluation.value': 5500000,
'kibana.alert.grouping': expect.anything(),
'kibana.alert.reason':
'Avg. latency is 5.5 s in the last 5 mins for service: opbeans-java, env: development, type: request, name: GET /products. Alert when > 3.0 s.',
'processor.event': 'transaction',
Expand Down Expand Up @@ -266,6 +268,7 @@ describe('registerTransactionDurationRuleType', () => {
payload: {
'kibana.alert.evaluation.threshold': 3000000,
'kibana.alert.evaluation.value': 5500000,
'kibana.alert.grouping': expect.anything(),
'kibana.alert.reason':
'Avg. latency is 5.5 s in the last 5 mins for service: opbeans-java, env: development, type: request. Alert when > 3.0 s.',
'processor.event': 'transaction',
Expand Down Expand Up @@ -356,6 +359,7 @@ describe('registerTransactionDurationRuleType', () => {
payload: {
'kibana.alert.evaluation.threshold': 3000000,
'kibana.alert.evaluation.value': 5500000,
'kibana.alert.grouping': expect.anything(),
'kibana.alert.reason':
'Avg. latency is 5.5 s in the last 5 mins for service: opbeans-java, env: Not defined, type: request, name: tx-java. Alert when > 3.0 s.',
'processor.event': 'transaction',
Expand Down Expand Up @@ -445,6 +449,7 @@ describe('registerTransactionDurationRuleType', () => {
payload: {
'kibana.alert.evaluation.threshold': 3000000,
'kibana.alert.evaluation.value': 5500000,
'kibana.alert.grouping': expect.anything(),
'kibana.alert.reason':
'Avg. latency is 5.5 s in the last 5 mins for service: opbeans-java, env: All, type: request, name: tx-java. Alert when > 3.0 s.',
'processor.event': 'transaction',
Expand Down Expand Up @@ -541,6 +546,7 @@ describe('registerTransactionDurationRuleType', () => {
payload: {
'kibana.alert.evaluation.threshold': 3000000,
'kibana.alert.evaluation.value': 5500000,
'kibana.alert.grouping': expect.anything(),
'kibana.alert.reason':
'Avg. latency is 5.5 s in the last 5 mins for service: opbeans-java, env: development, type: request. Alert when > 3.0 s.',
'processor.event': 'transaction',
Expand Down Expand Up @@ -596,6 +602,16 @@ describe('registerTransactionDurationRuleType', () => {
'processor.event': 'transaction',
'kibana.alert.evaluation.value': 1000000,
'kibana.alert.evaluation.threshold': 149000,
'kibana.alert.grouping': {
service: {
environment: 'Synthtrace: many_errors',
name: 'synthtrace-high-cardinality-0',
},
transaction: {
name: 'from-recovered-hit',
type: 'request',
},
},
'kibana.alert.reason':
'Avg. latency is 1,000 ms in the last 5 days for service: synthtrace-high-cardinality-0, env: Synthtrace: many_errors, type: request. Alert when > 149 ms.',
'agent.name': 'java',
Expand Down Expand Up @@ -667,6 +683,7 @@ describe('registerTransactionDurationRuleType', () => {
name: 'synthtrace-high-cardinality-0',
},
transaction: {
name: 'from-recovered-hit',
type: 'request',
},
},
Expand Down
7 changes: 5 additions & 2 deletions .../routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import { getParsedFilterQuery, termQuery } from '@kbn/observability-plugin/serve
import {
ALERT_EVALUATION_THRESHOLD,
ALERT_EVALUATION_VALUE,
ALERT_GROUPING,
ALERT_INDEX_PATTERN,
ALERT_REASON,
ALERT_RULE_PARAMETERS,
Expand Down Expand Up @@ -319,6 +320,7 @@ export function registerTransactionDurationRuleType({
[PROCESSOR_EVENT]: ProcessorEvent.transaction,
[ALERT_EVALUATION_VALUE]: transactionDuration,
[ALERT_EVALUATION_THRESHOLD]: thresholdMicroseconds,
[ALERT_GROUPING]: groupingObject,
[ALERT_REASON]: reason,
[ALERT_INDEX_PATTERN]: index,
...sourceFields,
Expand Down Expand Up @@ -369,7 +371,8 @@ export function registerTransactionDurationRuleType({
alertHits?.[ALERT_EVALUATION_VALUE]
).formatted;
const groupByActionVariables = getGroupByActionVariables(groupByFields);
const groupingObject = unflattenObject(groupByFields);
const groupingObjectFromRecoveredAlert =
alertHits?.[ALERT_GROUPING] ?? unflattenObject(groupByFields);

const recoveredContext = {
alertDetailsUrl,
Expand All @@ -383,7 +386,7 @@ export function registerTransactionDurationRuleType({
threshold: ruleParams.threshold,
triggerValue: transactionDurationFormatted,
viewInAppUrl,
grouping: groupingObject,
grouping: groupingObjectFromRecoveredAlert,
...groupByActionVariables,
};

Expand Down
42 changes: 34 additions & 8 deletions ...ity/test/api_integration_deployment_agnostic/apis/apm/alerts/transaction_duration.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,10 +204,23 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon
});

it('indexes alert document with all group-by fields', async () => {
expect(alerts[0]).property('service.name', 'opbeans-java');
expect(alerts[0]).property('service.environment', 'production');
expect(alerts[0]).property('transaction.type', 'request');
expect(alerts[0]).property('transaction.name', 'tx-java');
const alert = alerts[0];
expect({
'service.name': alert['service.name'],
'service.environment': alert['service.environment'],
'transaction.type': alert['transaction.type'],
'transaction.name': alert['transaction.name'],
'kibana.alert.grouping': alert['kibana.alert.grouping'],
}).to.eql({
'service.name': 'opbeans-java',
'service.environment': 'production',
'transaction.type': 'request',
'transaction.name': 'tx-java',
'kibana.alert.grouping': {
service: { name: 'opbeans-java', environment: 'production' },
transaction: { type: 'request', name: 'tx-java' },
},
});
});

it('shows the correct alert count for each service on service inventory', async () => {
Expand Down Expand Up @@ -304,10 +317,23 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon
});

it('indexes alert document with all group-by fields', async () => {
expect(alerts[0]).property('service.name', 'opbeans-node');
expect(alerts[0]).property('service.environment', 'production');
expect(alerts[0]).property('transaction.type', 'request');
expect(alerts[0]).property('transaction.name', 'tx-node');
const alert = alerts[0];
expect({
'service.name': alert['service.name'],
'service.environment': alert['service.environment'],
'transaction.type': alert['transaction.type'],
'transaction.name': alert['transaction.name'],
'kibana.alert.grouping': alert['kibana.alert.grouping'],
}).to.eql({
'service.name': 'opbeans-node',
'service.environment': 'production',
'transaction.type': 'request',
'transaction.name': 'tx-node',
'kibana.alert.grouping': {
service: { name: 'opbeans-node', environment: 'production' },
transaction: { type: 'request', name: 'tx-node' },
},
});
});

it('shows alert count=1 for opbeans-node on service inventory', async () => {
Expand Down
Loading