diff --git a/beater/beater.go b/beater/beater.go index 0ace5cbc907..89486168f00 100644 --- a/beater/beater.go +++ b/beater/beater.go @@ -35,6 +35,7 @@ import ( "github.com/elastic/apm-server/ingest/pipeline" logs "github.com/elastic/apm-server/log" "github.com/elastic/apm-server/publish" + "github.com/elastic/apm-server/sampling" ) var ( @@ -172,6 +173,14 @@ func (bt *beater) Run(b *beat.Beat) error { } defer publisher.Stop() + reporter := publisher.Send + if !bt.config.Sampling.KeepUnsampled { + // The server has been configured to discard unsampled + // transactions. Make sure this is done just before calling + // the publisher to avoid affecting aggregations. + reporter = sampling.NewDiscardUnsampledReporter(reporter) + } + stopped := make(chan struct{}) defer close(stopped) ctx, cancelContext := context.WithCancel(context.Background()) @@ -196,7 +205,7 @@ func (bt *beater) Run(b *beat.Beat) error { Config: bt.config, Logger: bt.logger, Tracer: tracer, - Reporter: publisher.Send, + Reporter: reporter, }) } diff --git a/beater/config/config.go b/beater/config/config.go index 48a2ebeb8cd..1f9c30caa5b 100644 --- a/beater/config/config.go +++ b/beater/config/config.go @@ -89,6 +89,7 @@ type Config struct { APIKeyConfig *APIKeyConfig `config:"api_key"` JaegerConfig JaegerConfig `config:"jaeger"` Aggregation AggregationConfig `config:"aggregation"` + Sampling SamplingConfig `config:"sampling"` Pipeline string } @@ -141,6 +142,18 @@ func NewConfig(version string, ucfg *common.Config, outputESCfg *common.Config) return nil, err } + if !c.Sampling.KeepUnsampled && !c.Aggregation.Enabled { + // Unsampled transactions should only be dropped + // when transaction aggregation is enabled in the + // server. This means the aggregations performed + // by the APM UI will not have access to a complete + // representation of the latency distribution. + logger.Warn("" + + "apm-server.sampling.keep_unsampled and " + + "apm-server.aggregation.enabled are both false, " + + "which will lead to incorrect metrics being reported in the APM UI", + ) + } return c, nil } @@ -174,5 +187,6 @@ func DefaultConfig(beatVersion string) *Config { APIKeyConfig: defaultAPIKeyConfig(), JaegerConfig: defaultJaeger(), Aggregation: defaultAggregationConfig(), + Sampling: defaultSamplingConfig(), } } diff --git a/beater/config/config_test.go b/beater/config/config_test.go index 9418c31f52a..a695521195b 100644 --- a/beater/config/config_test.go +++ b/beater/config/config_test.go @@ -210,6 +210,9 @@ func Test_UnpackConfig(t *testing.T) { MaxTransactionGroups: 123, HDRHistogramSignificantFigures: 1, }, + Sampling: SamplingConfig{ + KeepUnsampled: true, + }, }, }, "merge config with default": { @@ -239,9 +242,10 @@ func Test_UnpackConfig(t *testing.T) { }, }, }, - "jaeger.grpc.enabled": true, - "api_key.enabled": true, - "aggregation.enabled": true, + "jaeger.grpc.enabled": true, + "api_key.enabled": true, + "aggregation.enabled": true, + "sampling.keep_unsampled": false, }, outCfg: &Config{ Host: "localhost:3000", @@ -316,6 +320,9 @@ func Test_UnpackConfig(t *testing.T) { MaxTransactionGroups: 1000, HDRHistogramSignificantFigures: 2, }, + Sampling: SamplingConfig{ + KeepUnsampled: false, + }, }, }, "kibana trailing slash": { diff --git a/beater/config/sampling.go b/beater/config/sampling.go new file mode 100644 index 00000000000..c415705f0a3 --- /dev/null +++ b/beater/config/sampling.go @@ -0,0 +1,33 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package config + +// SamplingConfig holds configuration related to sampling. +type SamplingConfig struct { + // KeepUnsampled controls whether unsampled + // transactions should be recorded. + KeepUnsampled bool `config:"keep_unsampled"` +} + +func defaultSamplingConfig() SamplingConfig { + return SamplingConfig{ + // In a future major release we will set this to + // false, and then later remove the option. + KeepUnsampled: true, + } +} diff --git a/beater/test_approved_es_documents/TestDropNonSampledTransactions.approved.json b/beater/test_approved_es_documents/TestDropNonSampledTransactions.approved.json new file mode 100644 index 00000000000..e221283f97a --- /dev/null +++ b/beater/test_approved_es_documents/TestDropNonSampledTransactions.approved.json @@ -0,0 +1,1003 @@ +{ + "events": [ + { + "@metadata": { + "beat": "apm-test", + "pipeline": "apm", + "type": "_doc", + "version": "8.0.0" + }, + "@timestamp": "2017-05-30T18:53:27.154Z", + "agent": { + "name": "js-base", + "version": "1.3" + }, + "client": { + "ip": "8.8.8.8" + }, + "container": { + "id": "container-id" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "architecture": "x64", + "ip": "127.0.0.1", + "os": { + "platform": "darwin" + } + }, + "http": { + "request": { + "body": { + "original": { + "additional": { + "bar": 123, + "req": "additional information" + }, + "str": "hello world" + } + }, + "cookies": { + "c1": "v1", + "c2": "v2" + }, + "env": { + "GATEWAY_INTERFACE": "CGI/1.1", + "SERVER_SOFTWARE": "nginx" + }, + "headers": { + "Array": [ + "foo", + "bar", + "baz" + ], + "Content-Type": [ + "text/html" + ], + "Cookie": [ + "c1=v1,c2=v2" + ], + "Some-Other-Header": [ + "foo" + ], + "User-Agent": [ + "Mozilla Chrome Edge" + ] + }, + "method": "post", + "socket": { + "encrypted": true, + "remote_address": "8.8.8.8" + } + }, + "response": { + "finished": true, + "headers": { + "Content-Type": [ + "application/json" + ] + }, + "headers_sent": true, + "status_code": 200 + }, + "version": "1.1" + }, + "kubernetes": { + "namespace": "namespace1", + "pod": { + "name": "pod-name", + "uid": "pod-uid" + } + }, + "labels": { + "bool_error": false, + "number_code": 2, + "organization_uuid": "9f0e9d64-c185-4d21-a6f4-4673ed561ec8" + }, + "observer": { + "ephemeral_id": "00000000-0000-0000-0000-000000000000", + "hostname": "", + "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", + "type": "test-apm-server", + "version": "8.0.0", + "version_major": 8 + }, + "process": { + "args": [ + "node", + "server.js" + ], + "pid": 1234, + "ppid": 6789, + "title": "node" + }, + "processor": { + "event": "transaction", + "name": "transaction" + }, + "service": { + "environment": "staging", + "framework": { + "name": "Express", + "version": "1.2.3" + }, + "language": { + "name": "ecmascript", + "version": "8" + }, + "name": "serviceabc", + "node": { + "name": "special-name" + }, + "runtime": { + "name": "javascript", + "version": "8.0.0" + }, + "version": "5.1.3" + }, + "source": { + "ip": "8.8.8.8" + }, + "timestamp": { + "us": 1496170407154000 + }, + "trace": { + "id": "945254c567a5417eaaaaaaaaaaaaaaaa" + }, + "transaction": { + "custom": { + "(": "not a valid regex and that is fine", + "and_objects": { + "foo": [ + "bar", + "baz" + ] + }, + "my_key": 1, + "some_other_value": "foo bar" + }, + "duration": { + "us": 32592 + }, + "id": "945254c567a5417e", + "marks": { + "another_mark": { + "some_float": 10, + "some_long": 10 + }, + "navigationTiming": { + "appBeforeBootstrap": 608.9300000000001, + "navigationStart": -21 + } + }, + "name": "GET /api/types", + "page": { + "referer": "http://localhost:8000/test/e2e/", + "url": "http://localhost:8000/test/e2e/general-usecase/" + }, + "result": "success", + "sampled": true, + "span_count": { + "dropped": 2, + "started": 4 + }, + "type": "request" + }, + "url": { + "domain": "www.example.com", + "fragment": "#hash", + "full": "https://www.example.com/p/a/t/h?query=string#hash", + "original": "/p/a/t/h?query=string#hash", + "path": "/p/a/t/h", + "port": 8080, + "query": "?query=string", + "scheme": "https" + }, + "user": { + "email": "foo@example.com", + "id": "99" + }, + "user_agent": { + "original": "Mozilla Chrome Edge" + } + }, + { + "@metadata": { + "beat": "apm-test", + "pipeline": "apm", + "type": "_doc", + "version": "8.0.0" + }, + "@timestamp": "2017-05-30T18:53:27.154Z", + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "container": { + "id": "container-id" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "architecture": "x64", + "ip": "127.0.0.1", + "os": { + "platform": "darwin" + } + }, + "kubernetes": { + "namespace": "namespace1", + "pod": { + "name": "pod-name", + "uid": "pod-uid" + } + }, + "labels": { + "span_tag": "something" + }, + "observer": { + "ephemeral_id": "00000000-0000-0000-0000-000000000000", + "hostname": "", + "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", + "type": "test-apm-server", + "version": "8.0.0", + "version_major": 8 + }, + "parent": { + "id": "945254c567a5417e" + }, + "process": { + "args": [ + "node", + "server.js" + ], + "pid": 1234, + "ppid": 6789, + "title": "node" + }, + "processor": { + "event": "span", + "name": "transaction" + }, + "service": { + "environment": "staging", + "framework": { + "name": "Express", + "version": "1.2.3" + }, + "language": { + "name": "ecmascript", + "version": "8" + }, + "name": "1234_service-12a3", + "node": { + "name": "container-id" + }, + "runtime": { + "name": "node", + "version": "8.0.0" + }, + "version": "5.1.3" + }, + "span": { + "action": "query", + "db": { + "instance": "customers", + "statement": "SELECT * FROM product_types WHERE user_id=?", + "type": "sql", + "user": { + "name": "readonly_user" + } + }, + "duration": { + "us": 3781 + }, + "http": { + "method": "get", + "response": { + "status_code": 200 + }, + "url": { + "original": "http://localhost:8000" + } + }, + "id": "0aaaaaaaaaaaaaaa", + "name": "SELECT FROM product_types", + "stacktrace": [ + { + "abs_path": "net.js", + "context": { + "post": [ + " ins.currentTransaction = prev", + " return result", + "}" + ], + "pre": [ + " var trans = this.currentTransaction", + "" + ] + }, + "exclude_from_grouping": false, + "filename": "net.js", + "function": "onread", + "library_frame": true, + "line": { + "column": 4, + "context": "line3", + "number": 547 + }, + "module": "some module", + "vars": { + "key": "value" + } + }, + { + "exclude_from_grouping": false, + "filename": "my2file.js", + "line": { + "number": 10 + } + } + ], + "start": { + "us": 2830 + }, + "subtype": "postgresql", + "sync": false, + "type": "db" + }, + "timestamp": { + "us": 1496170407154000 + }, + "trace": { + "id": "945254c567a5417eaaaaaaaaaaaaaaaa" + }, + "transaction": { + "id": "945254c567a5417e" + }, + "user": { + "email": "foo@bar.com", + "id": "123user", + "name": "foo" + } + }, + { + "@metadata": { + "beat": "apm-test", + "pipeline": "apm", + "type": "_doc", + "version": "8.0.0" + }, + "@timestamp": "2017-05-30T18:53:27.154Z", + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "container": { + "id": "container-id" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "architecture": "x64", + "ip": "127.0.0.1", + "os": { + "platform": "darwin" + } + }, + "kubernetes": { + "namespace": "namespace1", + "pod": { + "name": "pod-name", + "uid": "pod-uid" + } + }, + "observer": { + "ephemeral_id": "00000000-0000-0000-0000-000000000000", + "hostname": "", + "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", + "type": "test-apm-server", + "version": "8.0.0", + "version_major": 8 + }, + "parent": { + "id": "945254c567a5417e" + }, + "process": { + "args": [ + "node", + "server.js" + ], + "pid": 1234, + "ppid": 6789, + "title": "node" + }, + "processor": { + "event": "span", + "name": "transaction" + }, + "service": { + "environment": "staging", + "framework": { + "name": "Express", + "version": "1.2.3" + }, + "language": { + "name": "ecmascript", + "version": "8" + }, + "name": "1234_service-12a3", + "node": { + "name": "container-id" + }, + "runtime": { + "name": "node", + "version": "8.0.0" + }, + "version": "5.1.3" + }, + "span": { + "duration": { + "us": 32592 + }, + "id": "1aaaaaaaaaaaaaaa", + "name": "GET /api/types", + "start": { + "us": 0 + }, + "subtype": "external", + "type": "request" + }, + "timestamp": { + "us": 1496170407154000 + }, + "trace": { + "id": "945254c567a5417eaaaaaaaaaaaaaaaa" + }, + "transaction": { + "id": "945254c567a5417e" + }, + "user": { + "email": "foo@bar.com", + "id": "123user", + "name": "foo" + } + }, + { + "@metadata": { + "beat": "apm-test", + "pipeline": "apm", + "type": "_doc", + "version": "8.0.0" + }, + "@timestamp": "2017-05-30T18:53:27.154Z", + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "container": { + "id": "container-id" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "architecture": "x64", + "ip": "127.0.0.1", + "os": { + "platform": "darwin" + } + }, + "kubernetes": { + "namespace": "namespace1", + "pod": { + "name": "pod-name", + "uid": "pod-uid" + } + }, + "observer": { + "ephemeral_id": "00000000-0000-0000-0000-000000000000", + "hostname": "", + "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", + "type": "test-apm-server", + "version": "8.0.0", + "version_major": 8 + }, + "parent": { + "id": "945254c567a5417e" + }, + "process": { + "args": [ + "node", + "server.js" + ], + "pid": 1234, + "ppid": 6789, + "title": "node" + }, + "processor": { + "event": "span", + "name": "transaction" + }, + "service": { + "environment": "staging", + "framework": { + "name": "Express", + "version": "1.2.3" + }, + "language": { + "name": "ecmascript", + "version": "8" + }, + "name": "1234_service-12a3", + "node": { + "name": "container-id" + }, + "runtime": { + "name": "node", + "version": "8.0.0" + }, + "version": "5.1.3" + }, + "span": { + "action": "post", + "duration": { + "us": 3564 + }, + "id": "2aaaaaaaaaaaaaaa", + "name": "GET /api/types", + "start": { + "us": 1845 + }, + "subtype": "http", + "type": "request" + }, + "timestamp": { + "us": 1496170407154000 + }, + "trace": { + "id": "945254c567a5417eaaaaaaaaaaaaaaaa" + }, + "transaction": { + "id": "945254c567a5417e" + }, + "user": { + "email": "foo@bar.com", + "id": "123user", + "name": "foo" + } + }, + { + "@metadata": { + "beat": "apm-test", + "pipeline": "apm", + "type": "_doc", + "version": "8.0.0" + }, + "@timestamp": "2017-05-30T18:53:27.154Z", + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "child": { + "id": [ + "4aaaaaaaaaaaaaaa" + ] + }, + "container": { + "id": "container-id" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "architecture": "x64", + "ip": "127.0.0.1", + "os": { + "platform": "darwin" + } + }, + "kubernetes": { + "namespace": "namespace1", + "pod": { + "name": "pod-name", + "uid": "pod-uid" + } + }, + "observer": { + "ephemeral_id": "00000000-0000-0000-0000-000000000000", + "hostname": "", + "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", + "type": "test-apm-server", + "version": "8.0.0", + "version_major": 8 + }, + "parent": { + "id": "945254c567a5417e" + }, + "process": { + "args": [ + "node", + "server.js" + ], + "pid": 1234, + "ppid": 6789, + "title": "node" + }, + "processor": { + "event": "span", + "name": "transaction" + }, + "service": { + "environment": "staging", + "framework": { + "name": "Express", + "version": "1.2.3" + }, + "language": { + "name": "ecmascript", + "version": "8" + }, + "name": "1234_service-12a3", + "node": { + "name": "container-id" + }, + "runtime": { + "name": "node", + "version": "8.0.0" + }, + "version": "5.1.3" + }, + "span": { + "duration": { + "us": 13980 + }, + "id": "3aaaaaaaaaaaaaaa", + "name": "GET /api/types", + "start": { + "us": 0 + }, + "type": "request" + }, + "timestamp": { + "us": 1496170407154000 + }, + "trace": { + "id": "945254c567a5417eaaaaaaaaaaaaaaaa" + }, + "transaction": { + "id": "945254c567a5417e" + }, + "user": { + "email": "foo@bar.com", + "id": "123user", + "name": "foo" + } + }, + { + "@metadata": { + "beat": "apm-test", + "pipeline": "apm", + "type": "_doc", + "version": "8.0.0" + }, + "@timestamp": "2017-05-30T18:53:42.281Z", + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "container": { + "id": "container-id" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "architecture": "x64", + "ip": "127.0.0.1", + "os": { + "platform": "darwin" + } + }, + "kubernetes": { + "namespace": "namespace1", + "pod": { + "name": "pod-name", + "uid": "pod-uid" + } + }, + "observer": { + "ephemeral_id": "00000000-0000-0000-0000-000000000000", + "hostname": "", + "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", + "type": "test-apm-server", + "version": "8.0.0", + "version_major": 8 + }, + "process": { + "args": [ + "node", + "server.js" + ], + "pid": 1234, + "ppid": 6789, + "title": "node" + }, + "processor": { + "event": "transaction", + "name": "transaction" + }, + "service": { + "environment": "staging", + "framework": { + "name": "Express", + "version": "1.2.3" + }, + "language": { + "name": "ecmascript", + "version": "8" + }, + "name": "1234_service-12a3", + "node": { + "name": "container-id" + }, + "runtime": { + "name": "node", + "version": "8.0.0" + }, + "version": "5.1.3" + }, + "timestamp": { + "us": 1496170422281000 + }, + "trace": { + "id": "85925e55b43f4340aaaaaaaaaaaaaaaa" + }, + "transaction": { + "duration": { + "us": 13980 + }, + "id": "85925e55b43f4340", + "name": "GET /api/types", + "result": "failure", + "sampled": true, + "span_count": { + "started": 0 + }, + "type": "request" + }, + "user": { + "email": "foo@bar.com", + "id": "123user", + "name": "foo" + } + }, + { + "@metadata": { + "beat": "apm-test", + "pipeline": "apm", + "type": "_doc", + "version": "8.0.0" + }, + "@timestamp": "2017-05-30T18:53:42.281Z", + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "container": { + "id": "container-id" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "architecture": "x64", + "ip": "127.0.0.1", + "os": { + "platform": "darwin" + } + }, + "kubernetes": { + "namespace": "namespace1", + "pod": { + "name": "pod-name", + "uid": "pod-uid" + } + }, + "observer": { + "ephemeral_id": "00000000-0000-0000-0000-000000000000", + "hostname": "", + "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", + "type": "test-apm-server", + "version": "8.0.0", + "version_major": 8 + }, + "process": { + "args": [ + "node", + "server.js" + ], + "pid": 1234, + "ppid": 6789, + "title": "node" + }, + "processor": { + "event": "transaction", + "name": "transaction" + }, + "service": { + "environment": "staging", + "framework": { + "name": "Express", + "version": "1.2.3" + }, + "language": { + "name": "ecmascript", + "version": "8" + }, + "name": "1234_service-12a3", + "node": { + "name": "container-id" + }, + "runtime": { + "name": "node", + "version": "8.0.0" + }, + "version": "5.1.3" + }, + "timestamp": { + "us": 1496170422281999 + }, + "trace": { + "id": "85925e55b43f4342aaaaaaaaaaaaaaaa" + }, + "transaction": { + "duration": { + "us": 13980 + }, + "id": "85925e55b43f4342", + "name": "GET /api/types", + "result": "200", + "sampled": true, + "span_count": { + "dropped": 258, + "started": 1 + }, + "type": "request" + }, + "user": { + "email": "foo@bar.com", + "id": "123user", + "name": "foo" + } + }, + { + "@metadata": { + "beat": "apm-test", + "pipeline": "apm", + "type": "_doc", + "version": "8.0.0" + }, + "@timestamp": "2017-05-30T18:53:42.281Z", + "agent": { + "name": "js-base", + "version": "1.3" + }, + "container": { + "id": "container-id" + }, + "destination": { + "address": "0:0::0:1", + "ip": "0:0::0:1", + "port": 5432 + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "architecture": "x64", + "ip": "127.0.0.1", + "os": { + "platform": "darwin" + } + }, + "kubernetes": { + "namespace": "namespace1", + "pod": { + "name": "pod-name", + "uid": "pod-uid" + } + }, + "observer": { + "ephemeral_id": "00000000-0000-0000-0000-000000000000", + "hostname": "", + "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", + "type": "test-apm-server", + "version": "8.0.0", + "version_major": 8 + }, + "parent": { + "id": "85925e55b43f4342" + }, + "process": { + "args": [ + "node", + "server.js" + ], + "pid": 1234, + "ppid": 6789, + "title": "node" + }, + "processor": { + "event": "span", + "name": "transaction" + }, + "service": { + "environment": "staging", + "framework": { + "name": "Express", + "version": "1.2.3" + }, + "language": { + "name": "ecmascript", + "version": "8" + }, + "name": "serviceabc", + "node": { + "name": "container-id" + }, + "runtime": { + "name": "javascript", + "version": "8.0.0" + }, + "version": "5.1.3" + }, + "span": { + "action": "query.custom", + "db": { + "instance": "customers", + "statement": "SELECT * FROM product_types WHERE user_id=?", + "type": "sql", + "user": { + "name": "readonly_user" + } + }, + "destination": { + "service": { + "name": "postgresql", + "resource": "postgresql", + "type": "db" + } + }, + "duration": { + "us": 3781 + }, + "id": "15aaaaaaaaaaaaaa", + "name": "SELECT FROM product_types", + "start": { + "us": 2830 + }, + "subtype": "postgresql", + "type": "db.postgresql.query" + }, + "timestamp": { + "us": 1496170422281000 + }, + "trace": { + "id": "85925e55b43f4342aaaaaaaaaaaaaaaa" + }, + "transaction": { + "id": "85925e55b43f4342" + }, + "user": { + "email": "foo@bar.com", + "id": "123user", + "name": "foo" + } + } + ] +} diff --git a/changelogs/head.asciidoc b/changelogs/head.asciidoc index 691e45575c8..60b24d7f8aa 100644 --- a/changelogs/head.asciidoc +++ b/changelogs/head.asciidoc @@ -19,3 +19,4 @@ https://github.com/elastic/apm-server/compare/7.7\...master[View commits] * RUM V3 endpoint {pull}3328[3328], {pull}3414[3414], {pull}3512[3512], {pull}3526[3526], {pull}3616[3616], {pull}3648[3648], {pull}3659[3659] * We now publish libbeat metrics through the /debug/vars endpoint {pull}3550[3550] * Introduce transaction.duration.histogram aggregation {pull}3651[3651] +* Introduce config to drop unsampled transactions {pull}3702[3702] diff --git a/publish/pub.go b/publish/pub.go index 521586f6948..f2afb75b30c 100644 --- a/publish/pub.go +++ b/publish/pub.go @@ -132,6 +132,10 @@ func (p *Publisher) Stop() { // an error is returned. // Calling send after Stop will return an error. func (p *Publisher) Send(ctx context.Context, req PendingReq) error { + if len(req.Transformables) == 0 { + return nil + } + p.m.RLock() defer p.m.RUnlock() if p.stopped { diff --git a/sampling/sampling.go b/sampling/sampling.go new file mode 100644 index 00000000000..2504b750974 --- /dev/null +++ b/sampling/sampling.go @@ -0,0 +1,59 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package sampling + +import ( + "context" + + "github.com/elastic/apm-server/model" + "github.com/elastic/apm-server/publish" + "github.com/elastic/beats/v7/libbeat/monitoring" +) + +var ( + monitoringRegistry = monitoring.Default.NewRegistry("apm-server.sampling") + transactionsDroppedCounter = monitoring.NewInt(monitoringRegistry, "transactions_dropped") +) + +// NewDiscardUnsampledReporter returns a publish.Reporter which discards +// unsampled transactions before deferring to reporter. +// +// The returned publish.Reporter does not guarantee order preservation of +// reported events. +func NewDiscardUnsampledReporter(reporter publish.Reporter) publish.Reporter { + return func(ctx context.Context, req publish.PendingReq) error { + var dropped int64 + events := req.Transformables + for i := 0; i < len(events); { + tx, ok := events[i].(*model.Transaction) + if !ok || tx.Sampled == nil || *tx.Sampled { + i++ + continue + } + n := len(req.Transformables) + events[i], events[n-1] = events[n-1], events[i] + events = events[:n-1] + dropped++ + } + if dropped > 0 { + transactionsDroppedCounter.Add(dropped) + } + req.Transformables = events + return reporter(ctx, req) + } +} diff --git a/sampling/sampling_test.go b/sampling/sampling_test.go new file mode 100644 index 00000000000..16452b3296a --- /dev/null +++ b/sampling/sampling_test.go @@ -0,0 +1,72 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package sampling_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/apm-server/model" + "github.com/elastic/apm-server/publish" + "github.com/elastic/apm-server/sampling" + "github.com/elastic/apm-server/transform" + "github.com/elastic/beats/v7/libbeat/monitoring" +) + +func TestNewDiscardUnsampledReporter(t *testing.T) { + var reported []transform.Transformable + reporter := sampling.NewDiscardUnsampledReporter( + func(ctx context.Context, req publish.PendingReq) error { + reported = req.Transformables + return nil + }, + ) + + t1 := &model.Transaction{} + t2 := &model.Transaction{Sampled: newBool(false)} + t3 := &model.Transaction{Sampled: newBool(true)} + span := &model.Span{} + + reporter(context.Background(), publish.PendingReq{ + Transformables: []transform.Transformable{t1, t2, t3, span}, + }) + + // Note that t3 gets sent to the back of the slice; + // this reporter is not order-preserving. + require.Len(t, reported, 3) + assert.Equal(t, t1, reported[0]) + assert.Equal(t, span, reported[1]) + assert.Equal(t, t3, reported[2]) + + expectedMonitoring := monitoring.MakeFlatSnapshot() + expectedMonitoring.Ints["transactions_dropped"] = 1 + + snapshot := monitoring.CollectFlatSnapshot( + monitoring.GetRegistry("apm-server.sampling"), + monitoring.Full, + false, // expvar + ) + assert.Equal(t, expectedMonitoring, snapshot) +} + +func newBool(v bool) *bool { + return &v +} diff --git a/tests/system/config/apm-server.yml.j2 b/tests/system/config/apm-server.yml.j2 index da96438157f..848cb6739d5 100644 --- a/tests/system/config/apm-server.yml.j2 +++ b/tests/system/config/apm-server.yml.j2 @@ -144,6 +144,9 @@ apm-server: aggregation.interval: {{ aggregation_interval }} {% endif %} + sampling: + keep_unsampled: {{ sampling_keep_unsampled }} + {% if mode %} mode: {{ mode }} {% endif %} diff --git a/tests/system/drop_unsampled_transactions.approved.json b/tests/system/drop_unsampled_transactions.approved.json new file mode 100644 index 00000000000..0ce7272ca58 --- /dev/null +++ b/tests/system/drop_unsampled_transactions.approved.json @@ -0,0 +1,405 @@ +[ + { + "@timestamp": "2017-05-30T18:53:42.281Z", + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "container": { + "id": "container-id" + }, + "ecs": { + "version": "1.5.0" + }, + "event": { + "ingested": "2020-04-29T02:24:19.182874Z" + }, + "host": { + "architecture": "x64", + "ip": "127.0.0.1", + "os": { + "platform": "darwin" + } + }, + "kubernetes": { + "namespace": "namespace1", + "pod": { + "name": "pod-name", + "uid": "pod-uid" + } + }, + "observer": { + "ephemeral_id": "ebd89158-9f85-469e-9bc0-9e19b174a1ba", + "hostname": "goat", + "id": "01019c57-ff25-4ddf-b9e9-2f421577a9a2", + "type": "apm-server", + "version": "8.0.0", + "version_major": 8 + }, + "process": { + "args": [ + "node", + "server.js" + ], + "pid": 1234, + "ppid": 6789, + "title": "node" + }, + "processor": { + "event": "transaction", + "name": "transaction" + }, + "service": { + "environment": "staging", + "framework": { + "name": "Express", + "version": "1.2.3" + }, + "language": { + "name": "ecmascript", + "version": "8" + }, + "name": "1234_service-12a3", + "node": { + "name": "container-id" + }, + "runtime": { + "name": "node", + "version": "8.0.0" + }, + "version": "5.1.3" + }, + "timestamp": { + "us": 1496170422281000 + }, + "trace": { + "id": "85925e55b43f4340aaaaaaaaaaaaaaaa" + }, + "transaction": { + "duration": { + "us": 13980 + }, + "id": "85925e55b43f4340", + "name": "GET /api/types", + "result": "failure", + "sampled": true, + "span_count": { + "started": 0 + }, + "type": "request" + }, + "user": { + "email": "foo@bar.com", + "id": "123user", + "name": "foo" + } + }, + { + "@timestamp": "2017-05-30T18:53:42.281Z", + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "container": { + "id": "container-id" + }, + "ecs": { + "version": "1.5.0" + }, + "event": { + "ingested": "2020-04-29T02:24:19.184164Z" + }, + "host": { + "architecture": "x64", + "ip": "127.0.0.1", + "os": { + "platform": "darwin" + } + }, + "kubernetes": { + "namespace": "namespace1", + "pod": { + "name": "pod-name", + "uid": "pod-uid" + } + }, + "observer": { + "ephemeral_id": "ebd89158-9f85-469e-9bc0-9e19b174a1ba", + "hostname": "goat", + "id": "01019c57-ff25-4ddf-b9e9-2f421577a9a2", + "type": "apm-server", + "version": "8.0.0", + "version_major": 8 + }, + "process": { + "args": [ + "node", + "server.js" + ], + "pid": 1234, + "ppid": 6789, + "title": "node" + }, + "processor": { + "event": "transaction", + "name": "transaction" + }, + "service": { + "environment": "staging", + "framework": { + "name": "Express", + "version": "1.2.3" + }, + "language": { + "name": "ecmascript", + "version": "8" + }, + "name": "1234_service-12a3", + "node": { + "name": "container-id" + }, + "runtime": { + "name": "node", + "version": "8.0.0" + }, + "version": "5.1.3" + }, + "timestamp": { + "us": 1496170422281999 + }, + "trace": { + "id": "85925e55b43f4342aaaaaaaaaaaaaaaa" + }, + "transaction": { + "duration": { + "us": 13980 + }, + "id": "85925e55b43f4342", + "name": "GET /api/types", + "result": "200", + "sampled": true, + "span_count": { + "dropped": 258, + "started": 1 + }, + "type": "request" + }, + "user": { + "email": "foo@bar.com", + "id": "123user", + "name": "foo" + } + }, + { + "@timestamp": "2017-05-30T18:53:27.154Z", + "agent": { + "name": "js-base", + "version": "1.3" + }, + "client": { + "geo": { + "continent_name": "North America", + "country_iso_code": "US", + "location": { + "lat": 37.751, + "lon": -97.822 + } + }, + "ip": "8.8.8.8" + }, + "container": { + "id": "container-id" + }, + "ecs": { + "version": "1.5.0" + }, + "event": { + "ingested": "2020-04-29T02:24:19.099344Z" + }, + "host": { + "architecture": "x64", + "ip": "127.0.0.1", + "os": { + "platform": "darwin" + } + }, + "http": { + "request": { + "body": { + "original": { + "additional": { + "bar": 123, + "req": "additional information" + }, + "str": "hello world" + } + }, + "cookies": { + "c1": "v1", + "c2": "v2" + }, + "env": { + "GATEWAY_INTERFACE": "CGI/1.1", + "SERVER_SOFTWARE": "nginx" + }, + "headers": { + "Array": [ + "foo", + "bar", + "baz" + ], + "Content-Type": [ + "text/html" + ], + "Cookie": [ + "c1=v1,c2=v2" + ], + "Some-Other-Header": [ + "foo" + ], + "User-Agent": [ + "Mozilla Chrome Edge" + ] + }, + "method": "post", + "socket": { + "encrypted": true, + "remote_address": "8.8.8.8" + } + }, + "response": { + "finished": true, + "headers": { + "Content-Type": [ + "application/json" + ] + }, + "headers_sent": true, + "status_code": 200 + }, + "version": "1.1" + }, + "kubernetes": { + "namespace": "namespace1", + "pod": { + "name": "pod-name", + "uid": "pod-uid" + } + }, + "labels": { + "bool_error": false, + "number_code": 2, + "organization_uuid": "9f0e9d64-c185-4d21-a6f4-4673ed561ec8" + }, + "observer": { + "ephemeral_id": "ebd89158-9f85-469e-9bc0-9e19b174a1ba", + "hostname": "goat", + "id": "01019c57-ff25-4ddf-b9e9-2f421577a9a2", + "type": "apm-server", + "version": "8.0.0", + "version_major": 8 + }, + "process": { + "args": [ + "node", + "server.js" + ], + "pid": 1234, + "ppid": 6789, + "title": "node" + }, + "processor": { + "event": "transaction", + "name": "transaction" + }, + "service": { + "environment": "staging", + "framework": { + "name": "Express", + "version": "1.2.3" + }, + "language": { + "name": "ecmascript", + "version": "8" + }, + "name": "serviceabc", + "node": { + "name": "special-name" + }, + "runtime": { + "name": "javascript", + "version": "8.0.0" + }, + "version": "5.1.3" + }, + "source": { + "ip": "8.8.8.8" + }, + "timestamp": { + "us": 1496170407154000 + }, + "trace": { + "id": "945254c567a5417eaaaaaaaaaaaaaaaa" + }, + "transaction": { + "custom": { + "(": "not a valid regex and that is fine", + "and_objects": { + "foo": [ + "bar", + "baz" + ] + }, + "my_key": 1, + "some_other_value": "foo bar" + }, + "duration": { + "us": 32592 + }, + "id": "945254c567a5417e", + "marks": { + "another_mark": { + "some_float": 10, + "some_long": 10 + }, + "navigationTiming": { + "appBeforeBootstrap": 608.9300000000001, + "navigationStart": -21 + } + }, + "name": "GET /api/types", + "page": { + "referer": "http://localhost:8000/test/e2e/", + "url": "http://localhost:8000/test/e2e/general-usecase/" + }, + "result": "success", + "sampled": true, + "span_count": { + "dropped": 2, + "started": 4 + }, + "type": "request" + }, + "url": { + "domain": "www.example.com", + "fragment": "#hash", + "full": "https://www.example.com/p/a/t/h?query=string#hash", + "original": "/p/a/t/h?query=string#hash", + "path": "/p/a/t/h", + "port": 8080, + "query": "?query=string", + "scheme": "https" + }, + "user": { + "email": "foo@example.com", + "id": "99" + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Other", + "original": "Mozilla Chrome Edge" + } + } +] \ No newline at end of file diff --git a/tests/system/keep_unsampled_transactions.approved.json b/tests/system/keep_unsampled_transactions.approved.json new file mode 100644 index 00000000000..3997e639dc4 --- /dev/null +++ b/tests/system/keep_unsampled_transactions.approved.json @@ -0,0 +1,500 @@ +[ + { + "@timestamp": "2017-05-30T18:53:42.281Z", + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "container": { + "id": "container-id" + }, + "ecs": { + "version": "1.5.0" + }, + "event": { + "ingested": "2020-04-29T02:16:27.090771Z" + }, + "host": { + "architecture": "x64", + "ip": "127.0.0.1", + "os": { + "platform": "darwin" + } + }, + "kubernetes": { + "namespace": "namespace1", + "pod": { + "name": "pod-name", + "uid": "pod-uid" + } + }, + "observer": { + "ephemeral_id": "3f31e73c-2fdf-4f39-8c81-57b25d00fc16", + "hostname": "goat", + "id": "0f00bacd-a23e-41c5-a2fb-4d90b2776018", + "type": "apm-server", + "version": "8.0.0", + "version_major": 8 + }, + "process": { + "args": [ + "node", + "server.js" + ], + "pid": 1234, + "ppid": 6789, + "title": "node" + }, + "processor": { + "event": "transaction", + "name": "transaction" + }, + "service": { + "environment": "staging", + "framework": { + "name": "Express", + "version": "1.2.3" + }, + "language": { + "name": "ecmascript", + "version": "8" + }, + "name": "1234_service-12a3", + "node": { + "name": "container-id" + }, + "runtime": { + "name": "node", + "version": "8.0.0" + }, + "version": "5.1.3" + }, + "timestamp": { + "us": 1496170422281000 + }, + "trace": { + "id": "85925e55b43f4340aaaaaaaaaaaaaaaa" + }, + "transaction": { + "duration": { + "us": 13980 + }, + "id": "85925e55b43f4340", + "name": "GET /api/types", + "result": "failure", + "sampled": true, + "span_count": { + "started": 0 + }, + "type": "request" + }, + "user": { + "email": "foo@bar.com", + "id": "123user", + "name": "foo" + } + }, + { + "@timestamp": "2017-05-30T18:53:42.000Z", + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "container": { + "id": "container-id" + }, + "ecs": { + "version": "1.5.0" + }, + "event": { + "ingested": "2020-04-29T02:16:27.091518Z" + }, + "host": { + "architecture": "x64", + "ip": "127.0.0.1", + "os": { + "platform": "darwin" + } + }, + "kubernetes": { + "namespace": "namespace1", + "pod": { + "name": "pod-name", + "uid": "pod-uid" + } + }, + "observer": { + "ephemeral_id": "3f31e73c-2fdf-4f39-8c81-57b25d00fc16", + "hostname": "goat", + "id": "0f00bacd-a23e-41c5-a2fb-4d90b2776018", + "type": "apm-server", + "version": "8.0.0", + "version_major": 8 + }, + "process": { + "args": [ + "node", + "server.js" + ], + "pid": 1234, + "ppid": 6789, + "title": "node" + }, + "processor": { + "event": "transaction", + "name": "transaction" + }, + "service": { + "environment": "staging", + "framework": { + "name": "Express", + "version": "1.2.3" + }, + "language": { + "name": "ecmascript", + "version": "8" + }, + "name": "1234_service-12a3", + "node": { + "name": "container-id" + }, + "runtime": { + "name": "node", + "version": "8.0.0" + }, + "version": "5.1.3" + }, + "timestamp": { + "us": 1496170422000000 + }, + "trace": { + "id": "85925e55b43f4341aaaaaaaaaaaaaaaa" + }, + "transaction": { + "duration": { + "us": 13980 + }, + "id": "85925e55b43f4341", + "name": "GET /api/types", + "result": "200", + "sampled": false, + "span_count": { + "started": 0 + }, + "type": "request" + }, + "user": { + "email": "foo@bar.com", + "id": "123user", + "name": "foo" + } + }, + { + "@timestamp": "2017-05-30T18:53:42.281Z", + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "container": { + "id": "container-id" + }, + "ecs": { + "version": "1.5.0" + }, + "event": { + "ingested": "2020-04-29T02:16:27.092239Z" + }, + "host": { + "architecture": "x64", + "ip": "127.0.0.1", + "os": { + "platform": "darwin" + } + }, + "kubernetes": { + "namespace": "namespace1", + "pod": { + "name": "pod-name", + "uid": "pod-uid" + } + }, + "observer": { + "ephemeral_id": "3f31e73c-2fdf-4f39-8c81-57b25d00fc16", + "hostname": "goat", + "id": "0f00bacd-a23e-41c5-a2fb-4d90b2776018", + "type": "apm-server", + "version": "8.0.0", + "version_major": 8 + }, + "process": { + "args": [ + "node", + "server.js" + ], + "pid": 1234, + "ppid": 6789, + "title": "node" + }, + "processor": { + "event": "transaction", + "name": "transaction" + }, + "service": { + "environment": "staging", + "framework": { + "name": "Express", + "version": "1.2.3" + }, + "language": { + "name": "ecmascript", + "version": "8" + }, + "name": "1234_service-12a3", + "node": { + "name": "container-id" + }, + "runtime": { + "name": "node", + "version": "8.0.0" + }, + "version": "5.1.3" + }, + "timestamp": { + "us": 1496170422281999 + }, + "trace": { + "id": "85925e55b43f4342aaaaaaaaaaaaaaaa" + }, + "transaction": { + "duration": { + "us": 13980 + }, + "id": "85925e55b43f4342", + "name": "GET /api/types", + "result": "200", + "sampled": true, + "span_count": { + "dropped": 258, + "started": 1 + }, + "type": "request" + }, + "user": { + "email": "foo@bar.com", + "id": "123user", + "name": "foo" + } + }, + { + "@timestamp": "2017-05-30T18:53:27.154Z", + "agent": { + "name": "js-base", + "version": "1.3" + }, + "client": { + "geo": { + "continent_name": "North America", + "country_iso_code": "US", + "location": { + "lat": 37.751, + "lon": -97.822 + } + }, + "ip": "8.8.8.8" + }, + "container": { + "id": "container-id" + }, + "ecs": { + "version": "1.5.0" + }, + "event": { + "ingested": "2020-04-29T02:16:26.984994Z" + }, + "host": { + "architecture": "x64", + "ip": "127.0.0.1", + "os": { + "platform": "darwin" + } + }, + "http": { + "request": { + "body": { + "original": { + "additional": { + "bar": 123, + "req": "additional information" + }, + "str": "hello world" + } + }, + "cookies": { + "c1": "v1", + "c2": "v2" + }, + "env": { + "GATEWAY_INTERFACE": "CGI/1.1", + "SERVER_SOFTWARE": "nginx" + }, + "headers": { + "Array": [ + "foo", + "bar", + "baz" + ], + "Content-Type": [ + "text/html" + ], + "Cookie": [ + "c1=v1,c2=v2" + ], + "Some-Other-Header": [ + "foo" + ], + "User-Agent": [ + "Mozilla Chrome Edge" + ] + }, + "method": "post", + "socket": { + "encrypted": true, + "remote_address": "8.8.8.8" + } + }, + "response": { + "finished": true, + "headers": { + "Content-Type": [ + "application/json" + ] + }, + "headers_sent": true, + "status_code": 200 + }, + "version": "1.1" + }, + "kubernetes": { + "namespace": "namespace1", + "pod": { + "name": "pod-name", + "uid": "pod-uid" + } + }, + "labels": { + "bool_error": false, + "number_code": 2, + "organization_uuid": "9f0e9d64-c185-4d21-a6f4-4673ed561ec8" + }, + "observer": { + "ephemeral_id": "3f31e73c-2fdf-4f39-8c81-57b25d00fc16", + "hostname": "goat", + "id": "0f00bacd-a23e-41c5-a2fb-4d90b2776018", + "type": "apm-server", + "version": "8.0.0", + "version_major": 8 + }, + "process": { + "args": [ + "node", + "server.js" + ], + "pid": 1234, + "ppid": 6789, + "title": "node" + }, + "processor": { + "event": "transaction", + "name": "transaction" + }, + "service": { + "environment": "staging", + "framework": { + "name": "Express", + "version": "1.2.3" + }, + "language": { + "name": "ecmascript", + "version": "8" + }, + "name": "serviceabc", + "node": { + "name": "special-name" + }, + "runtime": { + "name": "javascript", + "version": "8.0.0" + }, + "version": "5.1.3" + }, + "source": { + "ip": "8.8.8.8" + }, + "timestamp": { + "us": 1496170407154000 + }, + "trace": { + "id": "945254c567a5417eaaaaaaaaaaaaaaaa" + }, + "transaction": { + "custom": { + "(": "not a valid regex and that is fine", + "and_objects": { + "foo": [ + "bar", + "baz" + ] + }, + "my_key": 1, + "some_other_value": "foo bar" + }, + "duration": { + "us": 32592 + }, + "id": "945254c567a5417e", + "marks": { + "another_mark": { + "some_float": 10, + "some_long": 10 + }, + "navigationTiming": { + "appBeforeBootstrap": 608.9300000000001, + "navigationStart": -21 + } + }, + "name": "GET /api/types", + "page": { + "referer": "http://localhost:8000/test/e2e/", + "url": "http://localhost:8000/test/e2e/general-usecase/" + }, + "result": "success", + "sampled": true, + "span_count": { + "dropped": 2, + "started": 4 + }, + "type": "request" + }, + "url": { + "domain": "www.example.com", + "fragment": "#hash", + "full": "https://www.example.com/p/a/t/h?query=string#hash", + "original": "/p/a/t/h?query=string#hash", + "path": "/p/a/t/h", + "port": 8080, + "query": "?query=string", + "scheme": "https" + }, + "user": { + "email": "foo@example.com", + "id": "99" + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Other", + "original": "Mozilla Chrome Edge" + } + } +] \ No newline at end of file diff --git a/tests/system/test_aggregation.py b/tests/system/test_aggregation.py index 74bd23d32d2..4dc38325ceb 100644 --- a/tests/system/test_aggregation.py +++ b/tests/system/test_aggregation.py @@ -13,15 +13,19 @@ def config(self): cfg.update({ "aggregation_enabled": True, "aggregation_interval": "1s", + # Drop unsampled transaction events, + # to show that we aggregate before they + # are dropped. + "sampling_keep_unsampled": False, }) return cfg def test_transaction_metrics(self): self.load_docs_with_template(self.get_payload_path("transactions_spans.ndjson"), - self.intake_url, 'transaction', 9) + self.intake_url, 'transaction', 8) self.assert_no_logged_warnings() - self.wait_for_events('transaction', 4, index=index_transaction) + self.wait_for_events('transaction', 3, index=index_transaction) metric_docs = self.wait_for_events('metric', 3, index=index_metric) for doc in metric_docs: diff --git a/tests/system/test_sampling.py b/tests/system/test_sampling.py new file mode 100644 index 00000000000..410afa72812 --- /dev/null +++ b/tests/system/test_sampling.py @@ -0,0 +1,56 @@ +import time + +from apmserver import integration_test +from apmserver import ClientSideElasticTest, ElasticTest, ExpvarBaseTest, ProcStartupFailureTest +from helper import wait_until +from es_helper import index_smap, index_metric, index_transaction + + +@integration_test +class TestKeepUnsampled(ElasticTest): + def config(self): + cfg = super(TestKeepUnsampled, self).config() + cfg.update({"sampling_keep_unsampled": True}) + return cfg + + def test(self): + self.load_docs_with_template(self.get_payload_path("transactions_spans.ndjson"), + self.intake_url, 'transaction', 9) + self.assert_no_logged_warnings() + docs = self.wait_for_events('transaction', 4, index=index_transaction) + self.approve_docs('keep_unsampled_transactions', docs) + + +@integration_test +class TestDropUnsampled(ElasticTest): + def config(self): + cfg = super(TestDropUnsampled, self).config() + cfg.update({ + "sampling_keep_unsampled": False, + # Enable aggregation to avoid a warning. + "aggregation_enabled": True, + }) + return cfg + + def test(self): + self.load_docs_with_template(self.get_payload_path("transactions_spans.ndjson"), + self.intake_url, 'transaction', 8) + self.assert_no_logged_warnings() + docs = self.wait_for_events('transaction', 3, index=index_transaction) + self.approve_docs('drop_unsampled_transactions', docs) + + +@integration_test +class TestConfigWarning(ElasticTest): + def config(self): + cfg = super(TestConfigWarning, self).config() + cfg.update({ + "sampling_keep_unsampled": False, + # Disable aggregation to force a warning. + "aggregation_enabled": False, + }) + return cfg + + def test(self): + expected = "apm-server.sampling.keep_unsampled and apm-server.aggregation.enabled are both false, which will lead to incorrect metrics being reported in the APM UI" + self.assertIn(expected, self.get_log())