diff --git a/packages/gcp/_dev/build/docs/loadbalancing.md b/packages/gcp/_dev/build/docs/loadbalancing.md index 2055cde76ac..1981aa928b8 100644 --- a/packages/gcp/_dev/build/docs/loadbalancing.md +++ b/packages/gcp/_dev/build/docs/loadbalancing.md @@ -1,5 +1,13 @@ # Load Balancing +## Logs + +The `loadbalancing_logs` dataset collects logs of the requests sent to and handled by GCP Load Balancers. + +{{event "loadbalancing_logs"}} + +{{fields "loadbalancing_logs"}} + ## Metrics The `loadbalancing_metrics` dataset fetches HTTPS, HTTP, and Layer 3 metrics from [Load Balancing](https://cloud.google.com/load-balancing/) in Google Cloud Platform. It contains all metrics exported from the [GCP Load Balancing Monitoring API](https://cloud.google.com/monitoring/api/metrics_gcp#gcp-loadbalancing). diff --git a/packages/gcp/_dev/deploy/docker/docker-compose.yml b/packages/gcp/_dev/deploy/docker/docker-compose.yml index 20a769893f3..890f234a17d 100644 --- a/packages/gcp/_dev/deploy/docker/docker-compose.yml +++ b/packages/gcp/_dev/deploy/docker/docker-compose.yml @@ -61,3 +61,17 @@ services: - /sample_logs/vpcflow.log depends_on: - gcppubsub-emulator + gcppubsub-load-balancer: + image: docker.elastic.co/observability/stream:v0.7.0 + volumes: + - ./sample_logs:/sample_logs:ro + command: + - log + - --retry=30 + - --addr=gcppubsub-emulator:8681 + - -p=gcppubsub + - --gcppubsub-clear=true + - --gcppubsub-project=load_balancer + - /sample_logs/load_balancer.log + depends_on: + - gcppubsub-emulator diff --git a/packages/gcp/_dev/deploy/docker/sample_logs/load_balancer.log b/packages/gcp/_dev/deploy/docker/sample_logs/load_balancer.log new file mode 100644 index 00000000000..38af914a2cb --- /dev/null +++ b/packages/gcp/_dev/deploy/docker/sample_logs/load_balancer.log @@ -0,0 +1,3 @@ +{"insertId":"1oek5rg3l3fxj7","jsonPayload":{"@type":"type.googleapis.com/google.cloud.loadbalancin,g.type.LoadBalancerLogEntry","cacheId":"SFO-fbae48ad","statusDetails":"response_from_cache"},"httpRequest":{"requestMethod":"GET","requestUrl":"http://81.2.69.193/static/us/three-cats.jpg","requestSize":"577","status":304,"responseSize":"157","userAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36","remoteIp":"89.160.20.156","cacheHit":true,"cacheLookup":true},"resource":{"type":"http_load_balancer","labels":{"zone":"global","url_map_name":"URL_MAP_NAME","forwarding_rule_name":"FORWARDING_RULE_NAME","target_proxy_name":"TARGET_PROXY_NAME","backend_service_name":"","project_id":"PROJECT_ID"}},"timestamp":"2020-06-08T23:41:30.078651Z","severity":"INFO","logName":"projects/PROJECT_ID/logs/requests","trace":"projects/PROJECT_ID/traces/241d69833e64b3bf83fabac8c873d992","receiveTimestamp":"2020-06-08T23:41:30.588272510Z","spanId":"7b6537d3672e08e1"} +{"insertId":"1oek5rg3l3fxj7","jsonPayload":{"@type":"type.googleapis.com/google.cloud.loadbalancin,g.type.LoadBalancerLogEntry","cacheId":"SFO-fbae48ad","statusDetails":"response_from_cache"},"httpRequest":{"requestMethod":"GET","requestUrl":"http://pictures.example.com/static/us/three-cats.jpg","requestSize":"577","status":304,"responseSize":"157","userAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36","remoteIp":"89.160.20.156","cacheHit":true,"cacheLookup":true},"resource":{"type":"http_load_balancer","labels":{"zone":"global","url_map_name":"URL_MAP_NAME","forwarding_rule_name":"FORWARDING_RULE_NAME","target_proxy_name":"TARGET_PROXY_NAME","backend_service_name":"","project_id":"PROJECT_ID"}},"timestamp":"2020-06-08T23:41:30.078651Z","severity":"INFO","logName":"projects/PROJECT_ID/logs/requests","trace":"projects/PROJECT_ID/traces/241d69833e64b3bf83fabac8c873d992","receiveTimestamp":"2020-06-08T23:41:30.588272510Z","spanId":"7b6537d3672e08e1"} +{"insertId":"1oek5rg3l3fxj7","jsonPayload":{"@type":"type.googleapis.com/google.cloud.loadbalancin,g.type.LoadBalancerLogEntry","cacheId":"SFO-fbae48ad","statusDetails":"response_from_cache"},"httpRequest":{"requestMethod":"GET","requestUrl":"http://81.2.69.193:8080/static/us/three-cats.jpg","requestSize":"577","status":304,"responseSize":"157","userAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36","remoteIp":"89.160.20.156:9989","cacheHit":true,"cacheLookup":true,"serverIp":"10.5.3.1:9090","protocol":"HTTP/2.0","referer":"https://developer.mozilla.org/en-US/docs/Web/JavaScript"},"resource":{"type":"http_load_balancer","labels":{"zone":"global","url_map_name":"URL_MAP_NAME","forwarding_rule_name":"FORWARDING_RULE_NAME","target_proxy_name":"TARGET_PROXY_NAME","backend_service_name":"","project_id":"PROJECT_ID"}},"timestamp":"2020-06-08T23:41:30.078651Z","severity":"INFO","logName":"projects/PROJECT_ID/logs/requests","trace":"projects/PROJECT_ID/traces/241d69833e64b3bf83fabac8c873d992","receiveTimestamp":"2020-06-08T23:41:30.588272510Z","spanId":"7b6537d3672e08e1"} \ No newline at end of file diff --git a/packages/gcp/changelog.yml b/packages/gcp/changelog.yml index 01679b6da4b..cdb0f58a8db 100644 --- a/packages/gcp/changelog.yml +++ b/packages/gcp/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.6.0" + changes: + - description: Add Load Balancing logs datastream + type: enhancement + link: https://github.com/elastic/integrations/pull/3493 - version: "2.5.0" changes: - description: Add GCP Load Balancing Metricset diff --git a/packages/gcp/data_stream/loadbalancing_logs/_dev/test/pipeline/test-common-config.yml b/packages/gcp/data_stream/loadbalancing_logs/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..4da22641654 --- /dev/null +++ b/packages/gcp/data_stream/loadbalancing_logs/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,3 @@ +fields: + tags: + - preserve_original_event diff --git a/packages/gcp/data_stream/loadbalancing_logs/_dev/test/pipeline/test-load_balancer.log b/packages/gcp/data_stream/loadbalancing_logs/_dev/test/pipeline/test-load_balancer.log new file mode 100644 index 00000000000..38af914a2cb --- /dev/null +++ b/packages/gcp/data_stream/loadbalancing_logs/_dev/test/pipeline/test-load_balancer.log @@ -0,0 +1,3 @@ +{"insertId":"1oek5rg3l3fxj7","jsonPayload":{"@type":"type.googleapis.com/google.cloud.loadbalancin,g.type.LoadBalancerLogEntry","cacheId":"SFO-fbae48ad","statusDetails":"response_from_cache"},"httpRequest":{"requestMethod":"GET","requestUrl":"http://81.2.69.193/static/us/three-cats.jpg","requestSize":"577","status":304,"responseSize":"157","userAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36","remoteIp":"89.160.20.156","cacheHit":true,"cacheLookup":true},"resource":{"type":"http_load_balancer","labels":{"zone":"global","url_map_name":"URL_MAP_NAME","forwarding_rule_name":"FORWARDING_RULE_NAME","target_proxy_name":"TARGET_PROXY_NAME","backend_service_name":"","project_id":"PROJECT_ID"}},"timestamp":"2020-06-08T23:41:30.078651Z","severity":"INFO","logName":"projects/PROJECT_ID/logs/requests","trace":"projects/PROJECT_ID/traces/241d69833e64b3bf83fabac8c873d992","receiveTimestamp":"2020-06-08T23:41:30.588272510Z","spanId":"7b6537d3672e08e1"} +{"insertId":"1oek5rg3l3fxj7","jsonPayload":{"@type":"type.googleapis.com/google.cloud.loadbalancin,g.type.LoadBalancerLogEntry","cacheId":"SFO-fbae48ad","statusDetails":"response_from_cache"},"httpRequest":{"requestMethod":"GET","requestUrl":"http://pictures.example.com/static/us/three-cats.jpg","requestSize":"577","status":304,"responseSize":"157","userAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36","remoteIp":"89.160.20.156","cacheHit":true,"cacheLookup":true},"resource":{"type":"http_load_balancer","labels":{"zone":"global","url_map_name":"URL_MAP_NAME","forwarding_rule_name":"FORWARDING_RULE_NAME","target_proxy_name":"TARGET_PROXY_NAME","backend_service_name":"","project_id":"PROJECT_ID"}},"timestamp":"2020-06-08T23:41:30.078651Z","severity":"INFO","logName":"projects/PROJECT_ID/logs/requests","trace":"projects/PROJECT_ID/traces/241d69833e64b3bf83fabac8c873d992","receiveTimestamp":"2020-06-08T23:41:30.588272510Z","spanId":"7b6537d3672e08e1"} +{"insertId":"1oek5rg3l3fxj7","jsonPayload":{"@type":"type.googleapis.com/google.cloud.loadbalancin,g.type.LoadBalancerLogEntry","cacheId":"SFO-fbae48ad","statusDetails":"response_from_cache"},"httpRequest":{"requestMethod":"GET","requestUrl":"http://81.2.69.193:8080/static/us/three-cats.jpg","requestSize":"577","status":304,"responseSize":"157","userAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36","remoteIp":"89.160.20.156:9989","cacheHit":true,"cacheLookup":true,"serverIp":"10.5.3.1:9090","protocol":"HTTP/2.0","referer":"https://developer.mozilla.org/en-US/docs/Web/JavaScript"},"resource":{"type":"http_load_balancer","labels":{"zone":"global","url_map_name":"URL_MAP_NAME","forwarding_rule_name":"FORWARDING_RULE_NAME","target_proxy_name":"TARGET_PROXY_NAME","backend_service_name":"","project_id":"PROJECT_ID"}},"timestamp":"2020-06-08T23:41:30.078651Z","severity":"INFO","logName":"projects/PROJECT_ID/logs/requests","trace":"projects/PROJECT_ID/traces/241d69833e64b3bf83fabac8c873d992","receiveTimestamp":"2020-06-08T23:41:30.588272510Z","spanId":"7b6537d3672e08e1"} \ No newline at end of file diff --git a/packages/gcp/data_stream/loadbalancing_logs/_dev/test/pipeline/test-load_balancer.log-expected.json b/packages/gcp/data_stream/loadbalancing_logs/_dev/test/pipeline/test-load_balancer.log-expected.json new file mode 100644 index 00000000000..cb02b1de243 --- /dev/null +++ b/packages/gcp/data_stream/loadbalancing_logs/_dev/test/pipeline/test-load_balancer.log-expected.json @@ -0,0 +1,321 @@ +{ + "expected": [ + { + "@timestamp": "2020-06-08T23:41:30.078Z", + "cloud": { + "project": { + "id": "PROJECT_ID" + }, + "region": "global" + }, + "destination": { + "address": "81.2.69.193", + "ip": "81.2.69.193" + }, + "ecs": { + "version": "8.4.0" + }, + "event": { + "category": "network", + "created": "2020-06-08T23:41:30.588Z", + "id": "1oek5rg3l3fxj7", + "kind": "event", + "original": "{\"insertId\":\"1oek5rg3l3fxj7\",\"jsonPayload\":{\"@type\":\"type.googleapis.com/google.cloud.loadbalancin,g.type.LoadBalancerLogEntry\",\"cacheId\":\"SFO-fbae48ad\",\"statusDetails\":\"response_from_cache\"},\"httpRequest\":{\"requestMethod\":\"GET\",\"requestUrl\":\"http://81.2.69.193/static/us/three-cats.jpg\",\"requestSize\":\"577\",\"status\":304,\"responseSize\":\"157\",\"userAgent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36\",\"remoteIp\":\"89.160.20.156\",\"cacheHit\":true,\"cacheLookup\":true},\"resource\":{\"type\":\"http_load_balancer\",\"labels\":{\"zone\":\"global\",\"url_map_name\":\"URL_MAP_NAME\",\"forwarding_rule_name\":\"FORWARDING_RULE_NAME\",\"target_proxy_name\":\"TARGET_PROXY_NAME\",\"backend_service_name\":\"\",\"project_id\":\"PROJECT_ID\"}},\"timestamp\":\"2020-06-08T23:41:30.078651Z\",\"severity\":\"INFO\",\"logName\":\"projects/PROJECT_ID/logs/requests\",\"trace\":\"projects/PROJECT_ID/traces/241d69833e64b3bf83fabac8c873d992\",\"receiveTimestamp\":\"2020-06-08T23:41:30.588272510Z\",\"spanId\":\"7b6537d3672e08e1\"}", + "type": "info" + }, + "gcp": { + "load_balancer": { + "backend_service_name": "", + "cache_hit": true, + "cache_id": "SFO-fbae48ad", + "cache_lookup": true, + "forwarding_rule_name": "FORWARDING_RULE_NAME", + "status_details": "response_from_cache", + "target_proxy_name": "TARGET_PROXY_NAME", + "url_map_name": "URL_MAP_NAME" + } + }, + "http": { + "request": { + "bytes": 577, + "method": "GET" + }, + "response": { + "bytes": 157, + "status_code": 304 + } + }, + "log": { + "level": "INFO", + "logger": "projects/PROJECT_ID/logs/requests" + }, + "related": { + "ip": [ + "89.160.20.156", + "81.2.69.193" + ] + }, + "source": { + "address": "89.160.20.156", + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, + "ip": "89.160.20.156" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "domain": "81.2.69.193", + "extension": "jpg", + "original": "http://81.2.69.193/static/us/three-cats.jpg", + "path": "/static/us/three-cats.jpg", + "scheme": "http" + }, + "user_agent": { + "device": { + "name": "Mac" + }, + "name": "Chrome", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36", + "os": { + "full": "Mac OS X 10.14.6", + "name": "Mac OS X", + "version": "10.14.6" + }, + "version": "83.0.4103.61" + } + }, + { + "@timestamp": "2020-06-08T23:41:30.078Z", + "cloud": { + "project": { + "id": "PROJECT_ID" + }, + "region": "global" + }, + "destination": { + "address": "pictures.example.com", + "domain": "pictures.example.com" + }, + "ecs": { + "version": "8.4.0" + }, + "event": { + "category": "network", + "created": "2020-06-08T23:41:30.588Z", + "id": "1oek5rg3l3fxj7", + "kind": "event", + "original": "{\"insertId\":\"1oek5rg3l3fxj7\",\"jsonPayload\":{\"@type\":\"type.googleapis.com/google.cloud.loadbalancin,g.type.LoadBalancerLogEntry\",\"cacheId\":\"SFO-fbae48ad\",\"statusDetails\":\"response_from_cache\"},\"httpRequest\":{\"requestMethod\":\"GET\",\"requestUrl\":\"http://pictures.example.com/static/us/three-cats.jpg\",\"requestSize\":\"577\",\"status\":304,\"responseSize\":\"157\",\"userAgent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36\",\"remoteIp\":\"89.160.20.156\",\"cacheHit\":true,\"cacheLookup\":true},\"resource\":{\"type\":\"http_load_balancer\",\"labels\":{\"zone\":\"global\",\"url_map_name\":\"URL_MAP_NAME\",\"forwarding_rule_name\":\"FORWARDING_RULE_NAME\",\"target_proxy_name\":\"TARGET_PROXY_NAME\",\"backend_service_name\":\"\",\"project_id\":\"PROJECT_ID\"}},\"timestamp\":\"2020-06-08T23:41:30.078651Z\",\"severity\":\"INFO\",\"logName\":\"projects/PROJECT_ID/logs/requests\",\"trace\":\"projects/PROJECT_ID/traces/241d69833e64b3bf83fabac8c873d992\",\"receiveTimestamp\":\"2020-06-08T23:41:30.588272510Z\",\"spanId\":\"7b6537d3672e08e1\"}", + "type": "info" + }, + "gcp": { + "load_balancer": { + "backend_service_name": "", + "cache_hit": true, + "cache_id": "SFO-fbae48ad", + "cache_lookup": true, + "forwarding_rule_name": "FORWARDING_RULE_NAME", + "status_details": "response_from_cache", + "target_proxy_name": "TARGET_PROXY_NAME", + "url_map_name": "URL_MAP_NAME" + } + }, + "http": { + "request": { + "bytes": 577, + "method": "GET" + }, + "response": { + "bytes": 157, + "status_code": 304 + } + }, + "log": { + "level": "INFO", + "logger": "projects/PROJECT_ID/logs/requests" + }, + "related": { + "hosts": [ + "pictures.example.com" + ], + "ip": [ + "89.160.20.156" + ] + }, + "source": { + "address": "89.160.20.156", + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, + "ip": "89.160.20.156" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "extension": "jpg", + "original": "http://pictures.example.com/static/us/three-cats.jpg", + "path": "/static/us/three-cats.jpg", + "scheme": "http" + }, + "user_agent": { + "device": { + "name": "Mac" + }, + "name": "Chrome", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36", + "os": { + "full": "Mac OS X 10.14.6", + "name": "Mac OS X", + "version": "10.14.6" + }, + "version": "83.0.4103.61" + } + }, + { + "@timestamp": "2020-06-08T23:41:30.078Z", + "cloud": { + "project": { + "id": "PROJECT_ID" + }, + "region": "global" + }, + "destination": { + "address": "81.2.69.193", + "ip": "81.2.69.193", + "nat": { + "ip": "10.5.3.1", + "port": 9090 + }, + "port": 8080 + }, + "ecs": { + "version": "8.4.0" + }, + "event": { + "category": "network", + "created": "2020-06-08T23:41:30.588Z", + "id": "1oek5rg3l3fxj7", + "kind": "event", + "original": "{\"insertId\":\"1oek5rg3l3fxj7\",\"jsonPayload\":{\"@type\":\"type.googleapis.com/google.cloud.loadbalancin,g.type.LoadBalancerLogEntry\",\"cacheId\":\"SFO-fbae48ad\",\"statusDetails\":\"response_from_cache\"},\"httpRequest\":{\"requestMethod\":\"GET\",\"requestUrl\":\"http://81.2.69.193:8080/static/us/three-cats.jpg\",\"requestSize\":\"577\",\"status\":304,\"responseSize\":\"157\",\"userAgent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36\",\"remoteIp\":\"89.160.20.156:9989\",\"cacheHit\":true,\"cacheLookup\":true,\"serverIp\":\"10.5.3.1:9090\",\"protocol\":\"HTTP/2.0\",\"referer\":\"https://developer.mozilla.org/en-US/docs/Web/JavaScript\"},\"resource\":{\"type\":\"http_load_balancer\",\"labels\":{\"zone\":\"global\",\"url_map_name\":\"URL_MAP_NAME\",\"forwarding_rule_name\":\"FORWARDING_RULE_NAME\",\"target_proxy_name\":\"TARGET_PROXY_NAME\",\"backend_service_name\":\"\",\"project_id\":\"PROJECT_ID\"}},\"timestamp\":\"2020-06-08T23:41:30.078651Z\",\"severity\":\"INFO\",\"logName\":\"projects/PROJECT_ID/logs/requests\",\"trace\":\"projects/PROJECT_ID/traces/241d69833e64b3bf83fabac8c873d992\",\"receiveTimestamp\":\"2020-06-08T23:41:30.588272510Z\",\"spanId\":\"7b6537d3672e08e1\"}", + "type": "info" + }, + "gcp": { + "load_balancer": { + "backend_service_name": "", + "cache_hit": true, + "cache_id": "SFO-fbae48ad", + "cache_lookup": true, + "forwarding_rule_name": "FORWARDING_RULE_NAME", + "status_details": "response_from_cache", + "target_proxy_name": "TARGET_PROXY_NAME", + "url_map_name": "URL_MAP_NAME" + } + }, + "http": { + "request": { + "bytes": 577, + "method": "GET", + "referrer": "https://developer.mozilla.org/en-US/docs/Web/JavaScript" + }, + "response": { + "bytes": 157, + "status_code": 304 + }, + "version": "2.0" + }, + "log": { + "level": "INFO", + "logger": "projects/PROJECT_ID/logs/requests" + }, + "network": { + "protocol": "http" + }, + "related": { + "ip": [ + "89.160.20.156", + "81.2.69.193", + "10.5.3.1" + ] + }, + "source": { + "address": "89.160.20.156", + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, + "ip": "89.160.20.156", + "port": 9989 + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "domain": "81.2.69.193", + "extension": "jpg", + "original": "http://81.2.69.193:8080/static/us/three-cats.jpg", + "path": "/static/us/three-cats.jpg", + "port": 8080, + "scheme": "http" + }, + "user_agent": { + "device": { + "name": "Mac" + }, + "name": "Chrome", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36", + "os": { + "full": "Mac OS X 10.14.6", + "name": "Mac OS X", + "version": "10.14.6" + }, + "version": "83.0.4103.61" + } + } + ] +} \ No newline at end of file diff --git a/packages/gcp/data_stream/loadbalancing_logs/_dev/test/system/test-pubsub-config.yml b/packages/gcp/data_stream/loadbalancing_logs/_dev/test/system/test-pubsub-config.yml new file mode 100644 index 00000000000..5091bb81281 --- /dev/null +++ b/packages/gcp/data_stream/loadbalancing_logs/_dev/test/system/test-pubsub-config.yml @@ -0,0 +1,22 @@ +service: gcppubsub-emulator +input: gcp-pubsub +vars: + credentials_json: | + { + "type": "service_account", + "project_id": "foo", + "private_key_id": "x", + "private_key": "", + "client_email": "foo@bar.com", + "client_id": "0", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://foo.bar/path" + } + project_id: load_balancer +data_stream: + vars: + subscription_name: subscription + topic: topic + alternative_host: "{{Hostname}}:{{Port}}" diff --git a/packages/gcp/data_stream/loadbalancing_logs/agent/stream/gcp-pubsub.yml.hbs b/packages/gcp/data_stream/loadbalancing_logs/agent/stream/gcp-pubsub.yml.hbs new file mode 100644 index 00000000000..d582de0a805 --- /dev/null +++ b/packages/gcp/data_stream/loadbalancing_logs/agent/stream/gcp-pubsub.yml.hbs @@ -0,0 +1,27 @@ +project_id: {{project_id}} +topic: {{topic}} +subscription.name: {{subscription_name}} +{{#if credentials_file}} +credentials_file: {{credentials_file}} +{{/if}} +{{#if credentials_json}} +credentials_json: '{{credentials_json}}' +{{/if}} +{{#if alternative_host}} +alternative_host: {{alternative_host}} +{{/if}} +subscription.create: {{subscription_create}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/gcp/data_stream/loadbalancing_logs/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/loadbalancing_logs/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..eb4ae24a860 --- /dev/null +++ b/packages/gcp/data_stream/loadbalancing_logs/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,215 @@ +--- +description: Pipeline for Google Cloud DNS logs + +processors: + - set: + field: ecs.version + value: '8.4.0' + - rename: + field: message + target_field: event.original + ignore_missing: true + - json: + field: event.original + target_field: json + - set: + field: event.kind + value: event + - set: + field: event.category + value: network + - set: + field: event.type + value: info + - date: + field: json.timestamp + timezone: UTC + formats: + - ISO8601 + - date: + field: json.receiveTimestamp + target_field: event.created + timezone: UTC + formats: + - ISO8601 + - rename: + field: json.logName + target_field: log.logger + ignore_missing: true + - set: + field: event.id + copy_from: json.insertId + ignore_empty_value: true + ignore_failure: true + - convert: + field: json.resource.labels.project_id + target_field: cloud.project.id + type: string + ignore_failure: true + - convert: + field: json.resource.labels.zone + target_field: cloud.region + type: string + ignore_failure: true + - grok: + field: json.httpRequest.remoteIp + ignore_missing: true + patterns: + - ^%{IP:source.address}(:%{POSINT:source.port:long})?$ + - convert: + field: source.address + target_field: source.ip + type: ip + ignore_failure: true + - geoip: + field: source.ip + target_field: source.geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + - rename: + field: json.httpRequest.requestMethod + target_field: http.request.method + ignore_missing: true + - convert: + field: json.httpRequest.requestSize + target_field: http.request.bytes + type: long + ignore_missing: true + - convert: + field: json.httpRequest.responseSize + target_field: http.response.bytes + type: long + ignore_missing: true + - rename: + field: json.httpRequest.status + target_field: http.response.status_code + ignore_missing: true + - dissect: + field: json.httpRequest.protocol + pattern: "%{network.protocol}/%{http.version}" + ignore_failure: true + if: ctx.json?.httpRequest?.protocol != null + - lowercase: + field: network.protocol + ignore_missing: true + - user_agent: + field: json.httpRequest.userAgent + target_field: user_agent + ignore_missing: true + - uri_parts: + field: json.httpRequest.requestUrl + target_field: url + if: ctx.json?.httpRequest?.requestUrl != null + - rename: + field: json.httpRequest.referer + target_field: http.request.referrer + ignore_missing: true + - grok: + field: json.httpRequest.serverIp + ignore_missing: true + patterns: + - ^%{IP:destination.nat.ip}(:%{POSINT:destination.nat.port:long})?$ + - set: + field: destination.address + copy_from: url.domain + ignore_empty_value: true + ignore_failure: true + - set: + field: destination.port + copy_from: url.port + ignore_empty_value: true + ignore_failure: true + - convert: + field: destination.address + target_field: destination.ip + type: ip + ignore_missing: true + on_failure: + - rename: + field: url.domain + target_field: destination.domain + ignore_missing: true + ignore_failure: true + - rename: + field: json.severity + target_field: log.level + ignore_missing: true + - rename: + field: json.jsonPayload.cacheId + target_field: gcp.load_balancer.cache_id + ignore_missing: true + - rename: + field: json.jsonPayload.statusDetails + target_field: gcp.load_balancer.status_details + ignore_missing: true + - rename: + field: json.httpRequest.cacheHit + target_field: gcp.load_balancer.cache_hit + ignore_missing: true + - rename: + field: json.httpRequest.cacheLookup + target_field: gcp.load_balancer.cache_lookup + ignore_missing: true + - rename: + field: json.resource.labels.url_map_name + target_field: gcp.load_balancer.url_map_name + ignore_missing: true + - rename: + field: json.resource.labels.forwarding_rule_name + target_field: gcp.load_balancer.forwarding_rule_name + ignore_missing: true + - rename: + field: json.resource.labels.target_proxy_name + target_field: gcp.load_balancer.target_proxy_name + ignore_missing: true + - rename: + field: json.resource.labels.backend_service_name + target_field: gcp.load_balancer.backend_service_name + ignore_missing: true + - append: + field: related.ip + value: "{{source.ip}}" + allow_duplicates: false + if: ctx?.source?.ip != null + - append: + field: related.ip + value: "{{destination.ip}}" + allow_duplicates: false + if: ctx?.destination?.ip != null + - append: + field: related.ip + value: "{{destination.nat.ip}}" + allow_duplicates: false + if: ctx?.destination?.nat?.ip != null + - append: + field: related.hosts + value: "{{destination.domain}}" + allow_duplicates: false + if: ctx?.destination?.domain != null + - remove: + field: + - json + ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/packages/gcp/data_stream/loadbalancing_logs/fields/agent.yml b/packages/gcp/data_stream/loadbalancing_logs/fields/agent.yml new file mode 100644 index 00000000000..e313ec82874 --- /dev/null +++ b/packages/gcp/data_stream/loadbalancing_logs/fields/agent.yml @@ -0,0 +1,204 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/gcp/data_stream/loadbalancing_logs/fields/base-fields.yml b/packages/gcp/data_stream/loadbalancing_logs/fields/base-fields.yml new file mode 100644 index 00000000000..5d5236cac3a --- /dev/null +++ b/packages/gcp/data_stream/loadbalancing_logs/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: gcp +- name: event.dataset + type: constant_keyword + description: Event dataset + value: gcp.loadbalancing_logs diff --git a/packages/gcp/data_stream/loadbalancing_logs/fields/ecs.yml b/packages/gcp/data_stream/loadbalancing_logs/fields/ecs.yml new file mode 100644 index 00000000000..bfb550fbde5 --- /dev/null +++ b/packages/gcp/data_stream/loadbalancing_logs/fields/ecs.yml @@ -0,0 +1,100 @@ +- external: ecs + name: tags +- external: ecs + name: ecs.version +- external: ecs + name: event.created +- external: ecs + name: log.logger +- external: ecs + name: log.level +- external: ecs + name: http.request.bytes +- external: ecs + name: http.request.method +- external: ecs + name: http.response.bytes +- external: ecs + name: http.request.referrer +- external: ecs + name: http.version +- external: ecs + name: http.response.status_code +- external: ecs + name: source.as.number +- external: ecs + name: source.as.organization.name +- external: ecs + name: source.geo.city_name +- external: ecs + name: source.geo.continent_name +- external: ecs + name: source.geo.country_iso_code +- external: ecs + name: source.geo.country_name +- external: ecs + name: source.geo.location +- external: ecs + name: source.geo.region_iso_code +- external: ecs + name: source.geo.region_name +- external: ecs + name: source.ip +- external: ecs + name: source.address +- external: ecs + name: source.port +- external: ecs + name: url.domain +- external: ecs + name: url.extension +- external: ecs + name: url.original +- external: ecs + name: url.path +- external: ecs + name: url.query +- external: ecs + name: url.scheme +- external: ecs + name: url.port +- external: ecs + name: user_agent.device.name +- external: ecs + name: user_agent.device.name +- external: ecs + name: user_agent.name +- external: ecs + name: user_agent.name +- external: ecs + name: user_agent.original +- external: ecs + name: user_agent.original +- external: ecs + name: user_agent.os.full +- external: ecs + name: user_agent.os.name +- external: ecs + name: user_agent.os.name +- external: ecs + name: user_agent.os.version +- external: ecs + name: user_agent.version +- external: ecs + name: destination.ip +- external: ecs + name: destination.address +- external: ecs + name: destination.domain +- external: ecs + name: destination.port +- external: ecs + name: destination.nat.ip +- external: ecs + name: destination.nat.port +- external: ecs + name: related.ip +- external: ecs + name: related.hosts +- external: ecs + name: network.protocol diff --git a/packages/gcp/data_stream/loadbalancing_logs/fields/fields.yml b/packages/gcp/data_stream/loadbalancing_logs/fields/fields.yml new file mode 100644 index 00000000000..5c4162eb070 --- /dev/null +++ b/packages/gcp/data_stream/loadbalancing_logs/fields/fields.yml @@ -0,0 +1,35 @@ +- name: gcp.load_balancer + type: group + fields: + - name: backend_service_name + type: keyword + description: | + The backend service to which the load balancer is sending traffic + - name: cache_hit + type: boolean + description: | + Whether or not an entity was served from cache (with or without validation). + - name: cache_id + type: keyword + description: >- + Indicates the location and cache instance that the cache response was served from. For example, a cache response served from a cache in Amsterdam would have a cacheId value of AMS-85e2bd4b, where AMS is the IATA code, and 85e2bd4b is an opaque identifier of the cache instance (because some Cloud CDN locations have multiple discrete caches). + - name: cache_lookup + type: boolean + description: | + Whether or not a cache lookup was attempted. + - name: forwarding_rule_name + type: keyword + description: | + The name of the forwarding rule + - name: status_details + type: keyword + description: >- + Explains why the load balancer returned the HTTP status that it did. See https://cloud.google.com/cdn/docs/cdn-logging-monitoring#statusdetail_http_success_messages for specific messages. + - name: target_proxy_name + type: keyword + description: | + The target proxy name + - name: url_map_name + type: keyword + description: | + The URL map name diff --git a/packages/gcp/data_stream/loadbalancing_logs/manifest.yml b/packages/gcp/data_stream/loadbalancing_logs/manifest.yml new file mode 100644 index 00000000000..d5705643437 --- /dev/null +++ b/packages/gcp/data_stream/loadbalancing_logs/manifest.yml @@ -0,0 +1,65 @@ +type: logs +title: Google Cloud Platform (GCP) Load Balancing logs +streams: + - input: gcp-pubsub + vars: + - name: topic + type: text + title: Topic + description: Name of the topic where the logs are written to. + multi: false + required: true + show_user: true + default: cloud-logging-load_balancer + - name: subscription_name + type: text + title: Subscription Name + description: Use the short subscription name here, not the full-blown path with the project ID. You can find it as "Subscription ID" on the Google Cloud Console. + multi: false + required: true + show_user: true + default: filebeat-gcp-load_balancer + - name: subscription_create + type: bool + title: Subscription Create + description: If true, the integration will create the subscription on start. + multi: false + required: true + show_user: false + default: false + - name: alternative_host + type: text + title: Alternative host + multi: false + required: false + show_user: false + description: "Overrides the default Pub/Sub service address and disables TLS. For testing." + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - gcp-loadbalancing_logs + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + + template_path: gcp-pubsub.yml.hbs + title: Google Cloud Platform (GCP) Load Balancing logs (gcp-pubsub) + description: Collect Google Cloud Platform (GCP) Load Balancing logs using gcp-pubsub input diff --git a/packages/gcp/data_stream/loadbalancing_logs/sample_event.json b/packages/gcp/data_stream/loadbalancing_logs/sample_event.json new file mode 100644 index 00000000000..2f777d56509 --- /dev/null +++ b/packages/gcp/data_stream/loadbalancing_logs/sample_event.json @@ -0,0 +1,136 @@ +{ + "@timestamp": "2020-06-08T23:41:30.078Z", + "agent": { + "ephemeral_id": "1f7633a7-3410-4684-bb55-14b0bd0e2bd4", + "hostname": "docker-fleet-agent", + "id": "df142714-8028-4ef0-a80c-4eb03051c084", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "7.17.0" + }, + "cloud": { + "project": { + "id": "PROJECT_ID" + }, + "region": "global" + }, + "data_stream": { + "dataset": "gcp.loadbalancing_logs", + "namespace": "ep", + "type": "logs" + }, + "destination": { + "address": "81.2.69.193", + "ip": "81.2.69.193", + "nat": { + "ip": "10.5.3.1", + "port": 9090 + }, + "port": 8080 + }, + "ecs": { + "version": "8.2.0" + }, + "elastic_agent": { + "id": "df142714-8028-4ef0-a80c-4eb03051c084", + "snapshot": false, + "version": "7.17.0" + }, + "event": { + "category": "network", + "created": "2020-06-08T23:41:30.588Z", + "id": "1oek5rg3l3fxj7", + "kind": "event", + "original": "{\"insertId\":\"1oek5rg3l3fxj7\",\"jsonPayload\":{\"@type\":\"type.googleapis.com/google.cloud.loadbalancin,g.type.LoadBalancerLogEntry\",\"cacheId\":\"SFO-fbae48ad\",\"statusDetails\":\"response_from_cache\"},\"httpRequest\":{\"requestMethod\":\"GET\",\"requestUrl\":\"http://81.2.69.193:8080/static/us/three-cats.jpg\",\"requestSize\":\"577\",\"status\":304,\"responseSize\":\"157\",\"userAgent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36\",\"remoteIp\":\"89.160.20.156:9989\",\"cacheHit\":true,\"cacheLookup\":true,\"serverIp\":\"10.5.3.1:9090\",\"protocol\":\"HTTP/2.0\",\"referer\":\"https://developer.mozilla.org/en-US/docs/Web/JavaScript\"},\"resource\":{\"type\":\"http_load_balancer\",\"labels\":{\"zone\":\"global\",\"url_map_name\":\"URL_MAP_NAME\",\"forwarding_rule_name\":\"FORWARDING_RULE_NAME\",\"target_proxy_name\":\"TARGET_PROXY_NAME\",\"backend_service_name\":\"\",\"project_id\":\"PROJECT_ID\"}},\"timestamp\":\"2020-06-08T23:41:30.078651Z\",\"severity\":\"INFO\",\"logName\":\"projects/PROJECT_ID/logs/requests\",\"trace\":\"projects/PROJECT_ID/traces/241d69833e64b3bf83fabac8c873d992\",\"receiveTimestamp\":\"2020-06-08T23:41:30.588272510Z\",\"spanId\":\"7b6537d3672e08e1\"}", + "type": "info" + }, + "gcp": { + "load_balancer": { + "backend_service_name": "", + "cache_hit": true, + "cache_id": "SFO-fbae48ad", + "cache_lookup": true, + "forwarding_rule_name": "FORWARDING_RULE_NAME", + "status_details": "response_from_cache", + "target_proxy_name": "TARGET_PROXY_NAME", + "url_map_name": "URL_MAP_NAME" + } + }, + "http": { + "request": { + "bytes": 577, + "method": "GET", + "referrer": "https://developer.mozilla.org/en-US/docs/Web/JavaScript" + }, + "response": { + "bytes": 157, + "status_code": 304 + }, + "version": "2.0" + }, + "input": { + "type": "gcp-pubsub" + }, + "log": { + "level": "INFO", + "logger": "projects/PROJECT_ID/logs/requests" + }, + "network": { + "protocol": "http" + }, + "related": { + "ip": [ + "89.160.20.156", + "81.2.69.193", + "10.5.3.1" + ] + }, + "source": { + "address": "89.160.20.156", + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, + "ip": "89.160.20.156", + "port": 9989 + }, + "tags": [ + "forwarded", + "gcp-firewall" + ], + "url": { + "domain": "81.2.69.193", + "extension": "jpg", + "original": "http://81.2.69.193:8080/static/us/three-cats.jpg", + "path": "/static/us/three-cats.jpg", + "port": 8080, + "scheme": "http" + }, + "user_agent": { + "device": { + "name": "Mac" + }, + "name": "Chrome", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36", + "os": { + "full": "Mac OS X 10.14.6", + "name": "Mac OS X", + "version": "10.14.6" + }, + "version": "83.0.4103.61" + } +} \ No newline at end of file diff --git a/packages/gcp/docs/loadbalancing.md b/packages/gcp/docs/loadbalancing.md index 36f520a32e8..7835ad57cc4 100644 --- a/packages/gcp/docs/loadbalancing.md +++ b/packages/gcp/docs/loadbalancing.md @@ -1,5 +1,253 @@ # Load Balancing +## Logs + +The `loadbalancing_logs` dataset collects logs of the requests sent to and handled by GCP Load Balancers. + +An example event for `loadbalancing` looks as following: + +```json +{ + "@timestamp": "2020-06-08T23:41:30.078Z", + "agent": { + "ephemeral_id": "1f7633a7-3410-4684-bb55-14b0bd0e2bd4", + "hostname": "docker-fleet-agent", + "id": "df142714-8028-4ef0-a80c-4eb03051c084", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "7.17.0" + }, + "cloud": { + "project": { + "id": "PROJECT_ID" + }, + "region": "global" + }, + "data_stream": { + "dataset": "gcp.loadbalancing_logs", + "namespace": "ep", + "type": "logs" + }, + "destination": { + "address": "81.2.69.193", + "ip": "81.2.69.193", + "nat": { + "ip": "10.5.3.1", + "port": 9090 + }, + "port": 8080 + }, + "ecs": { + "version": "8.2.0" + }, + "elastic_agent": { + "id": "df142714-8028-4ef0-a80c-4eb03051c084", + "snapshot": false, + "version": "7.17.0" + }, + "event": { + "category": "network", + "created": "2020-06-08T23:41:30.588Z", + "id": "1oek5rg3l3fxj7", + "kind": "event", + "original": "{\"insertId\":\"1oek5rg3l3fxj7\",\"jsonPayload\":{\"@type\":\"type.googleapis.com/google.cloud.loadbalancin,g.type.LoadBalancerLogEntry\",\"cacheId\":\"SFO-fbae48ad\",\"statusDetails\":\"response_from_cache\"},\"httpRequest\":{\"requestMethod\":\"GET\",\"requestUrl\":\"http://81.2.69.193:8080/static/us/three-cats.jpg\",\"requestSize\":\"577\",\"status\":304,\"responseSize\":\"157\",\"userAgent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36\",\"remoteIp\":\"89.160.20.156:9989\",\"cacheHit\":true,\"cacheLookup\":true,\"serverIp\":\"10.5.3.1:9090\",\"protocol\":\"HTTP/2.0\",\"referer\":\"https://developer.mozilla.org/en-US/docs/Web/JavaScript\"},\"resource\":{\"type\":\"http_load_balancer\",\"labels\":{\"zone\":\"global\",\"url_map_name\":\"URL_MAP_NAME\",\"forwarding_rule_name\":\"FORWARDING_RULE_NAME\",\"target_proxy_name\":\"TARGET_PROXY_NAME\",\"backend_service_name\":\"\",\"project_id\":\"PROJECT_ID\"}},\"timestamp\":\"2020-06-08T23:41:30.078651Z\",\"severity\":\"INFO\",\"logName\":\"projects/PROJECT_ID/logs/requests\",\"trace\":\"projects/PROJECT_ID/traces/241d69833e64b3bf83fabac8c873d992\",\"receiveTimestamp\":\"2020-06-08T23:41:30.588272510Z\",\"spanId\":\"7b6537d3672e08e1\"}", + "type": "info" + }, + "gcp": { + "load_balancer": { + "backend_service_name": "", + "cache_hit": true, + "cache_id": "SFO-fbae48ad", + "cache_lookup": true, + "forwarding_rule_name": "FORWARDING_RULE_NAME", + "status_details": "response_from_cache", + "target_proxy_name": "TARGET_PROXY_NAME", + "url_map_name": "URL_MAP_NAME" + } + }, + "http": { + "request": { + "bytes": 577, + "method": "GET", + "referrer": "https://developer.mozilla.org/en-US/docs/Web/JavaScript" + }, + "response": { + "bytes": 157, + "status_code": 304 + }, + "version": "2.0" + }, + "input": { + "type": "gcp-pubsub" + }, + "log": { + "level": "INFO", + "logger": "projects/PROJECT_ID/logs/requests" + }, + "network": { + "protocol": "http" + }, + "related": { + "ip": [ + "89.160.20.156", + "81.2.69.193", + "10.5.3.1" + ] + }, + "source": { + "address": "89.160.20.156", + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, + "ip": "89.160.20.156", + "port": 9989 + }, + "tags": [ + "forwarded", + "gcp-firewall" + ], + "url": { + "domain": "81.2.69.193", + "extension": "jpg", + "original": "http://81.2.69.193:8080/static/us/three-cats.jpg", + "path": "/static/us/three-cats.jpg", + "port": 8080, + "scheme": "http" + }, + "user_agent": { + "device": { + "name": "Mac" + }, + "name": "Chrome", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36", + "os": { + "full": "Mac OS X 10.14.6", + "name": "Mac OS X", + "version": "10.14.6" + }, + "version": "83.0.4103.61" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | +| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | +| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | +| destination.nat.ip | Translated ip of destination based NAT sessions (e.g. internet to private DMZ) Typically used with load balancers, firewalls, or routers. | ip | +| destination.nat.port | Port the source session is translated to by NAT Device. Typically used with load balancers, firewalls, or routers. | long | +| destination.port | Port of the destination. | long | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | +| gcp.load_balancer.backend_service_name | The backend service to which the load balancer is sending traffic | keyword | +| gcp.load_balancer.cache_hit | Whether or not an entity was served from cache (with or without validation). | boolean | +| gcp.load_balancer.cache_id | Indicates the location and cache instance that the cache response was served from. For example, a cache response served from a cache in Amsterdam would have a cacheId value of AMS-85e2bd4b, where AMS is the IATA code, and 85e2bd4b is an opaque identifier of the cache instance (because some Cloud CDN locations have multiple discrete caches). | keyword | +| gcp.load_balancer.cache_lookup | Whether or not a cache lookup was attempted. | boolean | +| gcp.load_balancer.forwarding_rule_name | The name of the forwarding rule | keyword | +| gcp.load_balancer.status_details | Explains why the load balancer returned the HTTP status that it did. See https://cloud.google.com/cdn/docs/cdn-logging-monitoring#statusdetail_http_success_messages for specific messages. | keyword | +| gcp.load_balancer.target_proxy_name | The target proxy name | keyword | +| gcp.load_balancer.url_map_name | The URL map name | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.name.text | Multi-field of `host.os.name`. | text | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| http.request.bytes | Total size in bytes of the request (body and headers). | long | +| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | +| http.request.referrer | Referrer for this HTTP request. | keyword | +| http.response.bytes | Total size in bytes of the response (body and headers). | long | +| http.response.status_code | HTTP response status code. | long | +| http.version | HTTP version. | keyword | +| input.type | Input type | keyword | +| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | +| log.logger | The name of the logger inside an application. This is usually the name of the class which initialized the logger, or can be a custom name. | keyword | +| log.offset | Log offset | long | +| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | +| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | +| related.ip | All of the IPs seen on your event. | ip | +| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | +| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | +| source.as.organization.name | Organization name. | keyword | +| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | +| source.geo.city_name | City name. | keyword | +| source.geo.continent_name | Name of the continent. | keyword | +| source.geo.country_iso_code | Country ISO code. | keyword | +| source.geo.country_name | Country name. | keyword | +| source.geo.location | Longitude and latitude. | geo_point | +| source.geo.region_iso_code | Region ISO code. | keyword | +| source.geo.region_name | Region name. | keyword | +| source.ip | IP address of the source (IPv4 or IPv6). | ip | +| source.port | Port of the source. | long | +| tags | List of keywords used to tag each event. | keyword | +| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | +| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | +| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | +| url.original.text | Multi-field of `url.original`. | match_only_text | +| url.path | Path of the request, such as "/search". | wildcard | +| url.port | Port of the request, such as 443. | long | +| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | +| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | +| user_agent.device.name | Name of the device. | keyword | +| user_agent.name | Name of the user agent. | keyword | +| user_agent.original | Unparsed user_agent string. | keyword | +| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | +| user_agent.os.full | Operating system name, including the version or code name. | keyword | +| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | +| user_agent.os.name | Operating system name, without the version. | keyword | +| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | +| user_agent.os.version | Operating system version as a raw string. | keyword | +| user_agent.version | Version of the user agent. | keyword | + + ## Metrics The `loadbalancing_metrics` dataset fetches HTTPS, HTTP, and Layer 3 metrics from [Load Balancing](https://cloud.google.com/load-balancing/) in Google Cloud Platform. It contains all metrics exported from the [GCP Load Balancing Monitoring API](https://cloud.google.com/monitoring/api/metrics_gcp#gcp-loadbalancing). diff --git a/packages/gcp/manifest.yml b/packages/gcp/manifest.yml index 6fb83b4b8a6..d710fc6b152 100644 --- a/packages/gcp/manifest.yml +++ b/packages/gcp/manifest.yml @@ -1,6 +1,6 @@ name: gcp title: Google Cloud Platform -version: "2.5.0" +version: "2.6.0" release: ga description: Collect logs from Google Cloud Platform with Elastic Agent. type: integration @@ -151,10 +151,15 @@ policy_templates: description: Collect Load Balancing metrics from Google Cloud Platform (GCP) with Elastic Agent data_streams: - loadbalancing_metrics + - loadbalancing_logs inputs: - type: gcp/metrics title: Collect GCP Load Balancing Metrics description: Collect GCP Load Balancing Metrics input_group: metrics + - type: gcp-pubsub + title: "Collect Google Cloud Platform (GCP) load balancing logs (input: gcp-pubsub)" + description: "Collecting load balancing logs from Google Cloud Platform (GCP) (input: gcp-pubsub)" + input_group: logs owner: github: elastic/security-external-integrations