From 524f2ceebc7b55a84be4df3f365af2c65cfaadfa Mon Sep 17 00:00:00 2001 From: Ross Wolf <31489089+rw-access@users.noreply.github.com> Date: Tue, 20 Apr 2021 13:07:11 -0400 Subject: [PATCH] [Security][Fleet] Install the security_detection_engine package automatically (#97191) * Automatically install the security_detection_engine package via fleet * Update dockerImage to include the security_detection_engine package * Update api/fleet/setup install test * Update test data for Endpoint package * Fix JSON token * Update firis json entry in destination_index * Update destination_index structure * Update destination_index structure * Change KQL query to unblock testing * Restore KQL and fix JSON instead * update timestamps to pass tests --- x-pack/plugins/fleet/common/constants/epm.ts | 1 + .../fleet_api_integration/apis/fleet_setup.ts | 8 +- x-pack/test/fleet_api_integration/config.ts | 2 +- .../endpoint/metadata/api_feature/data.json | 36 +- .../metadata/destination_index/data.json | 343 +++++++++--------- .../apps/endpoint/endpoint_list.ts | 10 +- .../apis/metadata.ts | 4 +- .../apis/metadata_v1.ts | 4 +- 8 files changed, 200 insertions(+), 208 deletions(-) diff --git a/x-pack/plugins/fleet/common/constants/epm.ts b/x-pack/plugins/fleet/common/constants/epm.ts index faa1127cfe1da..7bf3c3e6205ec 100644 --- a/x-pack/plugins/fleet/common/constants/epm.ts +++ b/x-pack/plugins/fleet/common/constants/epm.ts @@ -15,6 +15,7 @@ export const requiredPackages = { System: 'system', Endpoint: 'endpoint', ElasticAgent: 'elastic_agent', + SecurityDetectionEngine: 'security_detection_engine', } as const; // these are currently identical. we can separate if they later diverge diff --git a/x-pack/test/fleet_api_integration/apis/fleet_setup.ts b/x-pack/test/fleet_api_integration/apis/fleet_setup.ts index 762a9f5302cef..5d0c40e63545a 100644 --- a/x-pack/test/fleet_api_integration/apis/fleet_setup.ts +++ b/x-pack/test/fleet_api_integration/apis/fleet_setup.ts @@ -75,7 +75,13 @@ export default function (providerContext: FtrProviderContext) { .map((p: any) => p.name) .sort(); - expect(installedPackages).to.eql(['elastic_agent', 'endpoint', 'fleet_server', 'system']); + expect(installedPackages).to.eql([ + 'elastic_agent', + 'endpoint', + 'fleet_server', + 'security_detection_engine', + 'system', + ]); }); }); } diff --git a/x-pack/test/fleet_api_integration/config.ts b/x-pack/test/fleet_api_integration/config.ts index 1257db7016501..2344bdc32904a 100644 --- a/x-pack/test/fleet_api_integration/config.ts +++ b/x-pack/test/fleet_api_integration/config.ts @@ -15,7 +15,7 @@ import { defineDockerServersConfig } from '@kbn/test'; // example: https://beats-ci.elastic.co/blue/organizations/jenkins/Ingest-manager%2Fpackage-storage/detail/snapshot/74/pipeline/257#step-302-log-1. // It should be updated any time there is a new Docker image published for the Snapshot Distribution of the Package Registry. export const dockerImage = - 'docker.elastic.co/package-registry/distribution:c5925eb82898dfc3e879a521871c7383513804c7'; + 'docker.elastic.co/package-registry/distribution:b6a53ac9300333a4a45f3f7d350c9aed72061a66'; export default async function ({ readConfigFile }: FtrConfigProviderContext) { const xPackAPITestsConfig = await readConfigFile(require.resolve('../api_integration/config.ts')); diff --git a/x-pack/test/functional/es_archives/endpoint/metadata/api_feature/data.json b/x-pack/test/functional/es_archives/endpoint/metadata/api_feature/data.json index 60679f9072c74..30b4e19dcb1d1 100644 --- a/x-pack/test/functional/es_archives/endpoint/metadata/api_feature/data.json +++ b/x-pack/test/functional/es_archives/endpoint/metadata/api_feature/data.json @@ -4,7 +4,7 @@ "id": "3KVN2G8BYQH1gtPUuYk7", "index": "metrics-endpoint.metadata-default", "source": { - "@timestamp": 1579881969541, + "@timestamp": 1618841405309, "agent": { "id": "963b081e-60d1-482c-befd-a5815fa8290f", "version": "6.6.1", @@ -26,7 +26,7 @@ } }, "event": { - "created": 1579881969541, + "created": 1618841405309, "id": "32f5fda2-48e4-4fae-b89e-a18038294d14", "kind": "metric", "category": [ @@ -74,7 +74,7 @@ "id": "3aVN2G8BYQH1gtPUuYk7", "index": "metrics-endpoint.metadata-default", "source": { - "@timestamp": 1579881969541, + "@timestamp": 1618841405309, "agent": { "id": "b3412d6f-b022-4448-8fee-21cc936ea86b", "version": "6.0.0", @@ -96,7 +96,7 @@ } }, "event": { - "created": 1579881969541, + "created": 1618841405309, "id": "32f5fda2-48e4-4fae-b89e-a18038294d15", "kind": "metric", "category": [ @@ -143,7 +143,7 @@ "id": "3qVN2G8BYQH1gtPUuYk7", "index": "metrics-endpoint.metadata-default", "source": { - "@timestamp": 1579881969541, + "@timestamp": 1618841405309, "agent": { "id": "3838df35-a095-4af4-8fce-0b6d78793f2e", "version": "6.8.0", @@ -165,7 +165,7 @@ } }, "event": { - "created": 1579881969541, + "created": 1618841405309, "id": "32f5fda2-48e4-4fae-b89e-a18038294d16", "kind": "metric", "category": [ @@ -210,7 +210,7 @@ "id": "36VN2G8BYQH1gtPUuYk7", "index": "metrics-endpoint.metadata-default", "source": { - "@timestamp": 1579878369541, + "@timestamp": 1618841405309, "agent": { "id": "963b081e-60d1-482c-befd-a5815fa8290f", "version": "6.6.1", @@ -232,7 +232,7 @@ } }, "event": { - "created": 1579878369541, + "created": 1618841405309, "id": "32f5fda2-48e4-4fae-b89e-a18038294d18", "kind": "metric", "category": [ @@ -280,7 +280,7 @@ "id": "4KVN2G8BYQH1gtPUuYk7", "index": "metrics-endpoint.metadata-default", "source": { - "@timestamp": 1579878369541, + "@timestamp": 1618841405309, "agent": { "id": "b3412d6f-b022-4448-8fee-21cc936ea86b", "version": "6.0.0", @@ -302,7 +302,7 @@ } }, "event": { - "created": 1579878369541, + "created": 1618841405309, "id": "32f5fda2-48e4-4fae-b89e-a18038294d19", "kind": "metric", "category": [ @@ -348,7 +348,7 @@ "id": "4aVN2G8BYQH1gtPUuYk7", "index": "metrics-endpoint.metadata-default", "source": { - "@timestamp": 1579878369541, + "@timestamp": 1618841405309, "agent": { "id": "3838df35-a095-4af4-8fce-0b6d78793f2e", "version": "6.8.0", @@ -370,7 +370,7 @@ } }, "event": { - "created": 1579878369541, + "created": 1618841405309, "id": "32f5fda2-48e4-4fae-b89e-a18038294d39", "kind": "metric", "category": [ @@ -416,7 +416,7 @@ "id": "4qVN2G8BYQH1gtPUuYk7", "index": "metrics-endpoint.metadata-default", "source": { - "@timestamp": 1579874769541, + "@timestamp": 1618841405309, "agent": { "id": "963b081e-60d1-482c-befd-a5815fa8290f", "version": "6.6.1", @@ -438,7 +438,7 @@ } }, "event": { - "created": 1579874769541, + "created": 1618841405309, "id": "32f5fda2-48e4-4fae-b89e-a18038294d31", "kind": "metric", "category": [ @@ -485,7 +485,7 @@ "id": "46VN2G8BYQH1gtPUuYk7", "index": "metrics-endpoint.metadata-default", "source": { - "@timestamp": 1579874769541, + "@timestamp": 1618841405309, "agent": { "id": "b3412d6f-b022-4448-8fee-21cc936ea86b", "version": "6.0.0", @@ -507,7 +507,7 @@ } }, "event": { - "created": 1579874769541, + "created": 1618841405309, "id": "32f5fda2-48e4-4fae-b89e-a18038294d23", "kind": "metric", "category": [ @@ -553,7 +553,7 @@ "id": "5KVN2G8BYQH1gtPUuYk7", "index": "metrics-endpoint.metadata-default", "source": { - "@timestamp": 1579874769541, + "@timestamp": 1618841405309, "agent": { "id": "3838df35-a095-4af4-8fce-0b6d78793f2e", "version": "6.8.0", @@ -575,7 +575,7 @@ } }, "event": { - "created": 1579874769541, + "created": 1618841405309, "id": "32f5fda2-48e4-4fae-b89e-a18038294d35", "kind": "metric", "category": [ diff --git a/x-pack/test/functional/es_archives/endpoint/metadata/destination_index/data.json b/x-pack/test/functional/es_archives/endpoint/metadata/destination_index/data.json index ef840d454a763..b70a9d5df0eb8 100644 --- a/x-pack/test/functional/es_archives/endpoint/metadata/destination_index/data.json +++ b/x-pack/test/functional/es_archives/endpoint/metadata/destination_index/data.json @@ -4,68 +4,63 @@ "id": "M92ScEJT9M9QusfIi3hpEb0AAAAAAAAA", "index": "metrics-endpoint.metadata_current_default", "source": { - "HostDetails": { - "@timestamp": 1579881969541, - "Endpoint": { - "policy": { - "applied": { - "id": "00000000-0000-0000-0000-000000000000", - "name": "Default", - "status": "failure" - } - }, - "status": "enrolled" - }, - "agent": { - "id": "3838df35-a095-4af4-8fce-0b6d78793f2e", - "name": "Elastic Endpoint", - "version": "6.8.0" - }, - "elastic": { - "agent": { - "id": "023fa40c-411d-4188-a941-4147bfadd095" + "@timestamp": 1618841405309, + "Endpoint": { + "policy": { + "applied": { + "id": "00000000-0000-0000-0000-000000000000", + "name": "Default", + "status": "failure" } }, - "event": { - "action": "endpoint_metadata", - "category": [ - "host" - ], - "created": 1579881969541, - "dataset": "endpoint.metadata", - "id": "32f5fda2-48e4-4fae-b89e-a18038294d16", - "ingested": "2020-09-09T18:25:15.853783Z", - "kind": "metric", - "module": "endpoint", - "type": [ - "info" - ] - }, - "host": { - "hostname": "rezzani-7.example.com", - "id": "fc0ff548-feba-41b6-8367-65e8790d0eaf", - "ip": [ - "10.101.149.26", - "2606:a000:ffc0:39:11ef:37b9:3371:578c" - ], - "mac": [ - "e2-6d-f9-0-46-2e" - ], - "name": "rezzani-7.example.com", - "os": { - "Ext": { - "variant": "Windows Pro" - }, - "family": "Windows", - "full": "Windows 10", - "name": "windows 10.0", - "platform": "Windows", - "version": "10.0" - } - } + "status": "enrolled" }, "agent": { - "id": "3838df35-a095-4af4-8fce-0b6d78793f2e" + "id": "3838df35-a095-4af4-8fce-0b6d78793f2e", + "name": "Elastic Endpoint", + "version": "6.8.0" + }, + "elastic": { + "agent": { + "id": "023fa40c-411d-4188-a941-4147bfadd095" + } + }, + "event": { + "action": "endpoint_metadata", + "category": [ + "host" + ], + "created": 1618841405309, + "dataset": "endpoint.metadata", + "id": "32f5fda2-48e4-4fae-b89e-a18038294d16", + "ingested": "2020-09-09T18:25:15.853783Z", + "kind": "metric", + "module": "endpoint", + "type": [ + "info" + ] + }, + "host": { + "hostname": "rezzani-7.example.com", + "id": "fc0ff548-feba-41b6-8367-65e8790d0eaf", + "ip": [ + "10.101.149.26", + "2606:a000:ffc0:39:11ef:37b9:3371:578c" + ], + "mac": [ + "e2-6d-f9-0-46-2e" + ], + "name": "rezzani-7.example.com", + "os": { + "Ext": { + "variant": "Windows Pro" + }, + "family": "Windows", + "full": "Windows 10", + "name": "windows 10.0", + "platform": "Windows", + "version": "10.0" + } } } } @@ -77,71 +72,66 @@ "id": "OU3RgCJaNnR90byeDEHutp8AAAAAAAAA", "index": "metrics-endpoint.metadata_current_default", "source": { - "HostDetails": { - "@timestamp": 1579881969541, - "Endpoint": { - "policy": { - "applied": { - "id": "C2A9093E-E289-4C0A-AA44-8C32A414FA7A", - "name": "Default", - "status": "failure" - } - }, - "status": "enrolled" - }, - "agent": { - "id": "963b081e-60d1-482c-befd-a5815fa8290f", - "name": "Elastic Endpoint", - "version": "6.6.1" - }, - "elastic": { - "agent": { - "id": "11488bae-880b-4e7b-8d28-aac2aa9de816" + "@timestamp": 1618841405309, + "Endpoint": { + "policy": { + "applied": { + "id": "C2A9093E-E289-4C0A-AA44-8C32A414FA7A", + "name": "Default", + "status": "failure" } }, - "event": { - "action": "endpoint_metadata", - "category": [ - "host" - ], - "created": 1579881969541, - "dataset": "endpoint.metadata", - "id": "32f5fda2-48e4-4fae-b89e-a18038294d14", - "ingested": "2020-09-09T18:25:14.919526Z", - "kind": "metric", - "module": "endpoint", - "type": [ - "info" - ] - }, - "host": { - "architecture": "x86", - "hostname": "cadmann-4.example.com", - "id": "1fb3e58f-6ab0-4406-9d2a-91911207a712", - "ip": [ - "10.192.213.130", - "10.70.28.129" - ], - "mac": [ - "a9-71-6a-cc-93-85", - "f7-31-84-d3-21-68", - "2-95-12-39-ca-71" - ], - "name": "cadmann-4.example.com", - "os": { - "Ext": { - "variant": "Windows Pro" - }, - "family": "Windows", - "full": "Windows 10", - "name": "windows 10.0", - "platform": "Windows", - "version": "10.0" - } - } + "status": "enrolled" }, "agent": { - "id": "963b081e-60d1-482c-befd-a5815fa8290f" + "id": "963b081e-60d1-482c-befd-a5815fa8290f", + "name": "Elastic Endpoint", + "version": "6.6.1" + }, + "elastic": { + "agent": { + "id": "11488bae-880b-4e7b-8d28-aac2aa9de816" + } + }, + "event": { + "action": "endpoint_metadata", + "category": [ + "host" + ], + "created": 1618841405309, + "dataset": "endpoint.metadata", + "id": "32f5fda2-48e4-4fae-b89e-a18038294d14", + "ingested": "2020-09-09T18:25:14.919526Z", + "kind": "metric", + "module": "endpoint", + "type": [ + "info" + ] + }, + "host": { + "architecture": "x86", + "hostname": "cadmann-4.example.com", + "id": "1fb3e58f-6ab0-4406-9d2a-91911207a712", + "ip": [ + "10.192.213.130", + "10.70.28.129" + ], + "mac": [ + "a9-71-6a-cc-93-85", + "f7-31-84-d3-21-68", + "2-95-12-39-ca-71" + ], + "name": "cadmann-4.example.com", + "os": { + "Ext": { + "variant": "Windows Pro" + }, + "family": "Windows", + "full": "Windows 10", + "name": "windows 10.0", + "platform": "Windows", + "version": "10.0" + } } } } @@ -153,70 +143,65 @@ "id": "YjqDCEuI6JmLeLOSyZx_NhMAAAAAAAAA", "index": "metrics-endpoint.metadata_current_default", "source": { - "HostDetails": { - "@timestamp": 1579881969541, - "Endpoint": { - "policy": { - "applied": { - "id": "C2A9093E-E289-4C0A-AA44-8C32A414FA7A", - "name": "Default", - "status": "success" - } - }, - "status": "enrolled" - }, - "agent": { - "id": "b3412d6f-b022-4448-8fee-21cc936ea86b", - "name": "Elastic Endpoint", - "version": "6.0.0" - }, - "elastic": { - "agent": { - "id": "92ac1ce0-e1f7-409e-8af6-f17e97b1fc71" + "@timestamp": 1618841405309, + "Endpoint": { + "policy": { + "applied": { + "id": "C2A9093E-E289-4C0A-AA44-8C32A414FA7A", + "name": "Default", + "status": "success" } }, - "event": { - "action": "endpoint_metadata", - "category": [ - "host" - ], - "created": 1579881969541, - "dataset": "endpoint.metadata", - "id": "32f5fda2-48e4-4fae-b89e-a18038294d15", - "ingested": "2020-09-09T18:25:15.853404Z", - "kind": "metric", - "module": "endpoint", - "type": [ - "info" - ] - }, - "host": { - "architecture": "x86_64", - "hostname": "thurlow-9.example.com", - "id": "2f735e3d-be14-483b-9822-bad06e9045ca", - "ip": [ - "10.46.229.234" - ], - "mac": [ - "30-8c-45-55-69-b8", - "e5-36-7e-8f-a3-84", - "39-a1-37-20-18-74" - ], - "name": "thurlow-9.example.com", - "os": { - "Ext": { - "variant": "Windows Server" - }, - "family": "Windows", - "full": "Windows Server 2016", - "name": "windows 10.0", - "platform": "Windows", - "version": "10.0" - } - } + "status": "enrolled" }, "agent": { - "id": "b3412d6f-b022-4448-8fee-21cc936ea86b" + "id": "b3412d6f-b022-4448-8fee-21cc936ea86b", + "name": "Elastic Endpoint", + "version": "6.0.0" + }, + "elastic": { + "agent": { + "id": "92ac1ce0-e1f7-409e-8af6-f17e97b1fc71" + } + }, + "event": { + "action": "endpoint_metadata", + "category": [ + "host" + ], + "created": 1618841405309, + "dataset": "endpoint.metadata", + "id": "32f5fda2-48e4-4fae-b89e-a18038294d15", + "ingested": "2020-09-09T18:25:15.853404Z", + "kind": "metric", + "module": "endpoint", + "type": [ + "info" + ] + }, + "host": { + "architecture": "x86_64", + "hostname": "thurlow-9.example.com", + "id": "2f735e3d-be14-483b-9822-bad06e9045ca", + "ip": [ + "10.46.229.234" + ], + "mac": [ + "30-8c-45-55-69-b8", + "e5-36-7e-8f-a3-84", + "39-a1-37-20-18-74" + ], + "name": "thurlow-9.example.com", + "os": { + "Ext": { + "variant": "Windows Server" + }, + "family": "Windows", + "full": "Windows Server 2016", + "name": "windows 10.0", + "platform": "Windows", + "version": "10.0" + } } } } diff --git a/x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts b/x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts index 9f9b24683dd1a..fec50bf52fa42 100644 --- a/x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts +++ b/x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts @@ -38,7 +38,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { 'windows 10.0', '10.101.149.26, 2606:a000:ffc0:39:11ef:37b9:3371:578c', '6.8.0', - 'Jan 24, 2020 @ 16:06:09.541', + 'Apr 19, 2021 @ 14:10:05.309', '', ], [ @@ -49,7 +49,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { 'windows 10.0', '10.192.213.130, 10.70.28.129', '6.6.1', - 'Jan 24, 2020 @ 16:06:09.541', + 'Apr 19, 2021 @ 14:10:05.309', '', ], [ @@ -60,7 +60,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { 'windows 10.0', '10.46.229.234', '6.0.0', - 'Jan 24, 2020 @ 16:06:09.541', + 'Apr 19, 2021 @ 14:10:05.309', '', ], ]; @@ -274,7 +274,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { 'windows 10.0', '10.192.213.130, 10.70.28.129', '6.6.1', - 'Jan 24, 2020 @ 16:06:09.541', + 'Apr 19, 2021 @ 14:10:05.309', '', ], [ @@ -285,7 +285,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { 'windows 10.0', '10.46.229.234', '6.0.0', - 'Jan 24, 2020 @ 16:06:09.541', + 'Apr 19, 2021 @ 14:10:05.309', '', ], ]; diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/metadata.ts b/x-pack/test/security_solution_endpoint_api_int/apis/metadata.ts index 07b046b0a95f7..8dd5adba43edb 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/metadata.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/metadata.ts @@ -225,7 +225,7 @@ export default function ({ getService }: FtrProviderContext) { (ip: string) => ip === targetEndpointIp ); expect(resultIp).to.eql([targetEndpointIp]); - expect(body.hosts[0].metadata.event.created).to.eql(1579881969541); + expect(body.hosts[0].metadata.event.created).to.eql(1618841405309); expect(body.hosts.length).to.eql(1); expect(body.request_page_size).to.eql(10); expect(body.request_page_index).to.eql(0); @@ -268,7 +268,7 @@ export default function ({ getService }: FtrProviderContext) { const resultElasticAgentId: string = body.hosts[0].metadata.elastic.agent.id; expect(resultHostId).to.eql(targetEndpointId); expect(resultElasticAgentId).to.eql(targetElasticAgentId); - expect(body.hosts[0].metadata.event.created).to.eql(1579881969541); + expect(body.hosts[0].metadata.event.created).to.eql(1618841405309); expect(body.hosts[0].host_status).to.eql('unhealthy'); expect(body.hosts.length).to.eql(1); expect(body.request_page_size).to.eql(10); diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/metadata_v1.ts b/x-pack/test/security_solution_endpoint_api_int/apis/metadata_v1.ts index 0e90b5c615c26..f3f86d4610d2b 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/metadata_v1.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/metadata_v1.ts @@ -214,7 +214,7 @@ export default function ({ getService }: FtrProviderContext) { (ip: string) => ip === targetEndpointIp ); expect(resultIp).to.eql([targetEndpointIp]); - expect(body.hosts[0].metadata.event.created).to.eql(1579881969541); + expect(body.hosts[0].metadata.event.created).to.eql(1618841405309); expect(body.hosts.length).to.eql(1); expect(body.request_page_size).to.eql(10); expect(body.request_page_index).to.eql(0); @@ -257,7 +257,7 @@ export default function ({ getService }: FtrProviderContext) { const resultElasticAgentId: string = body.hosts[0].metadata.elastic.agent.id; expect(resultHostId).to.eql(targetEndpointId); expect(resultElasticAgentId).to.eql(targetElasticAgentId); - expect(body.hosts[0].metadata.event.created).to.eql(1579881969541); + expect(body.hosts[0].metadata.event.created).to.eql(1618841405309); expect(body.hosts[0].host_status).to.eql('unhealthy'); expect(body.hosts.length).to.eql(1); expect(body.request_page_size).to.eql(10);