From ef704e1d3eea696bad836f6d865d1937fa575a38 Mon Sep 17 00:00:00 2001 From: Frank Hassanabad Date: Fri, 30 Apr 2021 12:36:06 -0600 Subject: [PATCH] [Security Solutions] (Phase 1) Adds an application cache called metrics entities and integrates it within Security Solutions behind a feature flag (#96446) ## Summary Phase 1 of a multi-phase cautious approach for adding an experimental application cache for Kibana solutions called `metric_entities` and integrates it within Security Solutions. Phase 1 is putting experimental support into the application without breaking existing features. Lots of TODO's, conversations and a possible RFC from phase 1 to phase 2 approach. Some features are missing, but for phase 1 the general idea and code is all there. To enable this first phase after checking out the branch add this to your `kibana.dev.yml` ```yml xpack.metricsEntities.enabled: true xpack.securitySolution.enableExperimental: ['metricsEntitiesEnabled'] ``` Then go into Stack Management -> Advanced Settings (Under Security Solutions) and set the enabled to true like so: Screen Shot 2021-04-08 at 2 21 02 PM Next go to the security_solutions page and you will see it being activated and you will have these transforms running if you look under stack management: Screen Shot 2021-04-29 at 2 00 27 PM On the hosts page, network, page, etc... You can see them being activated when you have no query/filter and you click on request: Screen Shot 2021-04-29 at 2 01 28 PM You will see in the request the index patterns all starting with `estc_xyz*` ### Checklist Delete any items that are not applicable to this PR. - [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md) We have lots of TODO's but no concrete docs with this just yet. - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials Behind a feature flag and this isn't there yet. - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .eslintrc.js | 112 ++++++ api_docs/metrics_entities.json | 151 ++++++++ api_docs/metrics_entities.mdx | 26 ++ docs/developer/plugin-list.asciidoc | 6 + .../src/get_server_watch_paths.test.ts | 1 + .../src/get_server_watch_paths.ts | 1 + tsconfig.json | 3 +- tsconfig.refs.json | 1 + x-pack/.i18nrc.json | 1 + x-pack/plugins/metrics_entities/README.md | 324 ++++++++++++++++++ .../metrics_entities/common/constants.ts | 21 ++ .../plugins/metrics_entities/common/index.ts | 11 + .../plugins/metrics_entities/jest.config.js | 12 + x-pack/plugins/metrics_entities/kibana.json | 10 + .../plugins/metrics_entities/server/config.ts | 14 + .../server/error_with_status_code.ts | 17 + .../plugins/metrics_entities/server/index.ts | 21 ++ .../metrics_entities/server/modules/README.md | 4 + .../modules/host_entities/host_entities.json | 38 ++ .../host_entities/host_entities_mapping.json | 45 +++ .../server/modules/host_entities/index.ts | 10 + .../modules/host_metrics/host_metrics.json | 21 ++ .../host_metrics/host_metrics_mapping.json | 83 +++++ .../server/modules/host_metrics/index.ts | 11 + .../metrics_entities/server/modules/index.ts | 71 ++++ ...destination_country_iso_code_entities.json | 51 +++ ...ion_country_iso_code_entities_mapping.json | 120 +++++++ .../destination_ip_entities.json | 46 +++ .../destination_ip_entities_mapping.json | 84 +++++ .../server/modules/network_entities/index.ts | 26 ++ .../source_country_iso_code_entities.json | 51 +++ ...rce_country_iso_code_entities_mapping.json | 120 +++++++ .../network_entities/source_ip_entities.json | 46 +++ .../source_ip_entities_mapping.json | 84 +++++ .../server/modules/network_metrics/index.ts | 11 + .../modules/network_metrics/ip_metrics.json | 116 +++++++ .../network_metrics/ip_metrics_mapping.json | 92 +++++ .../metrics_entities/server/modules/types.ts | 38 ++ .../server/modules/user_entities/index.ts | 10 + .../modules/user_entities/user_entities.json | 51 +++ .../user_entities/user_entities_mapping.json | 53 +++ .../server/modules/user_metrics/index.ts | 11 + .../modules/user_metrics/user_metrics.json | 56 +++ .../user_metrics/user_metrics_mapping.json | 46 +++ .../plugins/metrics_entities/server/plugin.ts | 89 +++++ .../server/routes/delete_transforms.ts | 50 +++ .../server/routes/get_transforms.ts | 36 ++ .../metrics_entities/server/routes/index.ts | 10 + .../server/routes/post_transforms.ts | 96 ++++++ .../utils/get_metrics_entities_client.ts | 21 ++ .../server/routes/utils/index.ts | 8 + .../server/scripts/check_env_variables.sh | 32 ++ .../server/scripts/delete_examples/all.json | 11 + .../delete_examples/all_prefix_auditbeat.json | 11 + .../network_entities_auditbeat.json | 3 + .../scripts/delete_examples/one_module.json | 3 + .../scripts/delete_examples/two_modules.json | 3 + .../server/scripts/delete_transforms.sh | 23 ++ .../server/scripts/get_transforms.sh | 16 + .../server/scripts/hard_reset.sh | 17 + .../server/scripts/post_examples/all.json | 32 ++ .../scripts/post_examples/all_auditbeat.json | 23 ++ .../network_entities_auditbeat.json | 4 + .../one_module_allindices_autostart.json | 24 ++ .../post_examples/one_module_auditbeat.json | 16 + .../post_examples/one_module_auto_start.json | 8 + .../one_module_prefix_auditbeat.json | 5 + .../post_examples/two_modules_all.json | 24 ++ .../post_examples/two_modules_auditbeat.json | 4 + .../server/scripts/post_transforms.sh | 24 ++ .../server/scripts/update_transforms.sh | 13 + .../server/services/delete_transforms.ts | 38 ++ .../server/services/get_transforms.ts | 24 ++ .../metrics_entities/server/services/index.ts | 15 + .../server/services/install_mappings.ts | 82 +++++ .../server/services/install_transforms.ts | 122 +++++++ .../services/metrics_entities_client.ts | 76 ++++ .../services/metrics_entities_client_types.ts | 41 +++ .../server/services/post_transforms.ts | 72 ++++ .../server/services/stop_transforms.ts | 8 + .../server/services/uninstall_mappings.ts | 55 +++ .../server/services/uninstall_transforms.ts | 92 +++++ .../services/utils/compute_mapping_index.ts | 24 ++ .../services/utils/compute_transform_id.ts | 33 ++ .../server/services/utils/get_index_exists.ts | 38 ++ .../server/services/utils/get_json.ts | 10 + .../services/utils/get_transform_exists.ts | 29 ++ .../server/services/utils/index.ts | 17 + .../services/utils/log_mapping_debug.ts | 20 ++ .../services/utils/log_mapping_error.ts | 27 ++ .../server/services/utils/log_mapping_info.ts | 20 ++ .../services/utils/log_transform_debug.ts | 20 ++ .../services/utils/log_transform_error.ts | 27 ++ .../services/utils/log_transform_info.ts | 20 ++ .../plugins/metrics_entities/server/types.ts | 36 ++ x-pack/plugins/metrics_entities/tsconfig.json | 27 ++ .../security_solution/common/constants.ts | 41 +++ .../common/experimental_features.ts | 1 + .../hosts/authentications/index.ts | 4 + .../security_solution/hosts/index.ts | 2 + .../hosts/kpi/authentications/index.ts | 2 + .../security_solution/hosts/kpi/index.ts | 3 + .../security_solution/index.ts | 4 +- .../matrix_histogram/index.ts | 2 + .../security_solution/network/index.ts | 2 + .../security_solution/network/kpi/index.ts | 4 + .../common/transforms/types.ts | 39 +++ .../containers/matrix_histogram/index.test.ts | 9 +- .../containers/matrix_histogram/index.ts | 59 +++- .../common/lib/kibana/kibana_react.mock.ts | 4 + .../public/common/mock/global_state.ts | 1 + .../components/user_info/index.test.tsx | 13 +- .../detections/components/user_info/index.tsx | 10 + .../containers/authentications/index.tsx | 32 +- .../public/hosts/containers/hosts/index.tsx | 21 +- .../kpi_hosts/authentications/index.tsx | 24 +- .../containers/kpi_hosts/hosts/index.tsx | 18 +- .../containers/kpi_hosts/unique_ips/index.tsx | 25 +- .../containers/kpi_network/dns/index.tsx | 19 +- .../kpi_network/network_events/index.tsx | 18 +- .../kpi_network/tls_handshakes/index.tsx | 19 +- .../kpi_network/unique_private_ips/index.tsx | 19 +- .../network_top_countries/index.tsx | 36 +- .../containers/network_top_n_flow/index.tsx | 35 +- .../public/transforms/containers/api.ts | 49 +++ .../transforms/containers/translations.ts | 15 + .../containers/use_create_transforms.ts | 112 ++++++ .../transforms/containers/use_transforms.ts | 111 ++++++ .../transforms/utils/adjust_timerange.ts | 30 ++ .../utils/create_indices_from_prefix.ts | 18 + .../transforms/utils/get_settings_match.ts | 28 ++ .../transforms/utils/get_transform_changes.ts | 48 +++ .../utils/get_transform_changes_for_hosts.ts | 39 +++ .../utils/get_transform_changes_for_kpi.ts | 44 +++ ..._transform_changes_for_matrix_histogram.ts | 43 +++ .../get_transform_changes_for_network.ts | 75 ++++ .../get_transform_changes_if_they_exist.ts | 55 +++ .../public/transforms/utils/index.ts | 17 + .../utils/is_filter_query_compatible.ts | 21 ++ .../public/transforms/utils/types.ts | 50 +++ .../security_solution/server/plugin.ts | 5 +- .../factory/hosts/all/index.ts | 41 +++ .../hosts/all/query.all_hosts_entities.dsl.ts | 92 +++++ .../authentications/dsl/query_entities.dsl.ts | 82 +++++ .../factory/hosts/authentications/helpers.ts | 67 ++++ .../factory/hosts/authentications/index.tsx | 52 ++- .../factory/hosts/index.test.ts | 12 +- .../security_solution/factory/hosts/index.ts | 15 +- .../hosts/kpi/authentications/helpers.ts | 10 + .../hosts/kpi/authentications/index.ts | 51 ++- ....hosts_kpi_authentications_entities.dsl.ts | 83 +++++ .../factory/hosts/kpi/hosts/index.ts | 25 ++ .../query.hosts_kpi_hosts_entities.dsl.ts | 65 ++++ .../factory/hosts/kpi/unique_ips/index.ts | 39 +++ ...query.hosts_kpi_unique_ips_entities.dsl.ts | 83 +++++ .../matrix_histogram/authentications/index.ts | 9 + ....authentications_histogram_entities.dsl.ts | 77 +++++ .../factory/matrix_histogram/helpers.ts | 16 + .../factory/matrix_histogram/index.ts | 46 ++- .../factory/network/index.test.ts | 15 +- .../factory/network/index.ts | 21 +- .../factory/network/kpi/dns/index.ts | 19 + .../dns/query.network_kip_dns_entities.dsl.ts | 52 +++ .../factory/network/kpi/index.ts | 15 +- .../network/kpi/network_events/index.ts | 21 ++ ...network_kpi_network_events_entities.dsl.ts | 52 +++ .../network/kpi/tls_handshakes/index.ts | 21 ++ ...network_kpi_tls_handshakes_entities.dsl.ts | 52 +++ .../network/kpi/unique_private_ips/index.ts | 45 +++ ...ork_kpi_unique_private_ips_entities.dsl.ts | 106 ++++++ .../factory/network/top_countries/index.ts | 39 +++ ...uery.top_countries_network_entities.dsl.ts | 162 +++++++++ .../factory/network/top_n_flow/index.ts | 36 ++ .../query.top_n_flow_network_entities.dsl.ts | 165 +++++++++ .../security_solution/server/ui_settings.ts | 29 +- x-pack/test/tsconfig.json | 1 + 176 files changed, 6528 insertions(+), 120 deletions(-) create mode 100644 api_docs/metrics_entities.json create mode 100644 api_docs/metrics_entities.mdx create mode 100755 x-pack/plugins/metrics_entities/README.md create mode 100644 x-pack/plugins/metrics_entities/common/constants.ts create mode 100644 x-pack/plugins/metrics_entities/common/index.ts create mode 100644 x-pack/plugins/metrics_entities/jest.config.js create mode 100644 x-pack/plugins/metrics_entities/kibana.json create mode 100644 x-pack/plugins/metrics_entities/server/config.ts create mode 100644 x-pack/plugins/metrics_entities/server/error_with_status_code.ts create mode 100644 x-pack/plugins/metrics_entities/server/index.ts create mode 100644 x-pack/plugins/metrics_entities/server/modules/README.md create mode 100644 x-pack/plugins/metrics_entities/server/modules/host_entities/host_entities.json create mode 100644 x-pack/plugins/metrics_entities/server/modules/host_entities/host_entities_mapping.json create mode 100644 x-pack/plugins/metrics_entities/server/modules/host_entities/index.ts create mode 100644 x-pack/plugins/metrics_entities/server/modules/host_metrics/host_metrics.json create mode 100644 x-pack/plugins/metrics_entities/server/modules/host_metrics/host_metrics_mapping.json create mode 100644 x-pack/plugins/metrics_entities/server/modules/host_metrics/index.ts create mode 100644 x-pack/plugins/metrics_entities/server/modules/index.ts create mode 100644 x-pack/plugins/metrics_entities/server/modules/network_entities/destination_country_iso_code_entities.json create mode 100644 x-pack/plugins/metrics_entities/server/modules/network_entities/destination_country_iso_code_entities_mapping.json create mode 100644 x-pack/plugins/metrics_entities/server/modules/network_entities/destination_ip_entities.json create mode 100644 x-pack/plugins/metrics_entities/server/modules/network_entities/destination_ip_entities_mapping.json create mode 100644 x-pack/plugins/metrics_entities/server/modules/network_entities/index.ts create mode 100644 x-pack/plugins/metrics_entities/server/modules/network_entities/source_country_iso_code_entities.json create mode 100644 x-pack/plugins/metrics_entities/server/modules/network_entities/source_country_iso_code_entities_mapping.json create mode 100644 x-pack/plugins/metrics_entities/server/modules/network_entities/source_ip_entities.json create mode 100644 x-pack/plugins/metrics_entities/server/modules/network_entities/source_ip_entities_mapping.json create mode 100644 x-pack/plugins/metrics_entities/server/modules/network_metrics/index.ts create mode 100644 x-pack/plugins/metrics_entities/server/modules/network_metrics/ip_metrics.json create mode 100644 x-pack/plugins/metrics_entities/server/modules/network_metrics/ip_metrics_mapping.json create mode 100644 x-pack/plugins/metrics_entities/server/modules/types.ts create mode 100644 x-pack/plugins/metrics_entities/server/modules/user_entities/index.ts create mode 100644 x-pack/plugins/metrics_entities/server/modules/user_entities/user_entities.json create mode 100644 x-pack/plugins/metrics_entities/server/modules/user_entities/user_entities_mapping.json create mode 100644 x-pack/plugins/metrics_entities/server/modules/user_metrics/index.ts create mode 100644 x-pack/plugins/metrics_entities/server/modules/user_metrics/user_metrics.json create mode 100644 x-pack/plugins/metrics_entities/server/modules/user_metrics/user_metrics_mapping.json create mode 100644 x-pack/plugins/metrics_entities/server/plugin.ts create mode 100644 x-pack/plugins/metrics_entities/server/routes/delete_transforms.ts create mode 100644 x-pack/plugins/metrics_entities/server/routes/get_transforms.ts create mode 100644 x-pack/plugins/metrics_entities/server/routes/index.ts create mode 100644 x-pack/plugins/metrics_entities/server/routes/post_transforms.ts create mode 100644 x-pack/plugins/metrics_entities/server/routes/utils/get_metrics_entities_client.ts create mode 100644 x-pack/plugins/metrics_entities/server/routes/utils/index.ts create mode 100755 x-pack/plugins/metrics_entities/server/scripts/check_env_variables.sh create mode 100644 x-pack/plugins/metrics_entities/server/scripts/delete_examples/all.json create mode 100644 x-pack/plugins/metrics_entities/server/scripts/delete_examples/all_prefix_auditbeat.json create mode 100644 x-pack/plugins/metrics_entities/server/scripts/delete_examples/network_entities_auditbeat.json create mode 100644 x-pack/plugins/metrics_entities/server/scripts/delete_examples/one_module.json create mode 100644 x-pack/plugins/metrics_entities/server/scripts/delete_examples/two_modules.json create mode 100755 x-pack/plugins/metrics_entities/server/scripts/delete_transforms.sh create mode 100755 x-pack/plugins/metrics_entities/server/scripts/get_transforms.sh create mode 100755 x-pack/plugins/metrics_entities/server/scripts/hard_reset.sh create mode 100644 x-pack/plugins/metrics_entities/server/scripts/post_examples/all.json create mode 100644 x-pack/plugins/metrics_entities/server/scripts/post_examples/all_auditbeat.json create mode 100644 x-pack/plugins/metrics_entities/server/scripts/post_examples/network_entities_auditbeat.json create mode 100644 x-pack/plugins/metrics_entities/server/scripts/post_examples/one_module_allindices_autostart.json create mode 100644 x-pack/plugins/metrics_entities/server/scripts/post_examples/one_module_auditbeat.json create mode 100644 x-pack/plugins/metrics_entities/server/scripts/post_examples/one_module_auto_start.json create mode 100644 x-pack/plugins/metrics_entities/server/scripts/post_examples/one_module_prefix_auditbeat.json create mode 100644 x-pack/plugins/metrics_entities/server/scripts/post_examples/two_modules_all.json create mode 100644 x-pack/plugins/metrics_entities/server/scripts/post_examples/two_modules_auditbeat.json create mode 100755 x-pack/plugins/metrics_entities/server/scripts/post_transforms.sh create mode 100755 x-pack/plugins/metrics_entities/server/scripts/update_transforms.sh create mode 100644 x-pack/plugins/metrics_entities/server/services/delete_transforms.ts create mode 100644 x-pack/plugins/metrics_entities/server/services/get_transforms.ts create mode 100644 x-pack/plugins/metrics_entities/server/services/index.ts create mode 100644 x-pack/plugins/metrics_entities/server/services/install_mappings.ts create mode 100644 x-pack/plugins/metrics_entities/server/services/install_transforms.ts create mode 100644 x-pack/plugins/metrics_entities/server/services/metrics_entities_client.ts create mode 100644 x-pack/plugins/metrics_entities/server/services/metrics_entities_client_types.ts create mode 100644 x-pack/plugins/metrics_entities/server/services/post_transforms.ts create mode 100644 x-pack/plugins/metrics_entities/server/services/stop_transforms.ts create mode 100644 x-pack/plugins/metrics_entities/server/services/uninstall_mappings.ts create mode 100644 x-pack/plugins/metrics_entities/server/services/uninstall_transforms.ts create mode 100644 x-pack/plugins/metrics_entities/server/services/utils/compute_mapping_index.ts create mode 100644 x-pack/plugins/metrics_entities/server/services/utils/compute_transform_id.ts create mode 100644 x-pack/plugins/metrics_entities/server/services/utils/get_index_exists.ts create mode 100644 x-pack/plugins/metrics_entities/server/services/utils/get_json.ts create mode 100644 x-pack/plugins/metrics_entities/server/services/utils/get_transform_exists.ts create mode 100644 x-pack/plugins/metrics_entities/server/services/utils/index.ts create mode 100644 x-pack/plugins/metrics_entities/server/services/utils/log_mapping_debug.ts create mode 100644 x-pack/plugins/metrics_entities/server/services/utils/log_mapping_error.ts create mode 100644 x-pack/plugins/metrics_entities/server/services/utils/log_mapping_info.ts create mode 100644 x-pack/plugins/metrics_entities/server/services/utils/log_transform_debug.ts create mode 100644 x-pack/plugins/metrics_entities/server/services/utils/log_transform_error.ts create mode 100644 x-pack/plugins/metrics_entities/server/services/utils/log_transform_info.ts create mode 100644 x-pack/plugins/metrics_entities/server/types.ts create mode 100644 x-pack/plugins/metrics_entities/tsconfig.json create mode 100644 x-pack/plugins/security_solution/common/transforms/types.ts create mode 100644 x-pack/plugins/security_solution/public/transforms/containers/api.ts create mode 100644 x-pack/plugins/security_solution/public/transforms/containers/translations.ts create mode 100644 x-pack/plugins/security_solution/public/transforms/containers/use_create_transforms.ts create mode 100644 x-pack/plugins/security_solution/public/transforms/containers/use_transforms.ts create mode 100644 x-pack/plugins/security_solution/public/transforms/utils/adjust_timerange.ts create mode 100644 x-pack/plugins/security_solution/public/transforms/utils/create_indices_from_prefix.ts create mode 100644 x-pack/plugins/security_solution/public/transforms/utils/get_settings_match.ts create mode 100644 x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes.ts create mode 100644 x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_hosts.ts create mode 100644 x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_kpi.ts create mode 100644 x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_matrix_histogram.ts create mode 100644 x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_network.ts create mode 100644 x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_if_they_exist.ts create mode 100644 x-pack/plugins/security_solution/public/transforms/utils/index.ts create mode 100644 x-pack/plugins/security_solution/public/transforms/utils/is_filter_query_compatible.ts create mode 100644 x-pack/plugins/security_solution/public/transforms/utils/types.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/query.all_hosts_entities.dsl.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/dsl/query_entities.dsl.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/authentications/query.hosts_kpi_authentications_entities.dsl.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/hosts/query.hosts_kpi_hosts_entities.dsl.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/unique_ips/query.hosts_kpi_unique_ips_entities.dsl.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/query.authentications_histogram_entities.dsl.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/dns/query.network_kip_dns_entities.dsl.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/network_events/query.network_kpi_network_events_entities.dsl.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/tls_handshakes/query.network_kpi_tls_handshakes_entities.dsl.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/unique_private_ips/query.network_kpi_unique_private_ips_entities.dsl.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/query.top_countries_network_entities.dsl.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/query.top_n_flow_network_entities.dsl.ts diff --git a/.eslintrc.js b/.eslintrc.js index 851ae299e3a38..35acbb423f62d 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -1118,6 +1118,118 @@ module.exports = { 'prefer-destructuring': 'error', }, }, + /** + * Metrics entities overrides + */ + { + // front end and common typescript and javascript files only + files: [ + 'x-pack/plugins/metrics_entities/public/**/*.{js,mjs,ts,tsx}', + 'x-pack/plugins/metrics_entities/common/**/*.{js,mjs,ts,tsx}', + ], + rules: { + 'import/no-nodejs-modules': 'error', + 'no-restricted-imports': [ + 'error', + { + // prevents UI code from importing server side code and then webpack including it when doing builds + patterns: ['**/server/*'], + }, + ], + }, + }, + { + // typescript and javascript for front and back end + files: ['x-pack/plugins/metrics_entities/**/*.{js,mjs,ts,tsx}'], + plugins: ['eslint-plugin-node'], + env: { + jest: true, + }, + rules: { + 'accessor-pairs': 'error', + 'array-callback-return': 'error', + 'no-array-constructor': 'error', + complexity: 'error', + 'consistent-return': 'error', + 'func-style': ['error', 'expression'], + 'import/order': [ + 'error', + { + groups: ['builtin', 'external', 'internal', 'parent', 'sibling', 'index'], + 'newlines-between': 'always', + }, + ], + 'sort-imports': [ + 'error', + { + ignoreDeclarationSort: true, + }, + ], + 'node/no-deprecated-api': 'error', + 'no-bitwise': 'error', + 'no-continue': 'error', + 'no-dupe-keys': 'error', + 'no-duplicate-case': 'error', + 'no-duplicate-imports': 'error', + 'no-empty-character-class': 'error', + 'no-empty-pattern': 'error', + 'no-ex-assign': 'error', + 'no-extend-native': 'error', + 'no-extra-bind': 'error', + 'no-extra-boolean-cast': 'error', + 'no-extra-label': 'error', + 'no-func-assign': 'error', + 'no-implicit-globals': 'error', + 'no-implied-eval': 'error', + 'no-invalid-regexp': 'error', + 'no-inner-declarations': 'error', + 'no-lone-blocks': 'error', + 'no-multi-assign': 'error', + 'no-misleading-character-class': 'error', + 'no-new-symbol': 'error', + 'no-obj-calls': 'error', + 'no-param-reassign': ['error', { props: true }], + 'no-process-exit': 'error', + 'no-prototype-builtins': 'error', + 'no-return-await': 'error', + 'no-self-compare': 'error', + 'no-shadow-restricted-names': 'error', + 'no-sparse-arrays': 'error', + 'no-this-before-super': 'error', + // rely on typescript + 'no-undef': 'off', + 'no-unreachable': 'error', + 'no-unsafe-finally': 'error', + 'no-useless-call': 'error', + 'no-useless-catch': 'error', + 'no-useless-concat': 'error', + 'no-useless-computed-key': 'error', + 'no-useless-escape': 'error', + 'no-useless-rename': 'error', + 'no-useless-return': 'error', + 'no-void': 'error', + 'one-var-declaration-per-line': 'error', + 'prefer-object-spread': 'error', + 'prefer-promise-reject-errors': 'error', + 'prefer-rest-params': 'error', + 'prefer-spread': 'error', + 'prefer-template': 'error', + 'require-atomic-updates': 'error', + 'symbol-description': 'error', + 'vars-on-top': 'error', + '@typescript-eslint/explicit-member-accessibility': 'error', + '@typescript-eslint/no-this-alias': 'error', + '@typescript-eslint/no-explicit-any': 'error', + '@typescript-eslint/no-useless-constructor': 'error', + '@typescript-eslint/unified-signatures': 'error', + '@typescript-eslint/explicit-function-return-type': 'error', + '@typescript-eslint/no-non-null-assertion': 'error', + '@typescript-eslint/no-unused-vars': 'error', + 'no-template-curly-in-string': 'error', + 'sort-keys': 'error', + 'prefer-destructuring': 'error', + }, + }, /** * Alerting Services overrides */ diff --git a/api_docs/metrics_entities.json b/api_docs/metrics_entities.json new file mode 100644 index 0000000000000..3b05ca066b0e2 --- /dev/null +++ b/api_docs/metrics_entities.json @@ -0,0 +1,151 @@ +{ + "id": "metricsEntities", + "client": { + "classes": [], + "functions": [], + "interfaces": [], + "enums": [], + "misc": [], + "objects": [] + }, + "server": { + "classes": [], + "functions": [], + "interfaces": [], + "enums": [], + "misc": [], + "objects": [], + "setup": { + "id": "def-server.MetricsEntitiesPluginSetup", + "type": "Interface", + "label": "MetricsEntitiesPluginSetup", + "description": [], + "tags": [], + "children": [ + { + "tags": [], + "id": "def-server.MetricsEntitiesPluginSetup.getMetricsEntitiesClient", + "type": "Function", + "label": "getMetricsEntitiesClient", + "description": [], + "source": { + "path": "x-pack/plugins/metrics_entities/server/types.ts", + "lineNumber": 15 + }, + "signature": [ + "GetMetricsEntitiesClientType" + ] + } + ], + "source": { + "path": "x-pack/plugins/metrics_entities/server/types.ts", + "lineNumber": 14 + }, + "lifecycle": "setup", + "initialIsOpen": true + }, + "start": { + "id": "def-server.MetricsEntitiesPluginStart", + "type": "Type", + "label": "MetricsEntitiesPluginStart", + "tags": [], + "description": [], + "source": { + "path": "x-pack/plugins/metrics_entities/server/types.ts", + "lineNumber": 18 + }, + "signature": [ + "void" + ], + "lifecycle": "start", + "initialIsOpen": true + } + }, + "common": { + "classes": [], + "functions": [], + "interfaces": [], + "enums": [], + "misc": [ + { + "tags": [], + "id": "def-common.ELASTIC_NAME", + "type": "string", + "label": "ELASTIC_NAME", + "description": [ + "\nGlobal prefix for all the transform jobs" + ], + "source": { + "path": "x-pack/plugins/metrics_entities/common/constants.ts", + "lineNumber": 21 + }, + "signature": [ + "\"estc\"" + ], + "initialIsOpen": false + }, + { + "tags": [], + "id": "def-common.METRICS_ENTITIES_TRANSFORMS", + "type": "string", + "label": "METRICS_ENTITIES_TRANSFORMS", + "description": [ + "\nTransforms route" + ], + "source": { + "path": "x-pack/plugins/metrics_entities/common/constants.ts", + "lineNumber": 16 + }, + "initialIsOpen": false + }, + { + "tags": [], + "id": "def-common.METRICS_ENTITIES_URL", + "type": "string", + "label": "METRICS_ENTITIES_URL", + "description": [ + "\nBase route" + ], + "source": { + "path": "x-pack/plugins/metrics_entities/common/constants.ts", + "lineNumber": 11 + }, + "signature": [ + "\"/api/metrics_entities\"" + ], + "initialIsOpen": false + }, + { + "tags": [], + "id": "def-common.PLUGIN_ID", + "type": "string", + "label": "PLUGIN_ID", + "description": [], + "source": { + "path": "x-pack/plugins/metrics_entities/common/index.ts", + "lineNumber": 8 + }, + "signature": [ + "\"metricsEntities\"" + ], + "initialIsOpen": false + }, + { + "tags": [], + "id": "def-common.PLUGIN_NAME", + "type": "string", + "label": "PLUGIN_NAME", + "description": [], + "source": { + "path": "x-pack/plugins/metrics_entities/common/index.ts", + "lineNumber": 9 + }, + "signature": [ + "\"metrics_entities\"" + ], + "initialIsOpen": false + } + ], + "objects": [] + } +} \ No newline at end of file diff --git a/api_docs/metrics_entities.mdx b/api_docs/metrics_entities.mdx new file mode 100644 index 0000000000000..19a27636511c3 --- /dev/null +++ b/api_docs/metrics_entities.mdx @@ -0,0 +1,26 @@ +--- +id: kibMetricsEntitiesPluginApi +slug: /kibana-dev-docs/metricsEntitiesPluginApi +title: metricsEntities +image: https://source.unsplash.com/400x175/?github +summary: API docs for the metricsEntities plugin +date: 2020-11-16 +tags: ['contributor', 'dev', 'apidocs', 'kibana', 'metricsEntities'] +warning: This document is auto-generated and is meant to be viewed inside our experimental, new docs system. Reach out in #docs-engineering for more info. +--- + +import metricsEntitiesObj from './metrics_entities.json'; + +## Server + +### Setup + + +### Start + + +## Common + +### Consts, variables and types + + diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc index 65c6698c896ed..be840997a2177 100644 --- a/docs/developer/plugin-list.asciidoc +++ b/docs/developer/plugin-list.asciidoc @@ -474,6 +474,12 @@ using the CURL scripts in the scripts folder. |Visualize geo data from Elasticsearch or 3rd party geo-services. +|{kib-repo}blob/{branch}/x-pack/plugins/metrics_entities/README.md[metricsEntities] +|This is the metrics and entities plugin where you add can add transforms for your project +and group those transforms into modules. You can also re-use existing transforms in your +modules as well. + + |{kib-repo}blob/{branch}/x-pack/plugins/ml/readme.md[ml] |This plugin provides access to the machine learning features provided by Elastic. diff --git a/packages/kbn-cli-dev-mode/src/get_server_watch_paths.test.ts b/packages/kbn-cli-dev-mode/src/get_server_watch_paths.test.ts index 2fd53dd83a1bd..f4017df600a48 100644 --- a/packages/kbn-cli-dev-mode/src/get_server_watch_paths.test.ts +++ b/packages/kbn-cli-dev-mode/src/get_server_watch_paths.test.ts @@ -75,6 +75,7 @@ it('produces the right watch and ignore list', () => { /x-pack/plugins/lists/server/scripts, /x-pack/plugins/security_solution/scripts, /x-pack/plugins/security_solution/server/lib/detection_engine/scripts, + /x-pack/plugins/metrics_entities/server/scripts, ] `); }); diff --git a/packages/kbn-cli-dev-mode/src/get_server_watch_paths.ts b/packages/kbn-cli-dev-mode/src/get_server_watch_paths.ts index 4a9dae5c6fee2..b0773fd567635 100644 --- a/packages/kbn-cli-dev-mode/src/get_server_watch_paths.ts +++ b/packages/kbn-cli-dev-mode/src/get_server_watch_paths.ts @@ -66,6 +66,7 @@ export function getServerWatchPaths({ pluginPaths, pluginScanDirs }: Options) { fromRoot('x-pack/plugins/lists/server/scripts'), fromRoot('x-pack/plugins/security_solution/scripts'), fromRoot('x-pack/plugins/security_solution/server/lib/detection_engine/scripts'), + fromRoot('x-pack/plugins/metrics_entities/server/scripts'), ]; return { diff --git a/tsconfig.json b/tsconfig.json index 87ee067002109..b7122a70cb471 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -17,7 +17,7 @@ "x-pack/typings/**/*", "x-pack/tasks/**/*", "x-pack/plugins/lists/**/*", - "x-pack/plugins/security_solution/**/*", + "x-pack/plugins/security_solution/**/*" ], "exclude": [ "x-pack/plugins/security_solution/cypress/**/*" @@ -110,6 +110,7 @@ { "path": "./x-pack/plugins/licensing/tsconfig.json" }, { "path": "./x-pack/plugins/logstash/tsconfig.json" }, { "path": "./x-pack/plugins/maps/tsconfig.json" }, + { "path": "./x-pack/plugins/metrics_entities/tsconfig.json" }, { "path": "./x-pack/plugins/ml/tsconfig.json" }, { "path": "./x-pack/plugins/monitoring/tsconfig.json" }, { "path": "./x-pack/plugins/observability/tsconfig.json" }, diff --git a/tsconfig.refs.json b/tsconfig.refs.json index 34d4672f54470..2aa098e21f283 100644 --- a/tsconfig.refs.json +++ b/tsconfig.refs.json @@ -88,6 +88,7 @@ { "path": "./x-pack/plugins/licensing/tsconfig.json" }, { "path": "./x-pack/plugins/logstash/tsconfig.json" }, { "path": "./x-pack/plugins/maps/tsconfig.json" }, + { "path": "./x-pack/plugins/metrics_entities/tsconfig.json" }, { "path": "./x-pack/plugins/ml/tsconfig.json" }, { "path": "./x-pack/plugins/monitoring/tsconfig.json" }, { "path": "./x-pack/plugins/observability/tsconfig.json" }, diff --git a/x-pack/.i18nrc.json b/x-pack/.i18nrc.json index ab1a6d3b14ec3..ab30c67fcc96b 100644 --- a/x-pack/.i18nrc.json +++ b/x-pack/.i18nrc.json @@ -39,6 +39,7 @@ "xpack.logstash": ["plugins/logstash"], "xpack.main": ["plugins/xpack_legacy"], "xpack.maps": ["plugins/maps"], + "xpack.metricsEntities": "plugins/metrics_entities", "xpack.ml": ["plugins/ml"], "xpack.monitoring": ["plugins/monitoring"], "xpack.osquery": ["plugins/osquery"], diff --git a/x-pack/plugins/metrics_entities/README.md b/x-pack/plugins/metrics_entities/README.md new file mode 100755 index 0000000000000..6c711ce4fed82 --- /dev/null +++ b/x-pack/plugins/metrics_entities/README.md @@ -0,0 +1,324 @@ +# metrics_entities + +This is the metrics and entities plugin where you add can add transforms for your project +and group those transforms into modules. You can also re-use existing transforms in your +modules as well. + +## Turn on experimental flags +During at least phase 1 of this development, please add these to your `kibana.dev.yml` file to turn on the feature: + +```ts +xpack.metricsEntities.enabled: true +xpack.securitySolution.enableExperimental: ['metricsEntitiesEnabled'] +``` + +## Quick start on using scripts to call the API + +The scripts rely on CURL and jq: + +- [CURL](https://curl.haxx.se) +- [jq](https://stedolan.github.io/jq/) + +Install curl and jq + +```sh +brew update +brew install curl +brew install jq +``` + +Open `$HOME/.zshrc` or `${HOME}.bashrc` depending on your SHELL output from `echo $SHELL` +and add these environment variables: + +```sh +export ELASTICSEARCH_USERNAME=${user} +export ELASTICSEARCH_PASSWORD=${password} +export ELASTICSEARCH_URL=https://${ip}:9200 +export KIBANA_URL=http://localhost:5601 +``` + +source `$HOME/.zshrc` or `${HOME}.bashrc` to ensure variables are set: + +```sh +source ~/.zshrc +``` + +Restart Kibana and ensure that you are using `--no-base-path` as changing the base path is a feature but will +get in the way of the CURL scripts written as is. + +Go to the scripts folder `cd kibana/x-pack/plugins/metrics_entities/server/scripts` and can run some of the scripts +such as: + +```sh +./post_transforms.sh ./post_examples/all.json +``` + +which will post transforms from the `all.json` + +You can also delete them by running: + +```sh +./delete_transforms.sh ./delete_examples/all.json +``` + +See the folder for other curl scripts that exercise parts of the REST API and feel free to add your own examples +in the folder as well. + +## Quick start on how to add a transform + +You will want to figure out how you want your transform from within Kibana roughly using +the UI and then copy the JSON. The JSON you will want to change and paste within a folder +which represents a module. + +For example, for the `host_entities` and a `host_entities_mapping` we created a folder called host_entities +here: + +```sh +sever/modules/host_entities +``` + +Then we add two files, a subset of the transform JSON and a mapping like so: + +```sh +server/modules/host_entities/host_entities_mapping.json <--- this is the mappings +server/modules/host_entities/host_entities.json <--- This is a subset of the transform JSON +index.ts <--- Import/export your json here +``` + +The mappings can be normal mapping like so with `host_entities_mapping.json`: +```json +{ + "mappings": { + "_meta": { + "index": "host_ent" + }, + "dynamic": "strict", + "properties": { + "@timestamp": { + "type": "date" + }, + "metrics": { + "properties": { + "host": { + "properties": { + "name": { + "properties": { + "value_count": { + "type": "long" + } + } + } + } + } + } + }, + "host": { + "properties": { + "name": { + "type": "keyword" + }, + "os": { + "properties": { + "name": { + "type": "keyword" + }, + "version": { + "type": "keyword" + } + } + } + } + } + } + } +} +``` + +One caveat is that you need to add this to the meta section to tell it what the name will be: +```json + "_meta": { + "index": "host_ent" + }, +``` + +Keep the name short as there is only 65 characters for a transform job and we prepend extra information to the mapping such as: +* prefix +* name of estc + +Although not required, a `"dynamic": "strict"` is strongly encouraged to prevent mapping guesses from elastic and it will be better for us +to spot errors quicker in the mappings such as type-o's if this is set to strict. + +Next, for the transform, you should add a subset that doesn't have any additional settings or meta associated like so for `host_entities.json`: + +```json +{ + "id": "host_ent", + "description": "[host.name entities] grouped by @timestamp, host.name, os.name, and os.version, and aggregated on host.name", + "pivot": { + "group_by": { + "@timestamp": { + "date_histogram": { + "field": "@timestamp", + "calendar_interval": "1h" + } + }, + "host.name": { + "terms": { + "field": "host.name" + } + }, + "host.os.name": { + "terms": { + "field": "host.os.name", + "missing_bucket": true + } + }, + "host.os.version": { + "terms": { + "field": "host.os.version", + "missing_bucket": true + } + } + }, + "aggregations": { + "metrics.host.name.value_count": { + "value_count": { + "field": "host.name" + } + } + } + } +} +``` + +Look in the `server/modules` for other examples, but it should be that clear cut. The final part is to wire everything up in the code by touching a few files +to either add this to an existing module or create your own module. In `server/module/host_entities` we add an `index.ts` like so that does an import/export +of the JSON: + +```sh +import hostEntities from './host_entities.json'; +import hostEntitiesMapping from './host_entities_mapping.json'; +export { hostEntities, hostEntitiesMapping }; +``` + +Then in `modules/index.ts` we add a new module name if we are creating a new module to the `export enum ModuleNames {` like so: + +```ts +// Import your host entities you just made +import { hostEntities, hostEntitiesMapping } from './host_entities'; + +/** + * These module names will map 1 to 1 to the REST interface. + */ +export enum ModuleNames { + hostSummaryMetrics = 'host_metrics', + hostSummaryEntities = 'host_entities', // <-- Add the entities/transform and give it a enum name and a module name + networkSummaryEntities = 'network_entities', + networkSummaryMetrics = 'network_metrics', + userSummaryEntities = 'user_entities', + userSummaryMetrics = 'user_metrics', +} +``` + +If you're not creating a new module but rather you are adding to an existing module, you can skip the above step. Next, you +just need to add your installable transform and installable mapping to the two data structures of `installableTransforms` and +`installableMappings` like so: + +```ts +/** + * Add any new folders as modules with their names below and grouped with + * key values. + */ +export const installableTransforms: Record = { + [ModuleNames.hostSummaryMetrics]: [hostMetrics], + [ModuleNames.hostSummaryEntities]: [hostEntities], // <-- Adds my new module name and transform to a new array. + [ModuleNames.networkSummaryEntities]: [ + destinationIpEntities, // <-- If instead I am adding to an existing module, I just add it to the array like these show + sourceIpEntities, + destinationCountryIsoCodeEntities, + sourceCountryIsoCodeEntities, + ], + [ModuleNames.networkSummaryMetrics]: [ipMetrics], + [ModuleNames.userSummaryEntities]: [userEntities], + [ModuleNames.userSummaryMetrics]: [userMetrics], +}; + +/** + * For all the mapping types, add each with their names below and grouped with + * key values. + */ +export const installableMappings: Record = { + [ModuleNames.hostSummaryMetrics]: [hostMetricsMapping], + [ModuleNames.hostSummaryEntities]: [hostEntitiesMapping], // <-- Adds my new module name and mapping to a new array. + [ModuleNames.networkSummaryEntities]: [ // <-- If instead I am adding to an existing module, I just add it to the array like these show + sourceIpEntitiesMapping, + destinationIpEntitiesMapping, + destinationCountryIsoCodeEntitiesMapping, + sourceCountryIsoCodeEntitiesMapping, + ], + [ModuleNames.networkSummaryMetrics]: [ipMetricsMapping], + [ModuleNames.userSummaryEntities]: [userEntitiesMapping], + [ModuleNames.userSummaryMetrics]: [userMetricsMapping], +}; +``` + +And after that, you should check out if there are any existing e2e tests or unit tests to update here to ensure that your mapping and transform will +pass ci. Create a pull request and your mapping and transform are completed. + +To call into the code to activate your module and create your transforms and mappings would be the following where you substitute your +${KIBANA_URL} with your kibana URL and the ${SPACE_URL} with any space id you have. If you're using the default space then you would use +an empty string: +```json +POST ${KIBANA_URL}${SPACE_URL}/api/metrics_entities/transforms +{ + "prefix": "all", + "modules": [ + "host_entities", + ], + "indices": [ + "auditbeat-*", + "endgame-*", + "filebeat-*", + "logs-*", + "packetbeat-*", + "winlogbeat-*", + "-*elastic-cloud-logs-*" + ], + "auto_start": true, + "settings": { + "max_page_search_size": 5000 + }, + "query": { + "range": { + "@timestamp": { + "gte": "now-1d/d", + "format": "strict_date_optional_time" + } + } + } +} +``` + +Very similar to the regular transform REST API, with the caveats that you define which modules you want to install, the prefix name you want to use, and +if you want to `auto_start` it or not. The rest such as `settings`, `query` will be the same as the transforms API. They will also push those same setting into +each of your transforms within your module(s) as the same setting for each individual ones. + +## TODO List +During the phase 1, phase 2, phase N, this TODO will exist as a reminder and notes for what still needs to be developed. These are not in a priority order, but +are notes during the phased approach. As we approach production and the feature flags are removed these TODO's should be removed in favor of Kibana issues or regular +left over TODO's in the code base. + +- Add these properties to the route which are: + - disable_transforms/exclude flag to exclude 1 or more transforms within a module, + - pipeline flag, + - Change the REST routes on post to change the indexes for whichever indexes you want + - Unit tests to ensure the data of the mapping.json includes the correct fields such as + _meta, at least one alias, a mapping section, etc... + - Add text/keyword and other things to the mappings (not just keyword maybe?) ... At least review the mappings one more time + - Add a sort of @timestamp to the output destination indexes? + - Add the REST Kibana security based tags if needed and push those to any plugins using this plugin. Something like: tags: ['access:metricsEntities-read'] and ['access:metricsEntities-all'], + - Add schema validation choosing some schema library (io-ts or Kibana Schema or ... ) + - Add unit tests + - Add e2e tests + - Move ui code into this plugin from security_solutions? (maybe?) + - UI code could be within `kibana/packages` instead of in here directly and I think we will be better off. diff --git a/x-pack/plugins/metrics_entities/common/constants.ts b/x-pack/plugins/metrics_entities/common/constants.ts new file mode 100644 index 0000000000000..8efa0327f5f41 --- /dev/null +++ b/x-pack/plugins/metrics_entities/common/constants.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Base route + */ +export const METRICS_ENTITIES_URL = '/api/metrics_entities'; + +/** + * Transforms route + */ +export const METRICS_ENTITIES_TRANSFORMS = `${METRICS_ENTITIES_URL}/transforms`; + +/** + * Global prefix for all the transform jobs + */ +export const ELASTIC_NAME = 'estc'; diff --git a/x-pack/plugins/metrics_entities/common/index.ts b/x-pack/plugins/metrics_entities/common/index.ts new file mode 100644 index 0000000000000..a532dc151bf46 --- /dev/null +++ b/x-pack/plugins/metrics_entities/common/index.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const PLUGIN_ID = 'metricsEntities'; +export const PLUGIN_NAME = 'metrics_entities'; + +export * from './constants'; diff --git a/x-pack/plugins/metrics_entities/jest.config.js b/x-pack/plugins/metrics_entities/jest.config.js new file mode 100644 index 0000000000000..402532aa44c41 --- /dev/null +++ b/x-pack/plugins/metrics_entities/jest.config.js @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../..', + roots: ['/x-pack/plugins/metrics_entities'], +}; diff --git a/x-pack/plugins/metrics_entities/kibana.json b/x-pack/plugins/metrics_entities/kibana.json new file mode 100644 index 0000000000000..17484c2c243ce --- /dev/null +++ b/x-pack/plugins/metrics_entities/kibana.json @@ -0,0 +1,10 @@ +{ + "id": "metricsEntities", + "version": "8.0.0", + "kibanaVersion": "kibana", + "configPath": ["xpack", "metricsEntities"], + "server": true, + "ui": false, + "requiredPlugins": ["data", "dataEnhanced"], + "optionalPlugins": [] +} diff --git a/x-pack/plugins/metrics_entities/server/config.ts b/x-pack/plugins/metrics_entities/server/config.ts new file mode 100644 index 0000000000000..31be256611803 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/config.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { TypeOf, schema } from '@kbn/config-schema'; + +export const ConfigSchema = schema.object({ + enabled: schema.boolean({ defaultValue: false }), +}); + +export type ConfigType = TypeOf; diff --git a/x-pack/plugins/metrics_entities/server/error_with_status_code.ts b/x-pack/plugins/metrics_entities/server/error_with_status_code.ts new file mode 100644 index 0000000000000..15f7797fa424f --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/error_with_status_code.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class ErrorWithStatusCode extends Error { + private readonly statusCode: number; + + constructor(message: string, statusCode: number) { + super(message); + this.statusCode = statusCode; + } + + public getStatusCode = (): number => this.statusCode; +} diff --git a/x-pack/plugins/metrics_entities/server/index.ts b/x-pack/plugins/metrics_entities/server/index.ts new file mode 100644 index 0000000000000..b4d35eb90f486 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { PluginInitializerContext } from '../../../../src/core/server'; + +import { ConfigSchema } from './config'; +import { MetricsEntitiesPlugin } from './plugin'; + +// This exports static code and TypeScript types, +// as well as, Kibana Platform `plugin()` initializer. + +export const config = { schema: ConfigSchema }; +export const plugin = (initializerContext: PluginInitializerContext): MetricsEntitiesPlugin => { + return new MetricsEntitiesPlugin(initializerContext); +}; + +export { MetricsEntitiesPluginSetup, MetricsEntitiesPluginStart } from './types'; diff --git a/x-pack/plugins/metrics_entities/server/modules/README.md b/x-pack/plugins/metrics_entities/server/modules/README.md new file mode 100644 index 0000000000000..d4e28a2f83ed0 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/modules/README.md @@ -0,0 +1,4 @@ +# Modules + +This is where all the module types exist so you can load different bundled modules +with a REST endpoint. \ No newline at end of file diff --git a/x-pack/plugins/metrics_entities/server/modules/host_entities/host_entities.json b/x-pack/plugins/metrics_entities/server/modules/host_entities/host_entities.json new file mode 100644 index 0000000000000..ef6bcfc452860 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/modules/host_entities/host_entities.json @@ -0,0 +1,38 @@ +{ + "id": "host_ent", + "description": "[host.name entities] grouped by @timestamp, host.name, os.name, and os.version, and aggregated on host.name", + "pivot": { + "group_by": { + "@timestamp": { + "date_histogram": { + "field": "@timestamp", + "calendar_interval": "1h" + } + }, + "host.name": { + "terms": { + "field": "host.name" + } + }, + "host.os.name": { + "terms": { + "field": "host.os.name", + "missing_bucket": true + } + }, + "host.os.version": { + "terms": { + "field": "host.os.version", + "missing_bucket": true + } + } + }, + "aggregations": { + "metrics.host.name.value_count": { + "value_count": { + "field": "host.name" + } + } + } + } +} diff --git a/x-pack/plugins/metrics_entities/server/modules/host_entities/host_entities_mapping.json b/x-pack/plugins/metrics_entities/server/modules/host_entities/host_entities_mapping.json new file mode 100644 index 0000000000000..1f1e93dabfb5f --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/modules/host_entities/host_entities_mapping.json @@ -0,0 +1,45 @@ +{ + "mappings": { + "_meta": { + "index": "host_ent" + }, + "dynamic": "strict", + "properties": { + "@timestamp": { + "type": "date" + }, + "metrics": { + "properties": { + "host": { + "properties": { + "name": { + "properties": { + "value_count": { + "type": "long" + } + } + } + } + } + } + }, + "host": { + "properties": { + "name": { + "type": "keyword" + }, + "os": { + "properties": { + "name": { + "type": "keyword" + }, + "version": { + "type": "keyword" + } + } + } + } + } + } + } +} diff --git a/x-pack/plugins/metrics_entities/server/modules/host_entities/index.ts b/x-pack/plugins/metrics_entities/server/modules/host_entities/index.ts new file mode 100644 index 0000000000000..c3f34cd0f535c --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/modules/host_entities/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import hostEntities from './host_entities.json'; +import hostEntitiesMapping from './host_entities_mapping.json'; +export { hostEntities, hostEntitiesMapping }; diff --git a/x-pack/plugins/metrics_entities/server/modules/host_metrics/host_metrics.json b/x-pack/plugins/metrics_entities/server/modules/host_metrics/host_metrics.json new file mode 100644 index 0000000000000..8388721f98926 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/modules/host_metrics/host_metrics.json @@ -0,0 +1,21 @@ +{ + "id": "host_met", + "description": "[host.name metrics] grouped by @timestamp and aggregated on host.name", + "pivot": { + "group_by": { + "@timestamp": { + "date_histogram": { + "field": "@timestamp", + "calendar_interval": "1h" + } + } + }, + "aggregations": { + "metrics.host.name.cardinality": { + "cardinality": { + "field": "host.name" + } + } + } + } +} diff --git a/x-pack/plugins/metrics_entities/server/modules/host_metrics/host_metrics_mapping.json b/x-pack/plugins/metrics_entities/server/modules/host_metrics/host_metrics_mapping.json new file mode 100644 index 0000000000000..7975fe3c6ed0a --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/modules/host_metrics/host_metrics_mapping.json @@ -0,0 +1,83 @@ +{ + "mappings": { + "_meta": { + "index": "host_met" + }, + "properties": { + "metrics": { + "properties": { + "source": { + "properties": { + "ip": { + "properties": { + "value_count": { + "type": "long" + }, + "cardinality": { + "type": "long" + } + } + }, + "bytes": { + "properties": { + "sum": { + "type": "long" + } + } + } + } + }, + "destination": { + "properties": { + "ip": { + "properties": { + "value_count": { + "type": "long" + }, + "cardinality": { + "type": "long" + } + } + }, + "bytes": { + "properties": { + "sum": { + "type": "long" + } + } + } + } + }, + "network": { + "properties": { + "community_id": { + "properties": { + "cardinality": { + "type": "long" + } + } + } + } + }, + "host": { + "properties": { + "name": { + "properties": { + "value_count": { + "type": "long" + }, + "cardinality": { + "type": "long" + } + } + } + } + } + } + }, + "@timestamp": { + "type": "date" + } + } + } +} diff --git a/x-pack/plugins/metrics_entities/server/modules/host_metrics/index.ts b/x-pack/plugins/metrics_entities/server/modules/host_metrics/index.ts new file mode 100644 index 0000000000000..e11c5321ede85 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/modules/host_metrics/index.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import hostMetrics from './host_metrics.json'; +import hostMetricsMapping from './host_metrics_mapping.json'; + +export { hostMetrics, hostMetricsMapping }; diff --git a/x-pack/plugins/metrics_entities/server/modules/index.ts b/x-pack/plugins/metrics_entities/server/modules/index.ts new file mode 100644 index 0000000000000..61aca783a6c03 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/modules/index.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { hostMetrics, hostMetricsMapping } from './host_metrics'; +import { userMetrics, userMetricsMapping } from './user_metrics'; +import { ipMetrics, ipMetricsMapping } from './network_metrics'; +import { hostEntities, hostEntitiesMapping } from './host_entities'; +import { + destinationCountryIsoCodeEntities, + destinationCountryIsoCodeEntitiesMapping, + destinationIpEntities, + destinationIpEntitiesMapping, + sourceCountryIsoCodeEntities, + sourceCountryIsoCodeEntitiesMapping, + sourceIpEntities, + sourceIpEntitiesMapping, +} from './network_entities'; +import { Mappings, Transforms } from './types'; +import { userEntities, userEntitiesMapping } from './user_entities'; + +/** + * These module names will map 1 to 1 to the REST interface. + */ +export enum ModuleNames { + hostSummaryMetrics = 'host_metrics', + hostSummaryEntities = 'host_entities', + networkSummaryEntities = 'network_entities', + networkSummaryMetrics = 'network_metrics', + userSummaryEntities = 'user_entities', + userSummaryMetrics = 'user_metrics', +} + +/** + * Add any new folders as modules with their names below and grouped with + * key values. + */ +export const installableTransforms: Record = { + [ModuleNames.hostSummaryMetrics]: [hostMetrics], + [ModuleNames.hostSummaryEntities]: [hostEntities], + [ModuleNames.networkSummaryEntities]: [ + destinationIpEntities, + sourceIpEntities, + destinationCountryIsoCodeEntities, + sourceCountryIsoCodeEntities, + ], + [ModuleNames.networkSummaryMetrics]: [ipMetrics], + [ModuleNames.userSummaryEntities]: [userEntities], + [ModuleNames.userSummaryMetrics]: [userMetrics], +}; + +/** + * For all the mapping types, add each with their names below and grouped with + * key values. + */ +export const installableMappings: Record = { + [ModuleNames.hostSummaryMetrics]: [hostMetricsMapping], + [ModuleNames.hostSummaryEntities]: [hostEntitiesMapping], + [ModuleNames.networkSummaryEntities]: [ + sourceIpEntitiesMapping, + destinationIpEntitiesMapping, + destinationCountryIsoCodeEntitiesMapping, + sourceCountryIsoCodeEntitiesMapping, + ], + [ModuleNames.networkSummaryMetrics]: [ipMetricsMapping], + [ModuleNames.userSummaryEntities]: [userEntitiesMapping], + [ModuleNames.userSummaryMetrics]: [userMetricsMapping], +}; diff --git a/x-pack/plugins/metrics_entities/server/modules/network_entities/destination_country_iso_code_entities.json b/x-pack/plugins/metrics_entities/server/modules/network_entities/destination_country_iso_code_entities.json new file mode 100644 index 0000000000000..1f39c6c9552bd --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/modules/network_entities/destination_country_iso_code_entities.json @@ -0,0 +1,51 @@ +{ + "id": "dest_iso_ent", + "description": "[destination.geo.country_iso_code entities] grouped by @timestamp and aggregated on source.bytes, destination.bytes, network.community_id, destination.ip, and source.ip", + "pivot": { + "group_by": { + "@timestamp": { + "date_histogram": { + "field": "@timestamp", + "calendar_interval": "1h" + } + }, + "destination.geo.country_iso_code": { + "terms": { + "field": "destination.geo.country_iso_code" + } + } + }, + "aggregations": { + "metrics.destination.geo.country_iso_code.value_count": { + "value_count": { + "field": "destination.geo.country_iso_code" + } + }, + "metrics.source.bytes.sum": { + "sum": { + "field": "source.bytes" + } + }, + "metrics.destination.bytes.sum": { + "sum": { + "field": "destination.bytes" + } + }, + "metrics.network.community_id.cardinality": { + "cardinality": { + "field": "network.community_id" + } + }, + "metrics.source.ip.cardinality": { + "cardinality": { + "field": "source.ip" + } + }, + "metrics.destination.ip.cardinality": { + "cardinality": { + "field": "destination.ip" + } + } + } + } +} diff --git a/x-pack/plugins/metrics_entities/server/modules/network_entities/destination_country_iso_code_entities_mapping.json b/x-pack/plugins/metrics_entities/server/modules/network_entities/destination_country_iso_code_entities_mapping.json new file mode 100644 index 0000000000000..e56ed7157afdc --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/modules/network_entities/destination_country_iso_code_entities_mapping.json @@ -0,0 +1,120 @@ +{ + "mappings": { + "_meta": { + "index": "dest_iso_ent" + }, + "dynamic": "strict", + "properties": { + "@timestamp": { + "type": "date" + }, + "metrics": { + "properties": { + "source": { + "properties": { + "ip": { + "properties": { + "value_count": { + "type": "long" + }, + "cardinality": { + "type": "long" + } + } + }, + "bytes": { + "properties": { + "sum": { + "type": "long" + } + } + }, + "geo": { + "properties": { + "country_iso_code": { + "properties": { + "value_count": { + "type": "long" + } + } + } + } + } + } + }, + "destination": { + "properties": { + "ip": { + "properties": { + "value_count": { + "type": "long" + }, + "cardinality": { + "type": "long" + } + } + }, + "bytes": { + "properties": { + "sum": { + "type": "long" + } + } + }, + "geo": { + "properties": { + "country_iso_code": { + "properties": { + "value_count": { + "type": "long" + } + } + } + } + } + } + }, + "network": { + "properties": { + "community_id": { + "properties": { + "cardinality": { + "type": "long" + } + } + } + } + } + } + }, + "source": { + "properties": { + "ip": { + "type": "ip" + }, + "geo": { + "properties": { + "country_iso_code": { + "type": "keyword" + } + } + } + } + }, + "destination": { + "properties": { + "ip": { + "type": "ip" + }, + "geo": { + "properties": { + "country_iso_code": { + "type": "keyword" + } + } + } + } + } + } + } +} diff --git a/x-pack/plugins/metrics_entities/server/modules/network_entities/destination_ip_entities.json b/x-pack/plugins/metrics_entities/server/modules/network_entities/destination_ip_entities.json new file mode 100644 index 0000000000000..7ecced9a11ebc --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/modules/network_entities/destination_ip_entities.json @@ -0,0 +1,46 @@ +{ + "id": "dest_ip_ent", + "description": "[destination.ip entities] grouped by @timestamp and aggregated on destination.ip, source.bytes, destination.bytes, network.community_id, and source.ip", + "pivot": { + "group_by": { + "@timestamp": { + "date_histogram": { + "field": "@timestamp", + "calendar_interval": "1h" + } + }, + "destination.ip": { + "terms": { + "field": "destination.ip" + } + } + }, + "aggregations": { + "metrics.destination.ip.value_count": { + "value_count": { + "field": "destination.ip" + } + }, + "metrics.source.bytes.sum": { + "sum": { + "field": "source.bytes" + } + }, + "metrics.destination.bytes.sum": { + "sum": { + "field": "destination.bytes" + } + }, + "metrics.network.community_id.cardinality": { + "cardinality": { + "field": "network.community_id" + } + }, + "metrics.source.ip.cardinality": { + "cardinality": { + "field": "source.ip" + } + } + } + } +} diff --git a/x-pack/plugins/metrics_entities/server/modules/network_entities/destination_ip_entities_mapping.json b/x-pack/plugins/metrics_entities/server/modules/network_entities/destination_ip_entities_mapping.json new file mode 100644 index 0000000000000..ef7e1050c9c5d --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/modules/network_entities/destination_ip_entities_mapping.json @@ -0,0 +1,84 @@ +{ + "mappings": { + "_meta": { + "index": "dest_ip_ent" + }, + "dynamic": "strict", + "properties": { + "@timestamp": { + "type": "date" + }, + "metrics": { + "properties": { + "source": { + "properties": { + "ip": { + "properties": { + "value_count": { + "type": "long" + }, + "cardinality": { + "type": "long" + } + } + }, + "bytes": { + "properties": { + "sum": { + "type": "long" + } + } + } + } + }, + "destination": { + "properties": { + "ip": { + "properties": { + "value_count": { + "type": "long" + }, + "cardinality": { + "type": "long" + } + } + }, + "bytes": { + "properties": { + "sum": { + "type": "long" + } + } + } + } + }, + "network": { + "properties": { + "community_id": { + "properties": { + "cardinality": { + "type": "long" + } + } + } + } + } + } + }, + "source": { + "properties": { + "ip": { + "type": "ip" + } + } + }, + "destination": { + "properties": { + "ip": { + "type": "ip" + } + } + } + } + } +} diff --git a/x-pack/plugins/metrics_entities/server/modules/network_entities/index.ts b/x-pack/plugins/metrics_entities/server/modules/network_entities/index.ts new file mode 100644 index 0000000000000..b54425763effb --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/modules/network_entities/index.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import sourceIpEntities from './source_ip_entities.json'; +import destinationIpEntities from './destination_ip_entities.json'; +import sourceIpEntitiesMapping from './source_ip_entities_mapping.json'; +import destinationIpEntitiesMapping from './destination_ip_entities_mapping.json'; +import destinationCountryIsoCodeEntities from './destination_country_iso_code_entities.json'; +import destinationCountryIsoCodeEntitiesMapping from './destination_country_iso_code_entities_mapping.json'; +import sourceCountryIsoCodeEntities from './source_country_iso_code_entities.json'; +import sourceCountryIsoCodeEntitiesMapping from './source_country_iso_code_entities_mapping.json'; + +export { + sourceIpEntities, + destinationIpEntities, + sourceCountryIsoCodeEntities, + sourceCountryIsoCodeEntitiesMapping, + destinationCountryIsoCodeEntities, + destinationCountryIsoCodeEntitiesMapping, + sourceIpEntitiesMapping, + destinationIpEntitiesMapping, +}; diff --git a/x-pack/plugins/metrics_entities/server/modules/network_entities/source_country_iso_code_entities.json b/x-pack/plugins/metrics_entities/server/modules/network_entities/source_country_iso_code_entities.json new file mode 100644 index 0000000000000..60021b975b21d --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/modules/network_entities/source_country_iso_code_entities.json @@ -0,0 +1,51 @@ +{ + "id": "src_iso_ent", + "description": "[source.geo.country_iso_code entities] grouped by @timestamp and aggregated on source.geo.country_iso_code, source.bytes, destination.bytes, network.community_id, source.ip, and destination.ip", + "pivot": { + "group_by": { + "@timestamp": { + "date_histogram": { + "field": "@timestamp", + "calendar_interval": "1h" + } + }, + "source.geo.country_iso_code": { + "terms": { + "field": "source.geo.country_iso_code" + } + } + }, + "aggregations": { + "metrics.source.geo.country_iso_code.value_count": { + "value_count": { + "field": "source.geo.country_iso_code" + } + }, + "metrics.source.bytes.sum": { + "sum": { + "field": "source.bytes" + } + }, + "metrics.destination.bytes.sum": { + "sum": { + "field": "destination.bytes" + } + }, + "metrics.network.community_id.cardinality": { + "cardinality": { + "field": "network.community_id" + } + }, + "metrics.source.ip.cardinality": { + "cardinality": { + "field": "source.ip" + } + }, + "metrics.destination.ip.cardinality": { + "cardinality": { + "field": "destination.ip" + } + } + } + } +} diff --git a/x-pack/plugins/metrics_entities/server/modules/network_entities/source_country_iso_code_entities_mapping.json b/x-pack/plugins/metrics_entities/server/modules/network_entities/source_country_iso_code_entities_mapping.json new file mode 100644 index 0000000000000..0a44016be6a2c --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/modules/network_entities/source_country_iso_code_entities_mapping.json @@ -0,0 +1,120 @@ +{ + "mappings": { + "_meta": { + "index": "src_iso_ent" + }, + "dynamic": "strict", + "properties": { + "@timestamp": { + "type": "date" + }, + "metrics": { + "properties": { + "source": { + "properties": { + "ip": { + "properties": { + "value_count": { + "type": "long" + }, + "cardinality": { + "type": "long" + } + } + }, + "bytes": { + "properties": { + "sum": { + "type": "long" + } + } + }, + "geo": { + "properties": { + "country_iso_code": { + "properties": { + "value_count": { + "type": "long" + } + } + } + } + } + } + }, + "destination": { + "properties": { + "ip": { + "properties": { + "value_count": { + "type": "long" + }, + "cardinality": { + "type": "long" + } + } + }, + "bytes": { + "properties": { + "sum": { + "type": "long" + } + } + }, + "geo": { + "properties": { + "country_iso_code": { + "properties": { + "value_count": { + "type": "long" + } + } + } + } + } + } + }, + "network": { + "properties": { + "community_id": { + "properties": { + "cardinality": { + "type": "long" + } + } + } + } + } + } + }, + "source": { + "properties": { + "ip": { + "type": "ip" + }, + "geo": { + "properties": { + "country_iso_code": { + "type": "keyword" + } + } + } + } + }, + "destination": { + "properties": { + "ip": { + "type": "ip" + }, + "geo": { + "properties": { + "country_iso_code": { + "type": "keyword" + } + } + } + } + } + } + } +} diff --git a/x-pack/plugins/metrics_entities/server/modules/network_entities/source_ip_entities.json b/x-pack/plugins/metrics_entities/server/modules/network_entities/source_ip_entities.json new file mode 100644 index 0000000000000..3de6669c7bedb --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/modules/network_entities/source_ip_entities.json @@ -0,0 +1,46 @@ +{ + "id": "src_ip_ent", + "description": "[source.ip entities] grouped by @timestamp and aggregated on destination.ip, source.bytes, destination.bytes, network.community_id, and destination.ip", + "pivot": { + "group_by": { + "@timestamp": { + "date_histogram": { + "field": "@timestamp", + "calendar_interval": "1h" + } + }, + "source.ip": { + "terms": { + "field": "source.ip" + } + } + }, + "aggregations": { + "metrics.source.ip.value_count": { + "value_count": { + "field": "source.ip" + } + }, + "metrics.source.bytes.sum": { + "sum": { + "field": "source.bytes" + } + }, + "metrics.destination.bytes.sum": { + "sum": { + "field": "destination.bytes" + } + }, + "metrics.network.community_id.cardinality": { + "cardinality": { + "field": "network.community_id" + } + }, + "metrics.destination.ip.cardinality": { + "cardinality": { + "field": "destination.ip" + } + } + } + } +} diff --git a/x-pack/plugins/metrics_entities/server/modules/network_entities/source_ip_entities_mapping.json b/x-pack/plugins/metrics_entities/server/modules/network_entities/source_ip_entities_mapping.json new file mode 100644 index 0000000000000..64d9e48afcee9 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/modules/network_entities/source_ip_entities_mapping.json @@ -0,0 +1,84 @@ +{ + "mappings": { + "_meta": { + "index": "src_ip_ent" + }, + "dynamic": "strict", + "properties": { + "@timestamp": { + "type": "date" + }, + "metrics": { + "properties": { + "source": { + "properties": { + "ip": { + "properties": { + "value_count": { + "type": "long" + }, + "cardinality": { + "type": "long" + } + } + }, + "bytes": { + "properties": { + "sum": { + "type": "long" + } + } + } + } + }, + "destination": { + "properties": { + "ip": { + "properties": { + "value_count": { + "type": "long" + }, + "cardinality": { + "type": "long" + } + } + }, + "bytes": { + "properties": { + "sum": { + "type": "long" + } + } + } + } + }, + "network": { + "properties": { + "community_id": { + "properties": { + "cardinality": { + "type": "long" + } + } + } + } + } + } + }, + "source": { + "properties": { + "ip": { + "type": "ip" + } + } + }, + "destination": { + "properties": { + "ip": { + "type": "ip" + } + } + } + } + } +} diff --git a/x-pack/plugins/metrics_entities/server/modules/network_metrics/index.ts b/x-pack/plugins/metrics_entities/server/modules/network_metrics/index.ts new file mode 100644 index 0000000000000..216b85234dda4 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/modules/network_metrics/index.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import ipMetrics from './ip_metrics.json'; +import ipMetricsMapping from './ip_metrics_mapping.json'; + +export { ipMetrics, ipMetricsMapping }; diff --git a/x-pack/plugins/metrics_entities/server/modules/network_metrics/ip_metrics.json b/x-pack/plugins/metrics_entities/server/modules/network_metrics/ip_metrics.json new file mode 100644 index 0000000000000..ed953be84f3da --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/modules/network_metrics/ip_metrics.json @@ -0,0 +1,116 @@ +{ + "id": "ip_met", + "description": "[source.ip metrics] grouped by @timestamp, source.ip, destination.ip and aggregated on tls.version, suricata.eve.tls.version, zeek.ssl.version, dns.question.name, and zeek.dns.query", + "pivot": { + "group_by": { + "@timestamp": { + "date_histogram": { + "field": "@timestamp", + "calendar_interval": "1h" + } + } + }, + "aggregations": { + "metrics.source.ip.cardinality": { + "cardinality": { + "field": "source.ip" + } + }, + "metrics.destination.ip.cardinality": { + "cardinality": { + "field": "destination.ip" + } + }, + "metrics.network": { + "filter": { + "bool": { + "should": [ + { + "exists": { + "field": "source.ip" + } + }, + { + "exists": { + "field": "destination.ip" + } + } + ], + "minimum_should_match": 1 + } + }, + "aggs": { + "events.value_count": { + "value_count": { + "field": "@timestamp" + } + }, + "tls": { + "filter": { + "bool": { + "should": [ + { + "exists": { + "field": "tls.version" + } + }, + { + "exists": { + "field": "suricata.eve.tls.version" + } + }, + { + "exists": { + "field": "zeek.ssl.version" + } + } + ], + "minimum_should_match": 1 + } + }, + "aggs": { + "version.value_count": { + "value_count": { + "field": "@timestamp" + } + } + } + } + } + }, + "metrics.dns": { + "filter": { + "bool": { + "should": [ + { + "exists": { + "field": "dns.question.name" + } + }, + { + "term": { + "suricata.eve.dns.type": { + "value": "query" + } + } + }, + { + "exists": { + "field": "zeek.dns.query" + } + } + ], + "minimum_should_match": 1 + } + }, + "aggs": { + "queries.value_count": { + "value_count": { + "field": "@timestamp" + } + } + } + } + } + } +} diff --git a/x-pack/plugins/metrics_entities/server/modules/network_metrics/ip_metrics_mapping.json b/x-pack/plugins/metrics_entities/server/modules/network_metrics/ip_metrics_mapping.json new file mode 100644 index 0000000000000..a855b6091f29c --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/modules/network_metrics/ip_metrics_mapping.json @@ -0,0 +1,92 @@ +{ + "mappings": { + "_meta": { + "index": "ip_met" + }, + "dynamic": "strict", + "properties": { + "@timestamp": { + "type": "date" + }, + "metrics": { + "properties": { + "source": { + "properties": { + "ip": { + "properties": { + "value_count": { + "type": "long" + }, + "cardinality": { + "type": "long" + } + } + }, + "bytes": { + "properties": { + "sum": { + "type": "long" + } + } + } + } + }, + "destination": { + "properties": { + "ip": { + "properties": { + "value_count": { + "type": "long" + }, + "cardinality": { + "type": "long" + } + } + }, + "bytes": { + "properties": { + "sum": { + "type": "long" + } + } + } + } + }, + "network": { + "properties": { + "events": { + "properties": { + "value_count": { + "type": "long" + } + } + }, + "tls": { + "properties": { + "version": { + "properties": { + "value_count": { + "type": "long" + } + } + } + } + } + } + }, + "dns": { + "properties": { + "queries": { + "properties": { + "value_count": { + "type": "long" + } + } + } + } + } + } + } + } + } +} diff --git a/x-pack/plugins/metrics_entities/server/modules/types.ts b/x-pack/plugins/metrics_entities/server/modules/types.ts new file mode 100644 index 0000000000000..22b11ed89f5c4 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/modules/types.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Loose type for the mappings + */ +export interface Mappings { + [key: string]: unknown; + mappings: { + [key: string]: unknown; + _meta: { + index: string; + }; + }; +} + +/** + * Loose type for the transforms. id is marked optional so we can delete it before + * pushing it through elastic client. + * TODO: Can we use stricter pre-defined typings for the transforms here or is this ours because we define it slightly different? + */ +export interface Transforms { + [key: string]: unknown; + id: string; + dest?: Partial<{ + index: string; + pipeline: string; + }>; + source?: Partial<{}>; + settings?: Partial<{ + max_page_search_size: number; + docs_per_second: number | null; + }>; +} diff --git a/x-pack/plugins/metrics_entities/server/modules/user_entities/index.ts b/x-pack/plugins/metrics_entities/server/modules/user_entities/index.ts new file mode 100644 index 0000000000000..9cc17c8f180f0 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/modules/user_entities/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import userEntities from './user_entities.json'; +import userEntitiesMapping from './user_entities_mapping.json'; +export { userEntities, userEntitiesMapping }; diff --git a/x-pack/plugins/metrics_entities/server/modules/user_entities/user_entities.json b/x-pack/plugins/metrics_entities/server/modules/user_entities/user_entities.json new file mode 100644 index 0000000000000..aa41edcf40d41 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/modules/user_entities/user_entities.json @@ -0,0 +1,51 @@ +{ + "id": "user_ent", + "description": "[user.name entities] grouped by @timestamp and aggregated on user.name, and event.categories of success, failure, and unknown", + "pivot": { + "group_by": { + "@timestamp": { + "date_histogram": { + "field": "@timestamp", + "calendar_interval": "1h" + } + }, + "user.name": { + "terms": { + "field": "user.name" + } + } + }, + "aggregations": { + "metrics.event.authentication": { + "filter": { + "term": { + "event.category": "authentication" + } + }, + "aggs": { + "success.value_count": { + "filter": { + "term": { + "event.outcome": "success" + } + } + }, + "failure.value_count": { + "filter": { + "term": { + "event.outcome": "failure" + } + } + }, + "unknown.value_count": { + "filter": { + "term": { + "event.outcome": "unknown" + } + } + } + } + } + } + } +} diff --git a/x-pack/plugins/metrics_entities/server/modules/user_entities/user_entities_mapping.json b/x-pack/plugins/metrics_entities/server/modules/user_entities/user_entities_mapping.json new file mode 100644 index 0000000000000..2532afa3040c6 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/modules/user_entities/user_entities_mapping.json @@ -0,0 +1,53 @@ +{ + "mappings": { + "_meta": { + "index": "user_ent" + }, + "dynamic": "strict", + "properties": { + "@timestamp": { + "type": "date" + }, + "metrics": { + "properties": { + "event": { + "properties": { + "authentication": { + "properties": { + "failure": { + "properties": { + "value_count": { + "type": "long" + } + } + }, + "success": { + "properties": { + "value_count": { + "type": "long" + } + } + }, + "unknown": { + "properties": { + "value_count": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "user": { + "properties": { + "name": { + "type": "keyword" + } + } + } + } + } +} diff --git a/x-pack/plugins/metrics_entities/server/modules/user_metrics/index.ts b/x-pack/plugins/metrics_entities/server/modules/user_metrics/index.ts new file mode 100644 index 0000000000000..b7c6e65155ed2 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/modules/user_metrics/index.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import userMetrics from './user_metrics.json'; +import userMetricsMapping from './user_metrics_mapping.json'; + +export { userMetrics, userMetricsMapping }; diff --git a/x-pack/plugins/metrics_entities/server/modules/user_metrics/user_metrics.json b/x-pack/plugins/metrics_entities/server/modules/user_metrics/user_metrics.json new file mode 100644 index 0000000000000..86154bd8c68ec --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/modules/user_metrics/user_metrics.json @@ -0,0 +1,56 @@ +{ + "id": "user_met", + "description": "[event.category authentication metrics] grouped by @timestamp and aggregated on success, failure, and unknown", + "source": { + "query": { + "bool": { + "filter": [ + { + "bool": { + "filter": [ + { + "term": { + "event.category": "authentication" + } + } + ] + } + } + ] + } + } + }, + "pivot": { + "group_by": { + "@timestamp": { + "date_histogram": { + "field": "@timestamp", + "calendar_interval": "1h" + } + } + }, + "aggregations": { + "metrics.event.authentication.success.value_count": { + "filter": { + "term": { + "event.outcome": "success" + } + } + }, + "metrics.event.authentication.failure.value_count": { + "filter": { + "term": { + "event.outcome": "failure" + } + } + }, + "metrics.event.authentication.unknown.value_count": { + "filter": { + "term": { + "event.outcome": "unknown" + } + } + } + } + } +} diff --git a/x-pack/plugins/metrics_entities/server/modules/user_metrics/user_metrics_mapping.json b/x-pack/plugins/metrics_entities/server/modules/user_metrics/user_metrics_mapping.json new file mode 100644 index 0000000000000..c63dcd2b4a429 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/modules/user_metrics/user_metrics_mapping.json @@ -0,0 +1,46 @@ +{ + "mappings": { + "_meta": { + "index": "user_met" + }, + "dynamic": "strict", + "properties": { + "metrics": { + "properties": { + "event": { + "properties": { + "authentication": { + "properties": { + "failure": { + "properties": { + "value_count": { + "type": "long" + } + } + }, + "success": { + "properties": { + "value_count": { + "type": "long" + } + } + }, + "unknown": { + "properties": { + "value_count": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "@timestamp": { + "type": "date" + } + } + } +} diff --git a/x-pack/plugins/metrics_entities/server/plugin.ts b/x-pack/plugins/metrics_entities/server/plugin.ts new file mode 100644 index 0000000000000..73d4ffc6367fe --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/plugin.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + CoreSetup, + CoreStart, + Logger, + Plugin, + PluginInitializerContext, +} from '../../../../src/core/server'; + +import { + ContextProvider, + ContextProviderReturn, + MetricsEntitiesPluginSetup, + MetricsEntitiesPluginStart, + MetricsEntitiesRequestHandlerContext, +} from './types'; +import { getTransforms, postTransforms } from './routes'; +import { MetricsEntitiesClient } from './services/metrics_entities_client'; +import { deleteTransforms } from './routes/delete_transforms'; + +export class MetricsEntitiesPlugin + implements Plugin { + private readonly logger: Logger; + private kibanaVersion: string; + + constructor(initializerContext: PluginInitializerContext) { + this.logger = initializerContext.logger.get(); + this.kibanaVersion = initializerContext.env.packageInfo.version; + } + + public setup(core: CoreSetup): MetricsEntitiesPluginSetup { + const router = core.http.createRouter(); + + core.http.registerRouteHandlerContext( + 'metricsEntities', + this.createRouteHandlerContext() + ); + + // Register server side APIs + // TODO: Add all of these into a separate file and call that file called init_routes.ts + getTransforms(router); + postTransforms(router); + deleteTransforms(router); + + return { + getMetricsEntitiesClient: (esClient): MetricsEntitiesClient => + new MetricsEntitiesClient({ + esClient, + kibanaVersion: this.kibanaVersion, + logger: this.logger, + }), + }; + } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + public start(core: CoreStart): void { + this.logger.debug('Starting plugin'); + } + + public stop(): void { + this.logger.debug('Stopping plugin'); + } + + private createRouteHandlerContext = (): ContextProvider => { + return async (context): ContextProviderReturn => { + const { + core: { + elasticsearch: { + client: { asCurrentUser: esClient }, + }, + }, + } = context; + return { + getMetricsEntitiesClient: (): MetricsEntitiesClient => + new MetricsEntitiesClient({ + esClient, + kibanaVersion: this.kibanaVersion, + logger: this.logger, + }), + }; + }; + }; +} diff --git a/x-pack/plugins/metrics_entities/server/routes/delete_transforms.ts b/x-pack/plugins/metrics_entities/server/routes/delete_transforms.ts new file mode 100644 index 0000000000000..f5236e462dd81 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/routes/delete_transforms.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema } from '@kbn/config-schema'; + +import { IRouter } from '../../../../../src/core/server'; +import { METRICS_ENTITIES_TRANSFORMS } from '../../common/constants'; +import { ModuleNames } from '../modules'; + +import { getMetricsEntitiesClient } from './utils/get_metrics_entities_client'; + +/** + * Deletes transforms. + * NOTE: We use a POST rather than a DELETE on purpose here to ensure that there + * are not problems with the body being sent. + * @param router The router to delete the collection of transforms + */ +export const deleteTransforms = (router: IRouter): void => { + router.post( + { + path: `${METRICS_ENTITIES_TRANSFORMS}/_delete`, + validate: { + // TODO: Add the validation instead of allowing handler to have access to raw non-validated in runtime + body: schema.object({}, { unknowns: 'allow' }), + query: schema.object({}, { unknowns: 'allow' }), + }, + }, + async (context, request, response) => { + // TODO: Type this through validation above and remove the weird casting of: "as { modules: ModuleNames };" + // TODO: Validate for runtime that the module exists or not and throw before pushing the module name lower + // TODO: Change modules to be part of the body and become an array of values + // TODO: Wrap this in a try catch block and report errors + const { modules, prefix = '', suffix = '' } = request.body as { + modules: ModuleNames[]; + prefix: string; + suffix: string; + }; + const metrics = getMetricsEntitiesClient(context); + await metrics.deleteTransforms({ modules, prefix, suffix }); + + return response.custom({ + statusCode: 204, + }); + } + ); +}; diff --git a/x-pack/plugins/metrics_entities/server/routes/get_transforms.ts b/x-pack/plugins/metrics_entities/server/routes/get_transforms.ts new file mode 100644 index 0000000000000..cda61512ce293 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/routes/get_transforms.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IRouter } from '../../../../../src/core/server'; +import { METRICS_ENTITIES_TRANSFORMS } from '../../common/constants'; + +import { getMetricsEntitiesClient } from './utils/get_metrics_entities_client'; + +/** + * Returns all transforms from all modules + * TODO: Add support for specific modules and prefix + * @param router The router to get the collection of transforms + */ +export const getTransforms = (router: IRouter): void => { + router.get( + { + path: METRICS_ENTITIES_TRANSFORMS, + // TODO: Add the validation instead of false + // TODO: Add the prefix and module support + validate: false, + }, + async (context, _, response) => { + const metrics = getMetricsEntitiesClient(context); + const summaries = await metrics.getTransforms(); + return response.ok({ + body: { + summaries, + }, + }); + } + ); +}; diff --git a/x-pack/plugins/metrics_entities/server/routes/index.ts b/x-pack/plugins/metrics_entities/server/routes/index.ts new file mode 100644 index 0000000000000..9470772f46d70 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/routes/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './delete_transforms'; +export * from './get_transforms'; +export * from './post_transforms'; diff --git a/x-pack/plugins/metrics_entities/server/routes/post_transforms.ts b/x-pack/plugins/metrics_entities/server/routes/post_transforms.ts new file mode 100644 index 0000000000000..d5b5648757e8b --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/routes/post_transforms.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema } from '@kbn/config-schema'; + +import { IRouter } from '../../../../../src/core/server'; +import { METRICS_ENTITIES_TRANSFORMS } from '../../common/constants'; +import { ModuleNames } from '../modules'; + +import { getMetricsEntitiesClient } from './utils/get_metrics_entities_client'; + +/** + * Creates transforms. + * @param router The router to get the collection of transforms + */ +export const postTransforms = (router: IRouter): void => { + router.post( + { + path: METRICS_ENTITIES_TRANSFORMS, + validate: { + // TODO: Add the validation instead of allowing handler to have access to raw non-validated in runtime + body: schema.object({}, { unknowns: 'allow' }), + query: schema.object({}, { unknowns: 'allow' }), + }, + }, + async (context, request, response) => { + // TODO: Type this through validation above and remove the weird casting of: "as { modules: ModuleNames };" + // TODO: Validate for runtime that the module exists or not and throw before pushing the module name lower + // TODO: Change modules to be part of the body and become an array of values + // TODO: Wrap this in a try catch block and report errors + const { + modules, + auto_start: autoStart = false, + settings: { + max_page_search_size: maxPageSearchSize = 500, + docs_per_second: docsPerSecond = null, + } = { + docsPerSecond: null, + maxPageSearchSize: 500, + }, + frequency = '1m', + indices, + query = { match_all: {} }, + prefix = '', + suffix = '', + sync = { + time: { + delay: '60s', + field: '@timestamp', + }, + }, + } = request.body as { + modules: ModuleNames[]; + auto_start: boolean; + indices: string[]; + // We can blow up at 65 character+ for transform id. We need to validate the prefix + transform jobs and return an error + prefix: string; + query: object; + suffix: string; + frequency: string; + settings: { + max_page_search_size: number; + docs_per_second: number; + }; + sync: { + time: { + delay: string; + field: string; + }; + }; + }; + const metrics = getMetricsEntitiesClient(context); + await metrics.postTransforms({ + autoStart, + docsPerSecond, + frequency, + indices, + maxPageSearchSize, + modules, + prefix, + query, + suffix, + sync, + }); + + return response.custom({ + body: { acknowledged: true }, + statusCode: 201, + }); + } + ); +}; diff --git a/x-pack/plugins/metrics_entities/server/routes/utils/get_metrics_entities_client.ts b/x-pack/plugins/metrics_entities/server/routes/utils/get_metrics_entities_client.ts new file mode 100644 index 0000000000000..fdbbd98128741 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/routes/utils/get_metrics_entities_client.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ErrorWithStatusCode } from '../../error_with_status_code'; +import { MetricsEntitiesClient } from '../../services/metrics_entities_client'; +import type { MetricsEntitiesRequestHandlerContext } from '../../types'; + +export const getMetricsEntitiesClient = ( + context: MetricsEntitiesRequestHandlerContext +): MetricsEntitiesClient => { + const metricsEntities = context.metricsEntities?.getMetricsEntitiesClient(); + if (metricsEntities == null) { + throw new ErrorWithStatusCode('Metrics Entities is not found as a plugin', 404); + } else { + return metricsEntities; + } +}; diff --git a/x-pack/plugins/metrics_entities/server/routes/utils/index.ts b/x-pack/plugins/metrics_entities/server/routes/utils/index.ts new file mode 100644 index 0000000000000..eee678d64b30d --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/routes/utils/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './get_metrics_entities_client'; diff --git a/x-pack/plugins/metrics_entities/server/scripts/check_env_variables.sh b/x-pack/plugins/metrics_entities/server/scripts/check_env_variables.sh new file mode 100755 index 0000000000000..df2354ed8398a --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/scripts/check_env_variables.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0; you may not use this file except in compliance with the Elastic License +# 2.0. +# + +# Add this to the start of any scripts to detect if env variables are set + +set -e + +if [ -z "${ELASTICSEARCH_USERNAME}" ]; then + echo "Set ELASTICSEARCH_USERNAME in your environment" + exit 1 +fi + +if [ -z "${ELASTICSEARCH_PASSWORD}" ]; then + echo "Set ELASTICSEARCH_PASSWORD in your environment" + exit 1 +fi + +if [ -z "${ELASTICSEARCH_URL}" ]; then + echo "Set ELASTICSEARCH_URL in your environment" + exit 1 +fi + +if [ -z "${KIBANA_URL}" ]; then + echo "Set KIBANA_URL in your environment" + exit 1 +fi diff --git a/x-pack/plugins/metrics_entities/server/scripts/delete_examples/all.json b/x-pack/plugins/metrics_entities/server/scripts/delete_examples/all.json new file mode 100644 index 0000000000000..b07028d0cab89 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/scripts/delete_examples/all.json @@ -0,0 +1,11 @@ +{ + "prefix": "all", + "modules": [ + "host_metrics", + "host_entities", + "network_metrics", + "network_entities", + "user_entities", + "user_metrics" + ] +} diff --git a/x-pack/plugins/metrics_entities/server/scripts/delete_examples/all_prefix_auditbeat.json b/x-pack/plugins/metrics_entities/server/scripts/delete_examples/all_prefix_auditbeat.json new file mode 100644 index 0000000000000..5b20203075924 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/scripts/delete_examples/all_prefix_auditbeat.json @@ -0,0 +1,11 @@ +{ + "prefix": "auditbeat", + "modules": [ + "host_metrics", + "host_entities", + "network_metrics", + "network_entities", + "user_entities", + "user_metrics" + ] +} diff --git a/x-pack/plugins/metrics_entities/server/scripts/delete_examples/network_entities_auditbeat.json b/x-pack/plugins/metrics_entities/server/scripts/delete_examples/network_entities_auditbeat.json new file mode 100644 index 0000000000000..b1e21ebbc9bd6 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/scripts/delete_examples/network_entities_auditbeat.json @@ -0,0 +1,3 @@ +{ + "modules": ["network_entities"] +} diff --git a/x-pack/plugins/metrics_entities/server/scripts/delete_examples/one_module.json b/x-pack/plugins/metrics_entities/server/scripts/delete_examples/one_module.json new file mode 100644 index 0000000000000..2e9a62b9fbe82 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/scripts/delete_examples/one_module.json @@ -0,0 +1,3 @@ +{ + "modules": ["user_entities"] +} diff --git a/x-pack/plugins/metrics_entities/server/scripts/delete_examples/two_modules.json b/x-pack/plugins/metrics_entities/server/scripts/delete_examples/two_modules.json new file mode 100644 index 0000000000000..e3292834f3d08 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/scripts/delete_examples/two_modules.json @@ -0,0 +1,3 @@ +{ + "modules": ["host_metrics", "host_entities"] +} diff --git a/x-pack/plugins/metrics_entities/server/scripts/delete_transforms.sh b/x-pack/plugins/metrics_entities/server/scripts/delete_transforms.sh new file mode 100755 index 0000000000000..d4c03411cbcca --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/scripts/delete_transforms.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0; you may not use this file except in compliance with the Elastic License +# 2.0. +# + +set -e +./check_env_variables.sh + +# Uses a default if no argument is specified +FILE=${1:-./post_examples/one_module.json} + +# Example: ./delete_transforms.sh ./delete_examples/one_module.json +curl -s -k \ + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: 123' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X POST ${KIBANA_URL}${SPACE_URL}/api/metrics_entities/transforms/_delete \ + -d @${FILE} \ + | jq . diff --git a/x-pack/plugins/metrics_entities/server/scripts/get_transforms.sh b/x-pack/plugins/metrics_entities/server/scripts/get_transforms.sh new file mode 100755 index 0000000000000..34f7e4b83cc39 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/scripts/get_transforms.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0; you may not use this file except in compliance with the Elastic License +# 2.0. +# + +set -e +./check_env_variables.sh + +# Example: ./get_transforms.sh +curl -s -k \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X GET ${KIBANA_URL}${SPACE_URL}/api/metrics_entities/transforms | jq . diff --git a/x-pack/plugins/metrics_entities/server/scripts/hard_reset.sh b/x-pack/plugins/metrics_entities/server/scripts/hard_reset.sh new file mode 100755 index 0000000000000..69acf10764936 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/scripts/hard_reset.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0; you may not use this file except in compliance with the Elastic License +# 2.0. +# + +# TODO Make this work + +set -e +./check_env_variables.sh + +# remove all templates +# add all templates again and start them + diff --git a/x-pack/plugins/metrics_entities/server/scripts/post_examples/all.json b/x-pack/plugins/metrics_entities/server/scripts/post_examples/all.json new file mode 100644 index 0000000000000..dac53a63dad55 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/scripts/post_examples/all.json @@ -0,0 +1,32 @@ +{ + "prefix": "all", + "modules": [ + "host_metrics", + "host_entities", + "network_metrics", + "network_entities", + "user_entities", + "user_metrics" + ], + "indices": [ + "auditbeat-*", + "endgame-*", + "filebeat-*", + "logs-*", + "packetbeat-*", + "winlogbeat-*", + "-*elastic-cloud-logs-*" + ], + "auto_start": true, + "settings": { + "max_page_search_size": 5000 + }, + "query": { + "range": { + "@timestamp": { + "gte": "now-1d/d", + "format": "strict_date_optional_time" + } + } + } +} diff --git a/x-pack/plugins/metrics_entities/server/scripts/post_examples/all_auditbeat.json b/x-pack/plugins/metrics_entities/server/scripts/post_examples/all_auditbeat.json new file mode 100644 index 0000000000000..5a2f6b5024689 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/scripts/post_examples/all_auditbeat.json @@ -0,0 +1,23 @@ +{ + "modules": [ + "host_metrics", + "host_entities", + "network_metrics", + "network_entities", + "user_entities", + "user_metrics" + ], + "indices": ["auditbeat-*"], + "auto_start": true, + "settings": { + "max_page_search_size": 5000 + }, + "query": { + "range": { + "@timestamp": { + "gte": "now-1d/d", + "format": "strict_date_optional_time" + } + } + } +} diff --git a/x-pack/plugins/metrics_entities/server/scripts/post_examples/network_entities_auditbeat.json b/x-pack/plugins/metrics_entities/server/scripts/post_examples/network_entities_auditbeat.json new file mode 100644 index 0000000000000..379a5733a91f9 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/scripts/post_examples/network_entities_auditbeat.json @@ -0,0 +1,4 @@ +{ + "modules": ["network_entities"], + "indices": ["auditbeat-*"] +} diff --git a/x-pack/plugins/metrics_entities/server/scripts/post_examples/one_module_allindices_autostart.json b/x-pack/plugins/metrics_entities/server/scripts/post_examples/one_module_allindices_autostart.json new file mode 100644 index 0000000000000..9872706ff1ac2 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/scripts/post_examples/one_module_allindices_autostart.json @@ -0,0 +1,24 @@ +{ + "modules": ["network_metrics"], + "indices": [ + "auditbeat-*", + "endgame-*", + "filebeat-*", + "logs-*", + "packetbeat-*", + "winlogbeat-*", + "-*elastic-cloud-logs-*" + ], + "auto_start": true, + "query": { + "range": { + "@timestamp": { + "gte": "now-1d/d", + "format": "strict_date_optional_time" + } + } + }, + "settings": { + "max_page_search_size": 5000 + } +} diff --git a/x-pack/plugins/metrics_entities/server/scripts/post_examples/one_module_auditbeat.json b/x-pack/plugins/metrics_entities/server/scripts/post_examples/one_module_auditbeat.json new file mode 100644 index 0000000000000..4ce4db5da9f23 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/scripts/post_examples/one_module_auditbeat.json @@ -0,0 +1,16 @@ +{ + "modules": ["network_metrics"], + "indices": ["auditbeat-*"], + "auto_start": true, + "query": { + "range": { + "@timestamp": { + "gte": "now-1d/d", + "format": "strict_date_optional_time" + } + } + }, + "settings": { + "max_page_search_size": 5000 + } +} diff --git a/x-pack/plugins/metrics_entities/server/scripts/post_examples/one_module_auto_start.json b/x-pack/plugins/metrics_entities/server/scripts/post_examples/one_module_auto_start.json new file mode 100644 index 0000000000000..d5a87c80a44a0 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/scripts/post_examples/one_module_auto_start.json @@ -0,0 +1,8 @@ +{ + "modules": ["host_metrics"], + "indices": ["auditbeat-*"], + "auto_start": true, + "settings": { + "max_page_search_size": 5000 + } +} diff --git a/x-pack/plugins/metrics_entities/server/scripts/post_examples/one_module_prefix_auditbeat.json b/x-pack/plugins/metrics_entities/server/scripts/post_examples/one_module_prefix_auditbeat.json new file mode 100644 index 0000000000000..f20875f28ffa3 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/scripts/post_examples/one_module_prefix_auditbeat.json @@ -0,0 +1,5 @@ +{ + "modules": ["host_metrics"], + "indices": ["auditbeat-*"], + "prefix": ["default_"] +} diff --git a/x-pack/plugins/metrics_entities/server/scripts/post_examples/two_modules_all.json b/x-pack/plugins/metrics_entities/server/scripts/post_examples/two_modules_all.json new file mode 100644 index 0000000000000..8ec9401b94433 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/scripts/post_examples/two_modules_all.json @@ -0,0 +1,24 @@ +{ + "modules": ["network_metrics", "network_entities"], + "indices": [ + "auditbeat-*", + "endgame-*", + "filebeat-*", + "logs-*", + "packetbeat-*", + "winlogbeat-*", + "-*elastic-cloud-logs-*" + ], + "auto_start": true, + "query": { + "range": { + "@timestamp": { + "gte": "now-1d/d", + "format": "strict_date_optional_time" + } + } + }, + "settings": { + "max_page_search_size": 5000 + } +} diff --git a/x-pack/plugins/metrics_entities/server/scripts/post_examples/two_modules_auditbeat.json b/x-pack/plugins/metrics_entities/server/scripts/post_examples/two_modules_auditbeat.json new file mode 100644 index 0000000000000..5229cd88fc433 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/scripts/post_examples/two_modules_auditbeat.json @@ -0,0 +1,4 @@ +{ + "modules": ["host_metrics", "host_entities"], + "indices": ["auditbeat-*"] +} diff --git a/x-pack/plugins/metrics_entities/server/scripts/post_transforms.sh b/x-pack/plugins/metrics_entities/server/scripts/post_transforms.sh new file mode 100755 index 0000000000000..9dd4169cc01d6 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/scripts/post_transforms.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0; you may not use this file except in compliance with the Elastic License +# 2.0. +# + +set -e +./check_env_variables.sh + +# Uses a default if no argument is specified +FILE=${1:-./post_examples/one_module_auditbeat.json} + +# Example: ./post_transforms.sh ./post_examples/one_module_auditbeat.json +# Example: ./post_transforms.sh ./post_examples/one_module_namespace_auditbeat.json +curl -s -k \ + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: 123' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X POST ${KIBANA_URL}${SPACE_URL}/api/metrics_entities/transforms \ + -d @${FILE} \ + | jq . diff --git a/x-pack/plugins/metrics_entities/server/scripts/update_transforms.sh b/x-pack/plugins/metrics_entities/server/scripts/update_transforms.sh new file mode 100755 index 0000000000000..bccf49e2d1b0d --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/scripts/update_transforms.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0; you may not use this file except in compliance with the Elastic License +# 2.0. +# + +set -e +./check_env_variables.sh + +# TODO Make this work diff --git a/x-pack/plugins/metrics_entities/server/services/delete_transforms.ts b/x-pack/plugins/metrics_entities/server/services/delete_transforms.ts new file mode 100644 index 0000000000000..ef172bcbf7c02 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/services/delete_transforms.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from 'kibana/server'; + +import { ModuleNames, installableMappings, installableTransforms } from '../modules'; +import type { Logger } from '../../../../../src/core/server'; + +import { uninstallMappings } from './uninstall_mappings'; +import { uninstallTransforms } from './uninstall_transforms'; + +interface DeleteTransformsOptions { + esClient: ElasticsearchClient; + logger: Logger; + modules: ModuleNames[]; + prefix: string; + suffix: string; +} + +export const deleteTransforms = async ({ + esClient, + logger, + modules, + prefix, + suffix, +}: DeleteTransformsOptions): Promise => { + for (const moduleName of modules) { + const mappings = installableMappings[moduleName]; + const transforms = installableTransforms[moduleName]; + + await uninstallTransforms({ esClient, logger, prefix, suffix, transforms }); + await uninstallMappings({ esClient, logger, mappings, prefix, suffix }); + } +}; diff --git a/x-pack/plugins/metrics_entities/server/services/get_transforms.ts b/x-pack/plugins/metrics_entities/server/services/get_transforms.ts new file mode 100644 index 0000000000000..08189f4b3361a --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/services/get_transforms.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from 'kibana/server'; + +import type { Logger } from '../../../../../src/core/server'; + +interface GetTransformsOptions { + esClient: ElasticsearchClient; + logger: Logger; +} + +// TODO: Type the Promise to a stronger type +export const getTransforms = async ({ esClient }: GetTransformsOptions): Promise => { + const { body } = await esClient.transform.getTransform({ + size: 1000, + transform_id: '*', + }); + return body; +}; diff --git a/x-pack/plugins/metrics_entities/server/services/index.ts b/x-pack/plugins/metrics_entities/server/services/index.ts new file mode 100644 index 0000000000000..71611d2a5eae0 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/services/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './delete_transforms'; +export * from './get_transforms'; +export * from './install_mappings'; +export * from './install_transforms'; +export * from './metrics_entities_client'; +export * from './post_transforms'; +export * from './uninstall_mappings'; +export * from './uninstall_transforms'; diff --git a/x-pack/plugins/metrics_entities/server/services/install_mappings.ts b/x-pack/plugins/metrics_entities/server/services/install_mappings.ts new file mode 100644 index 0000000000000..da42f9916ff9b --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/services/install_mappings.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from 'kibana/server'; + +import { Mappings } from '../modules/types'; +import type { Logger } from '../../../../../src/core/server'; + +import { + computeMappingId, + getIndexExists, + logMappingDebug, + logMappingError, + logMappingInfo, +} from './utils'; + +interface CreateMappingOptions { + esClient: ElasticsearchClient; + mappings: Mappings[]; + prefix: string; + suffix: string; + logger: Logger; + kibanaVersion: string; +} + +export const installMappings = async ({ + esClient, + kibanaVersion, + mappings, + prefix, + suffix, + logger, +}: CreateMappingOptions): Promise => { + for (const mapping of mappings) { + const { index } = mapping.mappings._meta; + const mappingId = computeMappingId({ id: index, prefix, suffix }); + const exists = await getIndexExists(esClient, mappingId); + const computedBody = { + ...mapping, + ...{ + mappings: { + ...mapping.mappings, + _meta: { + ...mapping.mappings._meta, + ...{ + created_by: 'metrics_entities', + index: mappingId, + version: kibanaVersion, + }, + }, + }, + }, + }; + if (!exists) { + try { + logMappingInfo({ id: mappingId, logger, message: 'does not exist, creating the mapping' }); + await esClient.indices.create({ + body: computedBody, + index: mappingId, + }); + } catch (error) { + logMappingError({ + error, + id: mappingId, + logger, + message: 'cannot install mapping', + postBody: computedBody, + }); + } + } else { + logMappingDebug({ + id: mappingId, + logger, + message: 'mapping already exists. It will not be recreated', + }); + } + } +}; diff --git a/x-pack/plugins/metrics_entities/server/services/install_transforms.ts b/x-pack/plugins/metrics_entities/server/services/install_transforms.ts new file mode 100644 index 0000000000000..d0a81955ca184 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/services/install_transforms.ts @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from 'kibana/server'; + +import { Transforms } from '../modules/types'; +import type { Logger } from '../../../../../src/core/server'; + +import { + computeMappingId, + computeTransformId, + getTransformExists, + logTransformDebug, + logTransformError, + logTransformInfo, +} from './utils'; + +interface CreateTransformOptions { + esClient: ElasticsearchClient; + transforms: Transforms[]; + autoStart: boolean; + indices: string[]; + frequency: string; + logger: Logger; + query: object; + docsPerSecond: number | null; + maxPageSearchSize: number; + sync: { + time: { + delay: string; + field: string; + }; + }; + prefix: string; + suffix: string; +} + +export const installTransforms = async ({ + autoStart, + esClient, + frequency, + indices, + docsPerSecond, + logger, + maxPageSearchSize, + prefix, + suffix, + transforms, + query, + sync, +}: CreateTransformOptions): Promise => { + for (const transform of transforms) { + const destIndex = transform?.dest?.index ?? transform.id; + const computedMappingIndex = computeMappingId({ id: destIndex, prefix, suffix }); + const { id, ...transformNoId } = { + ...transform, + ...{ source: { ...transform.source, index: indices, query } }, + ...{ dest: { ...transform.dest, index: computedMappingIndex } }, + ...{ + settings: { + ...transform.settings, + docs_per_second: docsPerSecond, + max_page_search_size: maxPageSearchSize, + }, + }, + frequency, + sync, + }; + + const computedName = computeTransformId({ id, prefix, suffix }); + const exists = await getTransformExists(esClient, computedName); + if (!exists) { + try { + logTransformInfo({ + id: computedName, + logger, + message: 'does not exist, creating the transform', + }); + await esClient.transform.putTransform({ + body: transformNoId, + defer_validation: true, + transform_id: computedName, + }); + + if (autoStart) { + logTransformInfo({ + id: computedName, + logger, + message: 'is being auto started', + }); + await esClient.transform.startTransform({ + transform_id: computedName, + }); + } else { + logTransformInfo({ + id: computedName, + logger, + message: 'is not being auto started', + }); + } + } catch (error) { + logTransformError({ + error, + id: computedName, + logger, + message: 'Could not create and/or start', + postBody: transformNoId, + }); + } + } else { + logTransformDebug({ + id: computedName, + logger, + message: 'already exists. It will not be recreated', + }); + } + } +}; diff --git a/x-pack/plugins/metrics_entities/server/services/metrics_entities_client.ts b/x-pack/plugins/metrics_entities/server/services/metrics_entities_client.ts new file mode 100644 index 0000000000000..3905503df876d --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/services/metrics_entities_client.ts @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from 'kibana/server'; + +import type { Logger } from '../../../../../src/core/server'; + +import { getTransforms } from './get_transforms'; +import { + ConstructorOptions, + DeleteTransformsOptions, + PostTransformsOptions, +} from './metrics_entities_client_types'; +import { postTransforms } from './post_transforms'; +import { deleteTransforms } from './delete_transforms'; + +export class MetricsEntitiesClient { + private readonly esClient: ElasticsearchClient; + private readonly logger: Logger; + private readonly kibanaVersion: string; + + constructor({ esClient, logger, kibanaVersion }: ConstructorOptions) { + this.esClient = esClient; + this.logger = logger; + this.kibanaVersion = kibanaVersion; + } + + // TODO: Type the unknown to be stronger + public getTransforms = async (): Promise => { + const { esClient, logger } = this; + return getTransforms({ esClient, logger }); + }; + + public postTransforms = async ({ + autoStart, + frequency, + docsPerSecond, + maxPageSearchSize, + modules, + indices, + prefix, + suffix, + query, + sync, + }: PostTransformsOptions): Promise => { + const { esClient, logger, kibanaVersion } = this; + return postTransforms({ + autoStart, + docsPerSecond, + esClient, + frequency, + indices, + kibanaVersion, + logger, + maxPageSearchSize, + modules, + prefix, + query, + suffix, + sync, + }); + }; + + public deleteTransforms = async ({ + modules, + prefix, + suffix, + }: DeleteTransformsOptions): Promise => { + const { esClient, logger } = this; + return deleteTransforms({ esClient, logger, modules, prefix, suffix }); + }; +} diff --git a/x-pack/plugins/metrics_entities/server/services/metrics_entities_client_types.ts b/x-pack/plugins/metrics_entities/server/services/metrics_entities_client_types.ts new file mode 100644 index 0000000000000..1ae9f0d7a2f53 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/services/metrics_entities_client_types.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from 'kibana/server'; + +import type { Logger } from '../../../../../src/core/server'; +import { ModuleNames } from '../modules'; + +export interface ConstructorOptions { + esClient: ElasticsearchClient; + logger: Logger; + kibanaVersion: string; +} + +export interface PostTransformsOptions { + modules: ModuleNames[]; + autoStart: boolean; + frequency: string; + indices: string[]; + docsPerSecond: number | null; + maxPageSearchSize: number; + prefix: string; + query: object; + suffix: string; + sync: { + time: { + delay: string; + field: string; + }; + }; +} + +export interface DeleteTransformsOptions { + modules: ModuleNames[]; + prefix: string; + suffix: string; +} diff --git a/x-pack/plugins/metrics_entities/server/services/post_transforms.ts b/x-pack/plugins/metrics_entities/server/services/post_transforms.ts new file mode 100644 index 0000000000000..1850047ae1e9d --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/services/post_transforms.ts @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from 'kibana/server'; + +import { ModuleNames, installableMappings, installableTransforms } from '../modules'; +import type { Logger } from '../../../../../src/core/server'; + +import { installMappings } from './install_mappings'; +import { installTransforms } from './install_transforms'; + +interface PostTransformsOptions { + logger: Logger; + esClient: ElasticsearchClient; + modules: ModuleNames[]; + autoStart: boolean; + frequency: string; + indices: string[]; + docsPerSecond: number | null; + kibanaVersion: string; + maxPageSearchSize: number; + query: object; + prefix: string; + suffix: string; + sync: { + time: { + delay: string; + field: string; + }; + }; +} + +export const postTransforms = async ({ + autoStart, + logger, + esClient, + frequency, + indices, + docsPerSecond, + kibanaVersion, + maxPageSearchSize, + modules, + prefix, + suffix, + query, + sync, +}: PostTransformsOptions): Promise => { + for (const moduleName of modules) { + const mappings = installableMappings[moduleName]; + const transforms = installableTransforms[moduleName]; + + await installMappings({ esClient, kibanaVersion, logger, mappings, prefix, suffix }); + await installTransforms({ + autoStart, + docsPerSecond, + esClient, + frequency, + indices, + logger, + maxPageSearchSize, + prefix, + query, + suffix, + sync, + transforms, + }); + } +}; diff --git a/x-pack/plugins/metrics_entities/server/services/stop_transforms.ts b/x-pack/plugins/metrics_entities/server/services/stop_transforms.ts new file mode 100644 index 0000000000000..18476d8345cf2 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/services/stop_transforms.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// TODO: Write this diff --git a/x-pack/plugins/metrics_entities/server/services/uninstall_mappings.ts b/x-pack/plugins/metrics_entities/server/services/uninstall_mappings.ts new file mode 100644 index 0000000000000..b2ea9d96cda13 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/services/uninstall_mappings.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from 'kibana/server'; + +import { Mappings } from '../modules/types'; +import type { Logger } from '../../../../../src/core/server'; + +import { computeMappingId, logMappingInfo } from './utils'; +import { logMappingError } from './utils/log_mapping_error'; + +interface UninstallMappingOptions { + esClient: ElasticsearchClient; + mappings: Mappings[]; + prefix: string; + suffix: string; + logger: Logger; +} + +export const uninstallMappings = async ({ + esClient, + logger, + mappings, + prefix, + suffix, +}: UninstallMappingOptions): Promise => { + const indices = mappings.map((mapping) => { + const { index } = mapping.mappings._meta; + return computeMappingId({ id: index, prefix, suffix }); + }); + logMappingInfo({ + id: indices.join(), + logger, + message: 'deleting indices', + }); + try { + await esClient.indices.delete({ + allow_no_indices: true, + ignore_unavailable: true, + index: indices, + }); + } catch (error) { + logMappingError({ + error, + id: indices.join(), + logger, + message: 'could not delete index', + postBody: undefined, + }); + } +}; diff --git a/x-pack/plugins/metrics_entities/server/services/uninstall_transforms.ts b/x-pack/plugins/metrics_entities/server/services/uninstall_transforms.ts new file mode 100644 index 0000000000000..11f12541bda0d --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/services/uninstall_transforms.ts @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from 'kibana/server'; + +import { Transforms } from '../modules/types'; +import type { Logger } from '../../../../../src/core/server'; + +import { + computeTransformId, + getTransformExists, + logTransformError, + logTransformInfo, +} from './utils'; + +interface UninstallTransformsOptions { + esClient: ElasticsearchClient; + transforms: Transforms[]; + prefix: string; + suffix: string; + logger: Logger; +} + +/** + * Uninstalls all the transforms underneath a given module + */ +export const uninstallTransforms = async ({ + esClient, + logger, + prefix, + suffix, + transforms, +}: UninstallTransformsOptions): Promise => { + transforms.forEach(async (transform) => { + const { id } = transform; + const computedId = computeTransformId({ id, prefix, suffix }); + const exists = await getTransformExists(esClient, computedId); + if (exists) { + logTransformInfo({ + id: computedId, + logger, + message: 'stopping transform', + }); + try { + await esClient.transform.stopTransform({ + allow_no_match: true, + force: true, + timeout: '5s', + transform_id: computedId, + wait_for_completion: true, + }); + } catch (error) { + logTransformError({ + error, + id: computedId, + logger, + message: 'Could not stop transform, still attempting to delete it', + postBody: undefined, + }); + } + logTransformInfo({ + id: computedId, + logger, + message: 'deleting transform', + }); + try { + await esClient.transform.deleteTransform({ + force: true, + transform_id: computedId, + }); + } catch (error) { + logTransformError({ + error, + id: computedId, + logger, + message: 'Could not create and/or start', + postBody: undefined, + }); + } + } else { + logTransformInfo({ + id: computedId, + logger, + message: 'transform does not exist to delete', + }); + } + }); +}; diff --git a/x-pack/plugins/metrics_entities/server/services/utils/compute_mapping_index.ts b/x-pack/plugins/metrics_entities/server/services/utils/compute_mapping_index.ts new file mode 100644 index 0000000000000..bb1a7720fc575 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/services/utils/compute_mapping_index.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { computeTransformId } from './compute_transform_id'; + +export const computeMappingId = ({ + prefix, + id, + suffix, +}: { + prefix: string; + id: string; + suffix: string; +}): string => { + // TODO: This causes issues if above 65 character limit. We should limit the prefix + // and anything else on the incoming routes to avoid this causing an issue. We should still + // throw here in case I change the prefix or other names and cause issues. + const computedId = computeTransformId({ id, prefix, suffix }); + return `.${computedId}`; +}; diff --git a/x-pack/plugins/metrics_entities/server/services/utils/compute_transform_id.ts b/x-pack/plugins/metrics_entities/server/services/utils/compute_transform_id.ts new file mode 100644 index 0000000000000..20951b0e447ff --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/services/utils/compute_transform_id.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ELASTIC_NAME } from '../../../common'; + +export const computeTransformId = ({ + prefix, + id, + suffix, +}: { + prefix: string; + id: string; + suffix: string; +}): string => { + const prefixExists = prefix.trim() !== ''; + const suffixExists = suffix.trim() !== ''; + + // TODO: Check for invalid characters on the main route for prefixExists and suffixExists and do an invalidation + // if either have invalid characters for a job name. Might want to add that same check within the API too at a top level? + if (prefixExists && suffixExists) { + return `${ELASTIC_NAME}_${prefix}_${id}_${suffix}`; + } else if (prefixExists) { + return `${ELASTIC_NAME}_${prefix}_${id}`; + } else if (suffixExists) { + return `${ELASTIC_NAME}_${id}_${suffix}`; + } else { + return `${ELASTIC_NAME}_${id}`; + } +}; diff --git a/x-pack/plugins/metrics_entities/server/services/utils/get_index_exists.ts b/x-pack/plugins/metrics_entities/server/services/utils/get_index_exists.ts new file mode 100644 index 0000000000000..bcc37ce047d24 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/services/utils/get_index_exists.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from 'kibana/server'; + +/** + * Tried and true, copied forever again and again, the way we check if an index exists + * with the least amount of privileges. + * @param esClient The client to check if the index already exists + * @param index The index to check for + * @returns true if it exists, otherwise false + */ +export const getIndexExists = async ( + esClient: ElasticsearchClient, + index: string +): Promise => { + try { + const { body: response } = await esClient.search({ + allow_no_indices: true, + body: { + terminate_after: 1, + }, + index, + size: 0, + }); + return response._shards.total > 0; + } catch (err) { + if (err.body?.status === 404) { + return false; + } else { + throw err.body ? err.body : err; + } + } +}; diff --git a/x-pack/plugins/metrics_entities/server/services/utils/get_json.ts b/x-pack/plugins/metrics_entities/server/services/utils/get_json.ts new file mode 100644 index 0000000000000..71853f2a4ee66 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/services/utils/get_json.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// TODO: Move indent to configuration part or flip to default false +export const getJSON = (body: unknown, indent: boolean = true): string => + indent ? JSON.stringify(body, null, 2) : JSON.stringify(body); diff --git a/x-pack/plugins/metrics_entities/server/services/utils/get_transform_exists.ts b/x-pack/plugins/metrics_entities/server/services/utils/get_transform_exists.ts new file mode 100644 index 0000000000000..4dffce5f4ecbe --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/services/utils/get_transform_exists.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from 'kibana/server'; + +export const getTransformExists = async ( + esClient: ElasticsearchClient, + id: string +): Promise => { + try { + const { + body: { count }, + } = await esClient.transform.getTransform({ + size: 1000, + transform_id: id, + }); + return count > 0; + } catch (err) { + if (err.body?.status === 404) { + return false; + } else { + throw err.body ? err.body : err; + } + } +}; diff --git a/x-pack/plugins/metrics_entities/server/services/utils/index.ts b/x-pack/plugins/metrics_entities/server/services/utils/index.ts new file mode 100644 index 0000000000000..0871c1bf3f7b4 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/services/utils/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './compute_mapping_index'; +export * from './compute_transform_id'; +export * from './get_index_exists'; +export * from './get_transform_exists'; +export * from './log_mapping_debug'; +export * from './log_mapping_error'; +export * from './log_mapping_info'; +export * from './log_transform_debug'; +export * from './log_transform_error'; +export * from './log_transform_info'; diff --git a/x-pack/plugins/metrics_entities/server/services/utils/log_mapping_debug.ts b/x-pack/plugins/metrics_entities/server/services/utils/log_mapping_debug.ts new file mode 100644 index 0000000000000..f3c56aac900f1 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/services/utils/log_mapping_debug.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '../../../../../../src/core/server'; + +export const logMappingDebug = ({ + logger, + id, + message, +}: { + logger: Logger; + id: string; + message: string; +}): void => { + logger.debug(`mapping id: "${id}", ${message}`); +}; diff --git a/x-pack/plugins/metrics_entities/server/services/utils/log_mapping_error.ts b/x-pack/plugins/metrics_entities/server/services/utils/log_mapping_error.ts new file mode 100644 index 0000000000000..43ae07619318c --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/services/utils/log_mapping_error.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '../../../../../../src/core/server'; + +import { getJSON } from './get_json'; + +export const logMappingError = ({ + logger, + id, + message, + error, + postBody, +}: { + logger: Logger; + id: string; + error: unknown; + message: string; + postBody: {} | undefined; +}): void => { + const postString = postBody != null ? `, post body: "${getJSON(postBody)}"` : ''; + logger.error(`${message}, mapping id: "${id}"${postString}, error: ${getJSON(error)}`); +}; diff --git a/x-pack/plugins/metrics_entities/server/services/utils/log_mapping_info.ts b/x-pack/plugins/metrics_entities/server/services/utils/log_mapping_info.ts new file mode 100644 index 0000000000000..e75c380aad38a --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/services/utils/log_mapping_info.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '../../../../../../src/core/server'; + +export const logMappingInfo = ({ + logger, + id, + message, +}: { + logger: Logger; + id: string; + message: string; +}): void => { + logger.info(`mapping id: "${id}", ${message}`); +}; diff --git a/x-pack/plugins/metrics_entities/server/services/utils/log_transform_debug.ts b/x-pack/plugins/metrics_entities/server/services/utils/log_transform_debug.ts new file mode 100644 index 0000000000000..61c5dd0b37947 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/services/utils/log_transform_debug.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '../../../../../../src/core/server'; + +export const logTransformDebug = ({ + logger, + id, + message, +}: { + logger: Logger; + id: string; + message: string; +}): void => { + logger.debug(`transform id: "${id}", ${message}`); +}; diff --git a/x-pack/plugins/metrics_entities/server/services/utils/log_transform_error.ts b/x-pack/plugins/metrics_entities/server/services/utils/log_transform_error.ts new file mode 100644 index 0000000000000..2d883ca68be75 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/services/utils/log_transform_error.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '../../../../../../src/core/server'; + +import { getJSON } from './get_json'; + +export const logTransformError = ({ + id, + logger, + error, + postBody, + message, +}: { + logger: Logger; + id: string; + error: unknown; + message: string; + postBody: {} | undefined; +}): void => { + const postString = postBody != null ? `, post body: "${getJSON(postBody)}"` : ''; + logger.error(`${message}, transform id: ${id}${postString}, response error: ${getJSON(error)}`); +}; diff --git a/x-pack/plugins/metrics_entities/server/services/utils/log_transform_info.ts b/x-pack/plugins/metrics_entities/server/services/utils/log_transform_info.ts new file mode 100644 index 0000000000000..1bfb918664007 --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/services/utils/log_transform_info.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '../../../../../../src/core/server'; + +export const logTransformInfo = ({ + logger, + id, + message, +}: { + logger: Logger; + id: string; + message: string; +}): void => { + logger.info(`transform id: "${id}", ${message}`); +}; diff --git a/x-pack/plugins/metrics_entities/server/types.ts b/x-pack/plugins/metrics_entities/server/types.ts new file mode 100644 index 0000000000000..41df562234c0d --- /dev/null +++ b/x-pack/plugins/metrics_entities/server/types.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, IContextProvider, RequestHandlerContext } from 'kibana/server'; + +import { MetricsEntitiesClient } from './services/metrics_entities_client'; + +export type GetMetricsEntitiesClientType = (esClient: ElasticsearchClient) => MetricsEntitiesClient; + +export interface MetricsEntitiesPluginSetup { + getMetricsEntitiesClient: GetMetricsEntitiesClientType; +} + +export type MetricsEntitiesPluginStart = void; + +export type ContextProvider = IContextProvider< + MetricsEntitiesRequestHandlerContext, + 'metricsEntities' +>; + +export interface MetricsEntitiesApiRequestHandlerContext { + getMetricsEntitiesClient: () => MetricsEntitiesClient; +} + +export interface MetricsEntitiesRequestHandlerContext extends RequestHandlerContext { + metricsEntities?: MetricsEntitiesApiRequestHandlerContext; +} + +/** + * @internal + */ +export type ContextProviderReturn = Promise; diff --git a/x-pack/plugins/metrics_entities/tsconfig.json b/x-pack/plugins/metrics_entities/tsconfig.json new file mode 100644 index 0000000000000..15e6aa1601627 --- /dev/null +++ b/x-pack/plugins/metrics_entities/tsconfig.json @@ -0,0 +1,27 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "composite": true, + "outDir": "./target/types", + "emitDeclarationOnly": true, + "declaration": true, + "declarationMap": true + }, + "include": [ + "common/**/*", + "public/**/*", + "server/**/*", + // have to declare *.json explicitly due to https://github.com/microsoft/TypeScript/issues/25636 + "server/**/*.json", + "../../../typings/**/*" + ], + "references": [ + { "path": "../../../src/core/tsconfig.json" }, + { "path": "../spaces/tsconfig.json" }, + { "path": "../security/tsconfig.json" }, + { "path": "../licensing/tsconfig.json" }, + { "path": "../features/tsconfig.json" }, + { "path": "../../../src/plugins/usage_collection/tsconfig.json" }, + { "path": "../../../src/plugins/kibana_utils/tsconfig.json" } + ] +} diff --git a/x-pack/plugins/security_solution/common/constants.ts b/x-pack/plugins/security_solution/common/constants.ts index a735f3885cf2c..fa6bb497d2434 100644 --- a/x-pack/plugins/security_solution/common/constants.ts +++ b/x-pack/plugins/security_solution/common/constants.ts @@ -5,6 +5,7 @@ * 2.0. */ +import type { TransformConfigSchema } from './transforms/types'; import { ENABLE_CASE_CONNECTOR } from '../../cases/common'; export const APP_ID = 'securitySolution'; @@ -38,6 +39,7 @@ export const DEFAULT_INTERVAL_PAUSE = true; export const DEFAULT_INTERVAL_TYPE = 'manual'; export const DEFAULT_INTERVAL_VALUE = 300000; // ms export const DEFAULT_TIMEPICKER_QUICK_RANGES = 'timepicker:quickRanges'; +export const DEFAULT_TRANSFORMS = 'securitySolution:transforms'; export const SCROLLING_DISABLED_CLASS_NAME = 'scrolling-disabled'; export const GLOBAL_HEADER_HEIGHT = 98; // px export const FILTERS_GLOBAL_HEIGHT = 109; // px @@ -106,6 +108,38 @@ export const IP_REPUTATION_LINKS_SETTING_DEFAULT = `[ { "name": "talosIntelligence.com", "url_template": "https://talosintelligence.com/reputation_center/lookup?search={{ip}}" } ]`; +/** The default settings for the transforms */ +export const defaultTransformsSetting: TransformConfigSchema = { + enabled: false, + auto_start: true, + auto_create: true, + query: { + range: { + '@timestamp': { + gte: 'now-1d/d', + format: 'strict_date_optional_time', + }, + }, + }, + retention_policy: { + time: { + field: '@timestamp', + max_age: '1w', + }, + }, + max_page_search_size: 5000, + settings: [ + { + prefix: 'all', + indices: ['auditbeat-*', 'endgame-*', 'filebeat-*', 'logs-*', 'packetbeat-*', 'winlogbeat-*'], + data_sources: [ + ['auditbeat-*', 'endgame-*', 'filebeat-*', 'logs-*', 'packetbeat-*', 'winlogbeat-*'], + ], + }, + ], +}; +export const DEFAULT_TRANSFORMS_SETTING = JSON.stringify(defaultTransformsSetting, null, 2); + /** * Id for the signals alerting type */ @@ -214,3 +248,10 @@ export const showAllOthersBucket: string[] = [ 'destination.ip', 'user.name', ]; + +/** + * Used for transforms for metrics_entities. If the security_solutions pulls in + * the metrics_entities plugin, then it should pull this constant from there rather + * than use it from here. + */ +export const ELASTIC_NAME = 'estc'; diff --git a/x-pack/plugins/security_solution/common/experimental_features.ts b/x-pack/plugins/security_solution/common/experimental_features.ts index 70ed468e61554..8d1cc4ca2c1f0 100644 --- a/x-pack/plugins/security_solution/common/experimental_features.ts +++ b/x-pack/plugins/security_solution/common/experimental_features.ts @@ -13,6 +13,7 @@ export type ExperimentalFeatures = typeof allowedExperimentalValues; */ const allowedExperimentalValues = Object.freeze({ trustedAppsByPolicyEnabled: false, + metricsEntitiesEnabled: false, eventFilteringEnabled: false, hostIsolationEnabled: false, }); diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/hosts/authentications/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/hosts/authentications/index.ts index 0905fc052d1a9..a000fcf6136e5 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/hosts/authentications/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/hosts/authentications/index.ts @@ -72,9 +72,13 @@ export interface AuthenticationBucket { doc_count: number; failures: { doc_count: number; + // TODO: Keep this or make a new structure? + value?: number; }; successes: { doc_count: number; + // TODO: Keep this or make a new structure? + value?: number; }; authentication: { hits: { diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/hosts/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/hosts/index.ts index 3926fdc72f73a..bae99649c2e01 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/hosts/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/hosts/index.ts @@ -16,9 +16,11 @@ export * from './uncommon_processes'; export enum HostsQueries { authentications = 'authentications', + authenticationsEntities = 'authenticationsEntities', details = 'hostDetails', firstOrLastSeen = 'firstOrLastSeen', hosts = 'hosts', + hostsEntities = 'hostsEntities', overview = 'overviewHost', uncommonProcesses = 'uncommonProcesses', } diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/hosts/kpi/authentications/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/hosts/kpi/authentications/index.ts index 4eb8af02af355..81e1945dcd010 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/hosts/kpi/authentications/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/hosts/kpi/authentications/index.ts @@ -12,6 +12,8 @@ import { HostsKpiHistogramData } from '../common'; export interface HostsKpiAuthenticationsHistogramCount { doc_count: number; + // TODO: Should I keep this or split this interface into two for entities and non-entities? + value?: number; } export type HostsKpiAuthenticationsRequestOptions = RequestBasicOptions; diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/hosts/kpi/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/hosts/kpi/index.ts index 79054fc736a80..d48172bebee4c 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/hosts/kpi/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/hosts/kpi/index.ts @@ -16,8 +16,11 @@ import { HostsKpiUniqueIpsStrategyResponse } from './unique_ips'; export enum HostsKpiQueries { kpiAuthentications = 'hostsKpiAuthentications', + kpiAuthenticationsEntities = 'hostsKpiAuthenticationsEntities', kpiHosts = 'hostsKpiHosts', + kpiHostsEntities = 'hostsKpiHostsEntities', kpiUniqueIps = 'hostsKpiUniqueIps', + kpiUniqueIpsEntities = 'hostsKpiUniqueIpsEntities', } export type HostsKpiStrategyResponse = diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts index ae2cff20717f3..936d9c360afb0 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts @@ -61,6 +61,7 @@ import { } from './network'; import { MatrixHistogramQuery, + MatrixHistogramQueryEntities, MatrixHistogramRequestOptions, MatrixHistogramStrategyResponse, } from './matrix_histogram'; @@ -75,7 +76,8 @@ export type FactoryQueryTypes = | HostsKpiQueries | NetworkQueries | NetworkKpiQueries - | typeof MatrixHistogramQuery; + | typeof MatrixHistogramQuery + | typeof MatrixHistogramQueryEntities; export interface RequestBasicOptions extends IEsSearchRequest { timerange: TimerangeInput; diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/index.ts index 81edb51e41458..fd1cf32e21400 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/index.ts @@ -23,9 +23,11 @@ export * from './dns'; export * from './events'; export const MatrixHistogramQuery = 'matrixHistogram'; +export const MatrixHistogramQueryEntities = 'matrixHistogramEntities'; export enum MatrixHistogramType { authentications = 'authentications', + authenticationsEntities = 'authenticationsEntities', anomalies = 'anomalies', events = 'events', alerts = 'alerts', diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/index.ts index 24c6484f94e71..2e0a5d7d2f0f1 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/index.ts @@ -23,6 +23,8 @@ export enum NetworkQueries { overview = 'overviewNetwork', tls = 'tls', topCountries = 'topCountries', + topCountriesEntities = 'topCountriesEntities', topNFlow = 'topNFlow', + topNFlowEntities = 'topNFlowEntities', users = 'users', } diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/kpi/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/kpi/index.ts index fa9e55096f7a6..cb18a3edb4937 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/kpi/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/kpi/index.ts @@ -19,10 +19,14 @@ import { NetworkKpiUniquePrivateIpsStrategyResponse } from './unique_private_ips export enum NetworkKpiQueries { dns = 'networkKpiDns', + dnsEntities = 'networkKpiDnsEntities', networkEvents = 'networkKpiNetworkEvents', + networkEventsEntities = 'networkKpiNetworkEventsEntities', tlsHandshakes = 'networkKpiTlsHandshakes', + tlsHandshakesEntities = 'networkKpiTlsHandshakesEntities', uniqueFlows = 'networkKpiUniqueFlows', uniquePrivateIps = 'networkKpiUniquePrivateIps', + uniquePrivateIpsEntities = 'networkKpiUniquePrivateIpsEntities', } export type NetworkKpiStrategyResponse = diff --git a/x-pack/plugins/security_solution/common/transforms/types.ts b/x-pack/plugins/security_solution/common/transforms/types.ts new file mode 100644 index 0000000000000..ac4e3cae92e22 --- /dev/null +++ b/x-pack/plugins/security_solution/common/transforms/types.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema, TypeOf } from '@kbn/config-schema'; + +/** + * Kibana configuration schema + */ +export const transformConfigSchema = schema.object({ + auto_start: schema.boolean(), + auto_create: schema.boolean(), + enabled: schema.boolean(), + query: schema.maybe(schema.object({}, { unknowns: 'allow' })), + retention_policy: schema.maybe( + schema.object({ + time: schema.object({ + field: schema.string(), + max_age: schema.string(), + }), + }) + ), + docs_per_second: schema.maybe(schema.number({ min: 1 })), + max_page_search_size: schema.maybe(schema.number({ min: 1, max: 10000 })), + settings: schema.arrayOf( + schema.object({ + prefix: schema.string(), + indices: schema.arrayOf(schema.string()), + data_sources: schema.arrayOf(schema.arrayOf(schema.string())), + disable_widgets: schema.maybe(schema.arrayOf(schema.string())), + disable_transforms: schema.maybe(schema.arrayOf(schema.string())), + }) + ), +}); + +export type TransformConfigSchema = TypeOf; diff --git a/x-pack/plugins/security_solution/public/common/containers/matrix_histogram/index.test.ts b/x-pack/plugins/security_solution/public/common/containers/matrix_histogram/index.test.ts index 05fce515ca3c7..045ffc6d26b4b 100644 --- a/x-pack/plugins/security_solution/public/common/containers/matrix_histogram/index.test.ts +++ b/x-pack/plugins/security_solution/public/common/containers/matrix_histogram/index.test.ts @@ -10,6 +10,7 @@ import { renderHook, act } from '@testing-library/react-hooks'; import { useKibana } from '../../../common/lib/kibana'; import { useMatrixHistogram } from '.'; import { MatrixHistogramType } from '../../../../common/search_strategy'; +import { TestProviders } from '../../mock/test_providers'; jest.mock('../../../common/lib/kibana'); @@ -26,7 +27,9 @@ describe('useMatrixHistogram', () => { it('should update request when props has changed', async () => { const localProps = { ...props }; - const { rerender } = renderHook(() => useMatrixHistogram(localProps)); + const { rerender } = renderHook(() => useMatrixHistogram(localProps), { + wrapper: TestProviders, + }); localProps.stackByField = 'event.action'; @@ -40,7 +43,9 @@ describe('useMatrixHistogram', () => { }); it('returns a memoized value', async () => { - const { result, rerender } = renderHook(() => useMatrixHistogram(props)); + const { result, rerender } = renderHook(() => useMatrixHistogram(props), { + wrapper: TestProviders, + }); const result1 = result.current[1]; act(() => rerender()); diff --git a/x-pack/plugins/security_solution/public/common/containers/matrix_histogram/index.ts b/x-pack/plugins/security_solution/public/common/containers/matrix_histogram/index.ts index 19c706b86577d..edc2d1e233c6e 100644 --- a/x-pack/plugins/security_solution/public/common/containers/matrix_histogram/index.ts +++ b/x-pack/plugins/security_solution/public/common/containers/matrix_histogram/index.ts @@ -24,6 +24,7 @@ import { isErrorResponse, isCompleteResponse } from '../../../../../../../src/pl import { getInspectResponse } from '../../../helpers'; import { InspectResponse } from '../../../types'; import * as i18n from './translations'; +import { useTransforms } from '../../../transforms/containers/use_transforms'; import { useAppToasts } from '../../hooks/use_app_toasts'; export type Buckets = Array<{ @@ -66,19 +67,34 @@ export const useMatrixHistogram = ({ const abortCtrl = useRef(new AbortController()); const searchSubscription$ = useRef(new Subscription()); const [loading, setLoading] = useState(false); - const [ - matrixHistogramRequest, - setMatrixHistogramRequest, - ] = useState({ - defaultIndex: indexNames, - factoryQueryType: MatrixHistogramQuery, - filterQuery: createFilter(filterQuery), + const { getTransformChangesIfTheyExist } = useTransforms(); + + const { + indices: initialIndexName, + factoryQueryType: initialFactoryQueryType, + histogramType: initialHistogramType, + timerange: initialTimerange, + } = getTransformChangesIfTheyExist({ histogramType, + factoryQueryType: MatrixHistogramQuery, + indices: indexNames, + filterQuery, timerange: { interval: '12h', from: startDate, to: endDate, }, + }); + + const [ + matrixHistogramRequest, + setMatrixHistogramRequest, + ] = useState({ + defaultIndex: initialIndexName, + factoryQueryType: initialFactoryQueryType, + filterQuery: createFilter(filterQuery), + histogramType: initialHistogramType ?? histogramType, + timerange: initialTimerange, stackByField, threshold, ...(isPtrIncluded != null ? { isPtrIncluded } : {}), @@ -150,17 +166,31 @@ export const useMatrixHistogram = ({ ); useEffect(() => { + const { + indices, + factoryQueryType, + histogramType: newHistogramType, + timerange, + } = getTransformChangesIfTheyExist({ + histogramType, + factoryQueryType: MatrixHistogramQuery, + indices: indexNames, + filterQuery, + timerange: { + interval: '12h', + from: startDate, + to: endDate, + }, + }); + setMatrixHistogramRequest((prevRequest) => { const myRequest = { ...prevRequest, - defaultIndex: indexNames, + defaultIndex: indices, + factoryQueryType, filterQuery: createFilter(filterQuery), - histogramType, - timerange: { - interval: '12h', - from: startDate, - to: endDate, - }, + histogramType: newHistogramType ?? histogramType, + timerange, stackByField, threshold, ...(isPtrIncluded != null ? { isPtrIncluded } : {}), @@ -181,6 +211,7 @@ export const useMatrixHistogram = ({ threshold, isPtrIncluded, docValueFields, + getTransformChangesIfTheyExist, ]); useEffect(() => { diff --git a/x-pack/plugins/security_solution/public/common/lib/kibana/kibana_react.mock.ts b/x-pack/plugins/security_solution/public/common/lib/kibana/kibana_react.mock.ts index 1527ea7dccac5..b7283eadd722d 100644 --- a/x-pack/plugins/security_solution/public/common/lib/kibana/kibana_react.mock.ts +++ b/x-pack/plugins/security_solution/public/common/lib/kibana/kibana_react.mock.ts @@ -33,6 +33,7 @@ import { DEFAULT_RULE_REFRESH_INTERVAL_ON, DEFAULT_RULE_REFRESH_INTERVAL_VALUE, DEFAULT_RULE_REFRESH_IDLE_VALUE, + DEFAULT_TRANSFORMS, } from '../../../../common/constants'; import { StartServices } from '../../../types'; import { createSecuritySolutionStorageMock } from '../../mock/mock_local_storage'; @@ -60,6 +61,9 @@ const mockUiSettings: Record = { value: DEFAULT_RULE_REFRESH_INTERVAL_VALUE, idleTimeout: DEFAULT_RULE_REFRESH_IDLE_VALUE, }, + [DEFAULT_TRANSFORMS]: { + enabled: false, + }, }; export const createUseUiSettingMock = () => (key: string, defaultValue?: unknown): unknown => { diff --git a/x-pack/plugins/security_solution/public/common/mock/global_state.ts b/x-pack/plugins/security_solution/public/common/mock/global_state.ts index d55c3c66e699f..b1b3147f4f494 100644 --- a/x-pack/plugins/security_solution/public/common/mock/global_state.ts +++ b/x-pack/plugins/security_solution/public/common/mock/global_state.ts @@ -42,6 +42,7 @@ export const mockGlobalState: State = { enableExperimental: { eventFilteringEnabled: false, trustedAppsByPolicyEnabled: false, + metricsEntitiesEnabled: false, hostIsolationEnabled: false, }, }, diff --git a/x-pack/plugins/security_solution/public/detections/components/user_info/index.test.tsx b/x-pack/plugins/security_solution/public/detections/components/user_info/index.test.tsx index 08612e0b6d00d..1c9069a9369df 100644 --- a/x-pack/plugins/security_solution/public/detections/components/user_info/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/user_info/index.test.tsx @@ -10,6 +10,8 @@ import { useUserInfo, ManageUserInfo } from './index'; import { useKibana } from '../../../common/lib/kibana'; import * as api from '../../containers/detection_engine/alerts/api'; +import { TestProviders } from '../../../common/mock/test_providers'; +import React from 'react'; jest.mock('../../../common/lib/kibana'); jest.mock('../../containers/detection_engine/alerts/api'); @@ -30,7 +32,9 @@ describe('useUserInfo', () => { }); it('returns default state', async () => { await act(async () => { - const { result, waitForNextUpdate } = renderHook(() => useUserInfo()); + const { result, waitForNextUpdate } = renderHook(() => useUserInfo(), { + wrapper: TestProviders, + }); await waitForNextUpdate(); expect(result.all).toHaveLength(1); @@ -57,8 +61,13 @@ describe('useUserInfo', () => { name: 'mock-signal-index', index_mapping_outdated: true, }); + const wrapper = ({ children }: { children: JSX.Element }) => ( + + {children} + + ); await act(async () => { - const { waitForNextUpdate } = renderHook(() => useUserInfo(), { wrapper: ManageUserInfo }); + const { waitForNextUpdate } = renderHook(() => useUserInfo(), { wrapper }); await waitForNextUpdate(); await waitForNextUpdate(); }); diff --git a/x-pack/plugins/security_solution/public/detections/components/user_info/index.tsx b/x-pack/plugins/security_solution/public/detections/components/user_info/index.tsx index 7fc90d49cc6b5..b196fd7998ea8 100644 --- a/x-pack/plugins/security_solution/public/detections/components/user_info/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/user_info/index.tsx @@ -11,6 +11,7 @@ import React, { useEffect, useReducer, Dispatch, createContext, useContext } fro import { usePrivilegeUser } from '../../containers/detection_engine/alerts/use_privilege_user'; import { useSignalIndex } from '../../containers/detection_engine/alerts/use_signal_index'; import { useKibana } from '../../../common/lib/kibana'; +import { useCreateTransforms } from '../../../transforms/containers/use_create_transforms'; export interface State { canUserCRUD: boolean | null; @@ -204,6 +205,8 @@ export const useUserInfo = (): State => { createDeSignalIndex: createSignalIndex, } = useSignalIndex(); + const { createTransforms } = useCreateTransforms(); + const uiCapabilities = useKibana().services.application.capabilities; const capabilitiesCanUserCRUD: boolean = typeof uiCapabilities.siem.crud === 'boolean' ? uiCapabilities.siem.crud : false; @@ -296,6 +299,13 @@ export const useUserInfo = (): State => { } }, [dispatch, loading, signalIndexMappingOutdated, apiSignalIndexMappingOutdated]); + // TODO: Get the permissions model and if the user has the correct permissions for transforms + // then activate the transforms similar to the createSignalIndex. + // TODO: This should move out of detections/components and into its own transform area + useEffect(() => { + createTransforms(); + }, [createTransforms]); + useEffect(() => { if ( isAuthenticated && diff --git a/x-pack/plugins/security_solution/public/hosts/containers/authentications/index.tsx b/x-pack/plugins/security_solution/public/hosts/containers/authentications/index.tsx index c31094b5778d5..f60e0b461d055 100644 --- a/x-pack/plugins/security_solution/public/hosts/containers/authentications/index.tsx +++ b/x-pack/plugins/security_solution/public/hosts/containers/authentications/index.tsx @@ -33,6 +33,7 @@ import { InspectResponse } from '../../../types'; import { hostsModel, hostsSelectors } from '../../store'; import * as i18n from './translations'; +import { useTransforms } from '../../../transforms/containers/use_transforms'; import { useAppToasts } from '../../../common/hooks/use_app_toasts'; const ID = 'hostsAuthenticationsQuery'; @@ -81,6 +82,7 @@ export const useAuthentications = ({ authenticationsRequest, setAuthenticationsRequest, ] = useState(null); + const { getTransformChangesIfTheyExist } = useTransforms(); const { addError, addWarning } = useAppToasts(); const wrappedLoadMore = useCallback( @@ -170,18 +172,25 @@ export const useAuthentications = ({ useEffect(() => { setAuthenticationsRequest((prevRequest) => { - const myRequest = { - ...(prevRequest ?? {}), - defaultIndex: indexNames, - docValueFields: docValueFields ?? [], + const { indices, factoryQueryType, timerange } = getTransformChangesIfTheyExist({ factoryQueryType: HostsQueries.authentications, - filterQuery: createFilter(filterQuery), - pagination: generateTablePaginationOptions(activePage, limit), + indices: indexNames, + filterQuery, timerange: { interval: '12h', from: startDate, to: endDate, }, + }); + + const myRequest = { + ...(prevRequest ?? {}), + defaultIndex: indices, + docValueFields: docValueFields ?? [], + factoryQueryType, + filterQuery: createFilter(filterQuery), + pagination: generateTablePaginationOptions(activePage, limit), + timerange, sort: {} as SortField, }; if (!deepEqual(prevRequest, myRequest)) { @@ -189,7 +198,16 @@ export const useAuthentications = ({ } return prevRequest; }); - }, [activePage, docValueFields, endDate, filterQuery, indexNames, limit, startDate]); + }, [ + activePage, + docValueFields, + endDate, + filterQuery, + indexNames, + limit, + startDate, + getTransformChangesIfTheyExist, + ]); useEffect(() => { authenticationsSearch(authenticationsRequest); diff --git a/x-pack/plugins/security_solution/public/hosts/containers/hosts/index.tsx b/x-pack/plugins/security_solution/public/hosts/containers/hosts/index.tsx index 7bf681092c075..6244427b45d11 100644 --- a/x-pack/plugins/security_solution/public/hosts/containers/hosts/index.tsx +++ b/x-pack/plugins/security_solution/public/hosts/containers/hosts/index.tsx @@ -30,6 +30,7 @@ import * as i18n from './translations'; import { isCompleteResponse, isErrorResponse } from '../../../../../../../src/plugins/data/common'; import { getInspectResponse } from '../../../helpers'; import { InspectResponse } from '../../../types'; +import { useTransforms } from '../../../transforms/containers/use_transforms'; import { useAppToasts } from '../../../common/hooks/use_app_toasts'; const ID = 'hostsAllQuery'; @@ -77,6 +78,7 @@ export const useAllHost = ({ const searchSubscription = useRef(new Subscription()); const [loading, setLoading] = useState(false); const [hostsRequest, setHostRequest] = useState(null); + const { getTransformChangesIfTheyExist } = useTransforms(); const { addError, addWarning } = useAppToasts(); const wrappedLoadMore = useCallback( @@ -166,18 +168,24 @@ export const useAllHost = ({ useEffect(() => { setHostRequest((prevRequest) => { - const myRequest = { - ...(prevRequest ?? {}), - defaultIndex: indexNames, - docValueFields: docValueFields ?? [], + const { indices, factoryQueryType, timerange } = getTransformChangesIfTheyExist({ factoryQueryType: HostsQueries.hosts, - filterQuery: createFilter(filterQuery), - pagination: generateTablePaginationOptions(activePage, limit), + indices: indexNames, + filterQuery, timerange: { interval: '12h', from: startDate, to: endDate, }, + }); + const myRequest = { + ...(prevRequest ?? {}), + defaultIndex: indices, + docValueFields: docValueFields ?? [], + factoryQueryType, + filterQuery: createFilter(filterQuery), + pagination: generateTablePaginationOptions(activePage, limit), + timerange, sort: { direction, field: sortField, @@ -198,6 +206,7 @@ export const useAllHost = ({ limit, startDate, sortField, + getTransformChangesIfTheyExist, ]); useEffect(() => { diff --git a/x-pack/plugins/security_solution/public/hosts/containers/kpi_hosts/authentications/index.tsx b/x-pack/plugins/security_solution/public/hosts/containers/kpi_hosts/authentications/index.tsx index 6a3323da4fb44..77196ae15646b 100644 --- a/x-pack/plugins/security_solution/public/hosts/containers/kpi_hosts/authentications/index.tsx +++ b/x-pack/plugins/security_solution/public/hosts/containers/kpi_hosts/authentications/index.tsx @@ -10,6 +10,7 @@ import { noop } from 'lodash/fp'; import { useCallback, useEffect, useRef, useState } from 'react'; import { Subscription } from 'rxjs'; +import { useTransforms } from '../../../../transforms/containers/use_transforms'; import { useAppToasts } from '../../../../common/hooks/use_app_toasts'; import { inputsModel } from '../../../../common/store'; import { createFilter } from '../../../../common/containers/helpers'; @@ -59,6 +60,7 @@ export const useHostsKpiAuthentications = ({ hostsKpiAuthenticationsRequest, setHostsKpiAuthenticationsRequest, ] = useState(null); + const { getTransformChangesIfTheyExist } = useTransforms(); const [ hostsKpiAuthenticationsResponse, @@ -134,24 +136,30 @@ export const useHostsKpiAuthentications = ({ ); useEffect(() => { + const { indices, factoryQueryType, timerange } = getTransformChangesIfTheyExist({ + factoryQueryType: HostsKpiQueries.kpiAuthentications, + indices: indexNames, + filterQuery, + timerange: { + interval: '12h', + from: startDate, + to: endDate, + }, + }); setHostsKpiAuthenticationsRequest((prevRequest) => { const myRequest = { ...(prevRequest ?? {}), - defaultIndex: indexNames, - factoryQueryType: HostsKpiQueries.kpiAuthentications, + defaultIndex: indices, + factoryQueryType, filterQuery: createFilter(filterQuery), - timerange: { - interval: '12h', - from: startDate, - to: endDate, - }, + timerange, }; if (!deepEqual(prevRequest, myRequest)) { return myRequest; } return prevRequest; }); - }, [indexNames, endDate, filterQuery, startDate]); + }, [indexNames, endDate, filterQuery, startDate, getTransformChangesIfTheyExist]); useEffect(() => { hostsKpiAuthenticationsSearch(hostsKpiAuthenticationsRequest); diff --git a/x-pack/plugins/security_solution/public/hosts/containers/kpi_hosts/hosts/index.tsx b/x-pack/plugins/security_solution/public/hosts/containers/kpi_hosts/hosts/index.tsx index 5af91539e8be3..3a478962559c2 100644 --- a/x-pack/plugins/security_solution/public/hosts/containers/kpi_hosts/hosts/index.tsx +++ b/x-pack/plugins/security_solution/public/hosts/containers/kpi_hosts/hosts/index.tsx @@ -10,6 +10,7 @@ import { noop } from 'lodash/fp'; import { useCallback, useEffect, useRef, useState } from 'react'; import { Subscription } from 'rxjs'; +import { useTransforms } from '../../../../transforms/containers/use_transforms'; import { useAppToasts } from '../../../../common/hooks/use_app_toasts'; import { inputsModel } from '../../../../common/store'; import { createFilter } from '../../../../common/containers/helpers'; @@ -58,6 +59,7 @@ export const useHostsKpiHosts = ({ hostsKpiHostsRequest, setHostsKpiHostsRequest, ] = useState(null); + const { getTransformChangesIfTheyExist } = useTransforms(); const [hostsKpiHostsResponse, setHostsKpiHostsResponse] = useState({ hosts: 0, @@ -123,23 +125,29 @@ export const useHostsKpiHosts = ({ useEffect(() => { setHostsKpiHostsRequest((prevRequest) => { - const myRequest = { - ...(prevRequest ?? {}), - defaultIndex: indexNames, + const { indices, factoryQueryType, timerange } = getTransformChangesIfTheyExist({ factoryQueryType: HostsKpiQueries.kpiHosts, - filterQuery: createFilter(filterQuery), + indices: indexNames, + filterQuery, timerange: { interval: '12h', from: startDate, to: endDate, }, + }); + const myRequest = { + ...(prevRequest ?? {}), + defaultIndex: indices, + factoryQueryType, + filterQuery: createFilter(filterQuery), + timerange, }; if (!deepEqual(prevRequest, myRequest)) { return myRequest; } return prevRequest; }); - }, [indexNames, endDate, filterQuery, startDate]); + }, [indexNames, endDate, filterQuery, startDate, getTransformChangesIfTheyExist]); useEffect(() => { hostsKpiHostsSearch(hostsKpiHostsRequest); diff --git a/x-pack/plugins/security_solution/public/hosts/containers/kpi_hosts/unique_ips/index.tsx b/x-pack/plugins/security_solution/public/hosts/containers/kpi_hosts/unique_ips/index.tsx index 9a72fa1d6cfca..d7f5469cf3117 100644 --- a/x-pack/plugins/security_solution/public/hosts/containers/kpi_hosts/unique_ips/index.tsx +++ b/x-pack/plugins/security_solution/public/hosts/containers/kpi_hosts/unique_ips/index.tsx @@ -10,6 +10,7 @@ import { noop } from 'lodash/fp'; import { useCallback, useEffect, useRef, useState } from 'react'; import { Subscription } from 'rxjs'; +import { useTransforms } from '../../../../transforms/containers/use_transforms'; import { useAppToasts } from '../../../../common/hooks/use_app_toasts'; import { inputsModel } from '../../../../common/store'; import { createFilter } from '../../../../common/containers/helpers'; @@ -55,6 +56,8 @@ export const useHostsKpiUniqueIps = ({ const abortCtrl = useRef(new AbortController()); const searchSubscription$ = useRef(new Subscription()); const [loading, setLoading] = useState(false); + const { getTransformChangesIfTheyExist } = useTransforms(); + const [ hostsKpiUniqueIpsRequest, setHostsKpiUniqueIpsRequest, @@ -129,24 +132,30 @@ export const useHostsKpiUniqueIps = ({ ); useEffect(() => { + const { indices, factoryQueryType, timerange } = getTransformChangesIfTheyExist({ + factoryQueryType: HostsKpiQueries.kpiUniqueIps, + indices: indexNames, + filterQuery, + timerange: { + interval: '12h', + from: startDate, + to: endDate, + }, + }); setHostsKpiUniqueIpsRequest((prevRequest) => { const myRequest = { ...(prevRequest ?? {}), - defaultIndex: indexNames, - factoryQueryType: HostsKpiQueries.kpiUniqueIps, + defaultIndex: indices, + factoryQueryType, filterQuery: createFilter(filterQuery), - timerange: { - interval: '12h', - from: startDate, - to: endDate, - }, + timerange, }; if (!deepEqual(prevRequest, myRequest)) { return myRequest; } return prevRequest; }); - }, [indexNames, endDate, filterQuery, skip, startDate]); + }, [indexNames, endDate, filterQuery, skip, startDate, getTransformChangesIfTheyExist]); useEffect(() => { hostsKpiUniqueIpsSearch(hostsKpiUniqueIpsRequest); diff --git a/x-pack/plugins/security_solution/public/network/containers/kpi_network/dns/index.tsx b/x-pack/plugins/security_solution/public/network/containers/kpi_network/dns/index.tsx index c835aa6c6a3e3..375b77d11d70c 100644 --- a/x-pack/plugins/security_solution/public/network/containers/kpi_network/dns/index.tsx +++ b/x-pack/plugins/security_solution/public/network/containers/kpi_network/dns/index.tsx @@ -10,6 +10,7 @@ import { noop } from 'lodash/fp'; import { useCallback, useEffect, useRef, useState } from 'react'; import { Subscription } from 'rxjs'; +import { useTransforms } from '../../../../transforms/containers/use_transforms'; import { useAppToasts } from '../../../../common/hooks/use_app_toasts'; import { inputsModel } from '../../../../common/store'; import { createFilter } from '../../../../common/containers/helpers'; @@ -63,6 +64,7 @@ export const useNetworkKpiDns = ({ networkKpiDnsRequest, setNetworkKpiDnsRequest, ] = useState(null); + const { getTransformChangesIfTheyExist } = useTransforms(); const [networkKpiDnsResponse, setNetworkKpiDnsResponse] = useState({ dnsQueries: 0, @@ -127,23 +129,30 @@ export const useNetworkKpiDns = ({ useEffect(() => { setNetworkKpiDnsRequest((prevRequest) => { - const myRequest = { - ...(prevRequest ?? {}), - defaultIndex: indexNames, + const { indices, factoryQueryType, timerange } = getTransformChangesIfTheyExist({ factoryQueryType: NetworkKpiQueries.dns, - filterQuery: createFilter(filterQuery), + indices: indexNames, + filterQuery, timerange: { interval: '12h', from: startDate, to: endDate, }, + }); + + const myRequest = { + ...(prevRequest ?? {}), + defaultIndex: indices, + factoryQueryType, + filterQuery: createFilter(filterQuery), + timerange, }; if (!deepEqual(prevRequest, myRequest)) { return myRequest; } return prevRequest; }); - }, [indexNames, endDate, filterQuery, startDate]); + }, [indexNames, endDate, filterQuery, startDate, getTransformChangesIfTheyExist]); useEffect(() => { networkKpiDnsSearch(networkKpiDnsRequest); diff --git a/x-pack/plugins/security_solution/public/network/containers/kpi_network/network_events/index.tsx b/x-pack/plugins/security_solution/public/network/containers/kpi_network/network_events/index.tsx index 2e4f3b83e6708..6b1f92a8dba19 100644 --- a/x-pack/plugins/security_solution/public/network/containers/kpi_network/network_events/index.tsx +++ b/x-pack/plugins/security_solution/public/network/containers/kpi_network/network_events/index.tsx @@ -10,6 +10,7 @@ import { noop } from 'lodash/fp'; import { useCallback, useEffect, useRef, useState } from 'react'; import { Subscription } from 'rxjs'; +import { useTransforms } from '../../../../transforms/containers/use_transforms'; import { useAppToasts } from '../../../../common/hooks/use_app_toasts'; import { inputsModel } from '../../../../common/store'; import { createFilter } from '../../../../common/containers/helpers'; @@ -63,6 +64,7 @@ export const useNetworkKpiNetworkEvents = ({ networkKpiNetworkEventsRequest, setNetworkKpiNetworkEventsRequest, ] = useState(null); + const { getTransformChangesIfTheyExist } = useTransforms(); const [ networkKpiNetworkEventsResponse, @@ -133,23 +135,29 @@ export const useNetworkKpiNetworkEvents = ({ useEffect(() => { setNetworkKpiNetworkEventsRequest((prevRequest) => { - const myRequest = { - ...(prevRequest ?? {}), - defaultIndex: indexNames, + const { indices, factoryQueryType, timerange } = getTransformChangesIfTheyExist({ factoryQueryType: NetworkKpiQueries.networkEvents, - filterQuery: createFilter(filterQuery), + indices: indexNames, + filterQuery, timerange: { interval: '12h', from: startDate, to: endDate, }, + }); + const myRequest = { + ...(prevRequest ?? {}), + defaultIndex: indices, + factoryQueryType, + filterQuery: createFilter(filterQuery), + timerange, }; if (!deepEqual(prevRequest, myRequest)) { return myRequest; } return prevRequest; }); - }, [indexNames, endDate, filterQuery, startDate]); + }, [indexNames, endDate, filterQuery, startDate, getTransformChangesIfTheyExist]); useEffect(() => { networkKpiNetworkEventsSearch(networkKpiNetworkEventsRequest); diff --git a/x-pack/plugins/security_solution/public/network/containers/kpi_network/tls_handshakes/index.tsx b/x-pack/plugins/security_solution/public/network/containers/kpi_network/tls_handshakes/index.tsx index b9d3e8639c560..84f108dad79f5 100644 --- a/x-pack/plugins/security_solution/public/network/containers/kpi_network/tls_handshakes/index.tsx +++ b/x-pack/plugins/security_solution/public/network/containers/kpi_network/tls_handshakes/index.tsx @@ -10,6 +10,7 @@ import { noop } from 'lodash/fp'; import { useCallback, useEffect, useRef, useState } from 'react'; import { Subscription } from 'rxjs'; +import { useTransforms } from '../../../../transforms/containers/use_transforms'; import { useAppToasts } from '../../../../common/hooks/use_app_toasts'; import { inputsModel } from '../../../../common/store'; import { createFilter } from '../../../../common/containers/helpers'; @@ -63,6 +64,7 @@ export const useNetworkKpiTlsHandshakes = ({ networkKpiTlsHandshakesRequest, setNetworkKpiTlsHandshakesRequest, ] = useState(null); + const { getTransformChangesIfTheyExist } = useTransforms(); const [ networkKpiTlsHandshakesResponse, @@ -132,23 +134,30 @@ export const useNetworkKpiTlsHandshakes = ({ useEffect(() => { setNetworkKpiTlsHandshakesRequest((prevRequest) => { - const myRequest = { - ...(prevRequest ?? {}), - defaultIndex: indexNames, + const { indices, factoryQueryType, timerange } = getTransformChangesIfTheyExist({ factoryQueryType: NetworkKpiQueries.tlsHandshakes, - filterQuery: createFilter(filterQuery), + indices: indexNames, + filterQuery, timerange: { interval: '12h', from: startDate, to: endDate, }, + }); + + const myRequest = { + ...(prevRequest ?? {}), + defaultIndex: indices, + factoryQueryType, + filterQuery: createFilter(filterQuery), + timerange, }; if (!deepEqual(prevRequest, myRequest)) { return myRequest; } return prevRequest; }); - }, [indexNames, endDate, filterQuery, startDate]); + }, [indexNames, endDate, filterQuery, startDate, getTransformChangesIfTheyExist]); useEffect(() => { networkKpiTlsHandshakesSearch(networkKpiTlsHandshakesRequest); diff --git a/x-pack/plugins/security_solution/public/network/containers/kpi_network/unique_private_ips/index.tsx b/x-pack/plugins/security_solution/public/network/containers/kpi_network/unique_private_ips/index.tsx index 488c526134525..b7c532e5867f0 100644 --- a/x-pack/plugins/security_solution/public/network/containers/kpi_network/unique_private_ips/index.tsx +++ b/x-pack/plugins/security_solution/public/network/containers/kpi_network/unique_private_ips/index.tsx @@ -10,6 +10,7 @@ import { noop } from 'lodash/fp'; import { useCallback, useEffect, useRef, useState } from 'react'; import { Subscription } from 'rxjs'; +import { useTransforms } from '../../../../transforms/containers/use_transforms'; import { useAppToasts } from '../../../../common/hooks/use_app_toasts'; import { inputsModel } from '../../../../common/store'; import { createFilter } from '../../../../common/containers/helpers'; @@ -67,6 +68,7 @@ export const useNetworkKpiUniquePrivateIps = ({ networkKpiUniquePrivateIpsRequest, setNetworkKpiUniquePrivateIpsRequest, ] = useState(null); + const { getTransformChangesIfTheyExist } = useTransforms(); const [ networkKpiUniquePrivateIpsResponse, @@ -144,23 +146,30 @@ export const useNetworkKpiUniquePrivateIps = ({ useEffect(() => { setNetworkKpiUniquePrivateIpsRequest((prevRequest) => { - const myRequest = { - ...(prevRequest ?? {}), - defaultIndex: indexNames, + const { indices, factoryQueryType, timerange } = getTransformChangesIfTheyExist({ factoryQueryType: NetworkKpiQueries.uniquePrivateIps, - filterQuery: createFilter(filterQuery), + indices: indexNames, + filterQuery, timerange: { interval: '12h', from: startDate, to: endDate, }, + }); + + const myRequest = { + ...(prevRequest ?? {}), + defaultIndex: indices, + factoryQueryType, + filterQuery: createFilter(filterQuery), + timerange, }; if (!deepEqual(prevRequest, myRequest)) { return myRequest; } return prevRequest; }); - }, [indexNames, endDate, filterQuery, startDate]); + }, [indexNames, endDate, filterQuery, startDate, getTransformChangesIfTheyExist]); useEffect(() => { networkKpiUniquePrivateIpsSearch(networkKpiUniquePrivateIpsRequest); diff --git a/x-pack/plugins/security_solution/public/network/containers/network_top_countries/index.tsx b/x-pack/plugins/security_solution/public/network/containers/network_top_countries/index.tsx index e7f3cf3f2675a..053dca60a740e 100644 --- a/x-pack/plugins/security_solution/public/network/containers/network_top_countries/index.tsx +++ b/x-pack/plugins/security_solution/public/network/containers/network_top_countries/index.tsx @@ -29,6 +29,7 @@ import { isCompleteResponse, isErrorResponse } from '../../../../../../../src/pl import { getInspectResponse } from '../../../helpers'; import { InspectResponse } from '../../../types'; import * as i18n from './translations'; +import { useTransforms } from '../../../transforms/containers/use_transforms'; import { useAppToasts } from '../../../common/hooks/use_app_toasts'; const ID = 'networkTopCountriesQuery'; @@ -75,6 +76,7 @@ export const useNetworkTopCountries = ({ const searchSubscription$ = useRef(new Subscription()); const [loading, setLoading] = useState(false); const queryId = useMemo(() => `${ID}-${flowTarget}`, [flowTarget]); + const { getTransformChangesIfTheyExist } = useTransforms(); const [ networkTopCountriesRequest, @@ -172,27 +174,45 @@ export const useNetworkTopCountries = ({ useEffect(() => { setHostRequest((prevRequest) => { + const { indices, factoryQueryType, timerange } = getTransformChangesIfTheyExist({ + factoryQueryType: NetworkQueries.topCountries, + indices: indexNames, + filterQuery, + timerange: { + interval: '12h', + from: startDate, + to: endDate, + }, + }); + const myRequest = { ...(prevRequest ?? {}), - defaultIndex: indexNames, - factoryQueryType: NetworkQueries.topCountries, + defaultIndex: indices, + factoryQueryType, filterQuery: createFilter(filterQuery), flowTarget, ip, pagination: generateTablePaginationOptions(activePage, limit), sort, - timerange: { - interval: '12h', - from: startDate, - to: endDate, - }, + timerange, }; if (!deepEqual(prevRequest, myRequest)) { return myRequest; } return prevRequest; }); - }, [activePage, indexNames, endDate, filterQuery, ip, limit, startDate, sort, flowTarget]); + }, [ + activePage, + indexNames, + endDate, + filterQuery, + ip, + limit, + startDate, + sort, + flowTarget, + getTransformChangesIfTheyExist, + ]); useEffect(() => { networkTopCountriesSearch(networkTopCountriesRequest); diff --git a/x-pack/plugins/security_solution/public/network/containers/network_top_n_flow/index.tsx b/x-pack/plugins/security_solution/public/network/containers/network_top_n_flow/index.tsx index 3cbaf0fbc976c..8e9f64d19fe39 100644 --- a/x-pack/plugins/security_solution/public/network/containers/network_top_n_flow/index.tsx +++ b/x-pack/plugins/security_solution/public/network/containers/network_top_n_flow/index.tsx @@ -29,6 +29,7 @@ import { isCompleteResponse, isErrorResponse } from '../../../../../../../src/pl import { getInspectResponse } from '../../../helpers'; import { InspectResponse } from '../../../types'; import * as i18n from './translations'; +import { useTransforms } from '../../../transforms/containers/use_transforms'; import { useAppToasts } from '../../../common/hooks/use_app_toasts'; const ID = 'networkTopNFlowQuery'; @@ -74,6 +75,7 @@ export const useNetworkTopNFlow = ({ const abortCtrl = useRef(new AbortController()); const searchSubscription$ = useRef(new Subscription()); const [loading, setLoading] = useState(false); + const { getTransformChangesIfTheyExist } = useTransforms(); const [ networkTopNFlowRequest, @@ -168,19 +170,25 @@ export const useNetworkTopNFlow = ({ useEffect(() => { setTopNFlowRequest((prevRequest) => { - const myRequest = { - ...(prevRequest ?? {}), - defaultIndex: indexNames, + const { indices, factoryQueryType, timerange } = getTransformChangesIfTheyExist({ factoryQueryType: NetworkQueries.topNFlow, - filterQuery: createFilter(filterQuery), - flowTarget, - ip, - pagination: generateTablePaginationOptions(activePage, limit), + indices: indexNames, + filterQuery, timerange: { interval: '12h', from: startDate, to: endDate, }, + }); + const myRequest = { + ...(prevRequest ?? {}), + defaultIndex: indices, + factoryQueryType, + filterQuery: createFilter(filterQuery), + flowTarget, + ip, + pagination: generateTablePaginationOptions(activePage, limit), + timerange, sort, }; if (!deepEqual(prevRequest, myRequest)) { @@ -188,7 +196,18 @@ export const useNetworkTopNFlow = ({ } return prevRequest; }); - }, [activePage, endDate, filterQuery, indexNames, ip, limit, startDate, sort, flowTarget]); + }, [ + activePage, + endDate, + filterQuery, + indexNames, + ip, + limit, + startDate, + sort, + flowTarget, + getTransformChangesIfTheyExist, + ]); useEffect(() => { networkTopNFlowSearch(networkTopNFlowRequest); diff --git a/x-pack/plugins/security_solution/public/transforms/containers/api.ts b/x-pack/plugins/security_solution/public/transforms/containers/api.ts new file mode 100644 index 0000000000000..0a796286de1aa --- /dev/null +++ b/x-pack/plugins/security_solution/public/transforms/containers/api.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { KibanaServices } from '../../common/lib/kibana'; + +export interface CreateTransforms { + signal: AbortSignal; + // TODO: Stronger types from the metrics_entities project + bodies: unknown[]; +} + +export interface CreateTransform { + signal: AbortSignal; + // TODO: Stronger types from the metrics_entities project + body: unknown; +} + +/** + * Creates transforms given a configuration + * @param signal AbortSignal for cancelling request + * @param bodies The bodies for the REST interface that is going to create them one at a time. + * + * TODO: Once there is a _bulk API, then we can do these all at once + * @throws An error if response is not OK + */ +export const createTransforms = async ({ bodies, signal }: CreateTransforms): Promise => { + for (const body of bodies) { + await createTransform({ body, signal }); + } +}; + +/** + * Creates a single transform given a configuration + * @param signal AbortSignal for cancelling request + * @param bodies The body for the REST interface that is going to it. + * @throws An error if response is not OK + */ +export const createTransform = async ({ body, signal }: CreateTransform): Promise => { + // TODO: Use constants for the url here or from the metrics package. + return KibanaServices.get().http.fetch('/api/metrics_entities/transforms', { + method: 'POST', + body: JSON.stringify(body), + signal, + }); +}; diff --git a/x-pack/plugins/security_solution/public/transforms/containers/translations.ts b/x-pack/plugins/security_solution/public/transforms/containers/translations.ts new file mode 100644 index 0000000000000..2fdd285dddd85 --- /dev/null +++ b/x-pack/plugins/security_solution/public/transforms/containers/translations.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; + +export const TRANSFORM_CREATE_FAILURE = i18n.translate( + 'xpack.securitySolution.containers.transforms.errorCreatingTransformsLabel', + { + defaultMessage: 'Failed to create transforms', + } +); diff --git a/x-pack/plugins/security_solution/public/transforms/containers/use_create_transforms.ts b/x-pack/plugins/security_solution/public/transforms/containers/use_create_transforms.ts new file mode 100644 index 0000000000000..ab38c4cca1860 --- /dev/null +++ b/x-pack/plugins/security_solution/public/transforms/containers/use_create_transforms.ts @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { useEffect, useState } from 'react'; +import { defaultTransformsSetting, DEFAULT_TRANSFORMS } from '../../../common/constants'; +import { TransformConfigSchema } from '../../../common/transforms/types'; +import { errorToToaster, useStateToaster } from '../../common/components/toasters'; +import * as i18n from './translations'; +import { createTransforms } from './api'; +import { useUiSetting$ } from '../../common/lib/kibana'; +import { useIsExperimentalFeatureEnabled } from '../../common/hooks/use_experimental_features'; + +type Func = () => Promise; + +export interface ReturnTransform { + loading: boolean; + createTransforms: Func; +} + +export const noop: Func = () => Promise.resolve(); + +export const useCreateTransforms = (): ReturnTransform => { + const [loading, setLoading] = useState(true); + const [, dispatchToaster] = useStateToaster(); + const [transformSettings] = useUiSetting$( + DEFAULT_TRANSFORMS, + (JSON.stringify(defaultTransformsSetting) as unknown) as TransformConfigSchema // TODO: The types are not 100% correct within uiSettings$, so I have to cast here. Once that is fixed, this cast can be removed + ); + const [transforms, setTransforms] = useState>({ + createTransforms: noop, + }); + // TODO: Once we are past experimental phase this code should be removed + const metricsEntitiesEnabled = useIsExperimentalFeatureEnabled('metricsEntitiesEnabled'); + + useEffect(() => { + let isSubscribed = true; + const abortCtrl = new AbortController(); + + const createTheTransforms = async () => { + // TODO: Once we are past experimental phase this code should be removed + if (!metricsEntitiesEnabled) { + return; + } + + // double check one more time and not create the transform if the settings are not enabled. + if (!transformSettings.enabled || !transformSettings.auto_create) { + return; + } + let isFetchingData = false; + setLoading(true); + const bodies = getTransformBodies(transformSettings); + try { + await createTransforms({ bodies, signal: abortCtrl.signal }); + if (isSubscribed) { + isFetchingData = true; + } + } catch (error) { + if (isSubscribed) { + if (error.body.statusCode !== 404 && error.body.status_code !== 404) { + errorToToaster({ title: i18n.TRANSFORM_CREATE_FAILURE, error, dispatchToaster }); + } else { + // This means that the plugin is disabled and/or the user does not have permissions + // so we do not show an error toaster for this condition since this is a 404 error message + } + } + } + if (isSubscribed && !isFetchingData) { + setLoading(false); + } + }; + + if (transformSettings.enabled) { + setTransforms({ createTransforms: createTheTransforms }); + } else { + setTransforms({ createTransforms: noop }); + } + return () => { + isSubscribed = false; + abortCtrl.abort(); + }; + }, [dispatchToaster, transformSettings, metricsEntitiesEnabled]); + + return { loading, ...transforms }; +}; + +export const getTransformBodies = (transformSettings: TransformConfigSchema) => { + // eslint-disable-next-line @typescript-eslint/naming-convention + const { query, auto_start, max_page_search_size, docs_per_second } = transformSettings; + return transformSettings.settings.map(({ prefix, indices }) => { + return { + query, + prefix, + modules: [ + 'host_metrics', + 'host_entities', + 'network_entities', + 'network_metrics', + 'user_entities', + 'user_metrics', + ], + indices, + auto_start, + settings: { + max_page_search_size, + docs_per_second, + }, + }; + }); +}; diff --git a/x-pack/plugins/security_solution/public/transforms/containers/use_transforms.ts b/x-pack/plugins/security_solution/public/transforms/containers/use_transforms.ts new file mode 100644 index 0000000000000..acb1ac6d1e77f --- /dev/null +++ b/x-pack/plugins/security_solution/public/transforms/containers/use_transforms.ts @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { useMemo, useState } from 'react'; +import { ESQuery } from '../../../common/typed_json'; +import { + FactoryQueryTypes, + MatrixHistogramType, + TimerangeInput, +} from '../../../common/search_strategy'; +import { TransformConfigSchema } from '../../../common/transforms/types'; +import { defaultTransformsSetting, DEFAULT_TRANSFORMS } from '../../../common/constants'; +import { useUiSetting$ } from '../../common/lib/kibana'; +import { getTransformChangesIfTheyExist } from '../utils'; +import { useIsExperimentalFeatureEnabled } from '../../common/hooks/use_experimental_features'; + +export type TransformChangesIfTheyExist = ({ + factoryQueryType, + indices, + filterQuery, + histogramType, + timerange, +}: { + factoryQueryType: FactoryQueryTypes; + indices: string[]; + filterQuery: ESQuery | string | undefined; + histogramType?: MatrixHistogramType; + timerange: TimerangeInput; +}) => { + indices: string[]; + factoryQueryType: FactoryQueryTypes; + histogramType?: MatrixHistogramType; + timerange: TimerangeInput; +}; + +export interface ReturnTransform { + getTransformChangesIfTheyExist: TransformChangesIfTheyExist; +} + +export const useTransforms = (): ReturnTransform => { + const [transformSettings] = useUiSetting$( + DEFAULT_TRANSFORMS, + (JSON.stringify(defaultTransformsSetting) as unknown) as TransformConfigSchema // TODO: The types are not 100% correct within uiSettings$, so I have to cast here. Once that is fixed, this cast can be removed + ); + // TODO: Once we are past experimental phase this code should be removed + const metricsEntitiesEnabled = useIsExperimentalFeatureEnabled('metricsEntitiesEnabled'); + const [transforms, setTransforms] = useState({ + getTransformChangesIfTheyExist: ({ + factoryQueryType, + indices, + filterQuery, + histogramType, + timerange, + }) => { + if (metricsEntitiesEnabled) { + return getTransformChangesIfTheyExist({ + factoryQueryType, + indices, + filterQuery, + transformSettings, + histogramType, + timerange, + }); + } else { + // TODO: Once the experimental flag is removed, then remove this return statement + return { + indices, + filterQuery, + timerange, + factoryQueryType, + }; + } + }, + }); + + useMemo(() => { + setTransforms({ + getTransformChangesIfTheyExist: ({ + factoryQueryType, + indices, + filterQuery, + histogramType, + timerange, + }) => { + if (metricsEntitiesEnabled) { + return getTransformChangesIfTheyExist({ + factoryQueryType, + indices, + transformSettings, + filterQuery, + histogramType, + timerange, + }); + } else { + // TODO: Once the experimental flag is removed, then remove this return statement + return { + indices, + filterQuery, + timerange, + factoryQueryType, + }; + } + }, + }); + }, [transformSettings, metricsEntitiesEnabled]); + + return { ...transforms }; +}; diff --git a/x-pack/plugins/security_solution/public/transforms/utils/adjust_timerange.ts b/x-pack/plugins/security_solution/public/transforms/utils/adjust_timerange.ts new file mode 100644 index 0000000000000..5c99524694a66 --- /dev/null +++ b/x-pack/plugins/security_solution/public/transforms/utils/adjust_timerange.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import dateMath from '@elastic/datemath'; +import moment, { Duration } from 'moment'; +import type { TimerangeInput } from '../../../common/search_strategy'; + +export type ParseTimeRange = ( + timeRange: TimerangeInput +) => { timeRangeAdjusted: TimerangeInput | undefined; duration: Duration | undefined }; + +export const adjustTimeRange: ParseTimeRange = (timerange) => { + const from = dateMath.parse(timerange.from); + const to = dateMath.parse(timerange.to); + if (from == null || to == null) { + return { timeRangeAdjusted: undefined, duration: undefined }; + } else { + const newTimerange: TimerangeInput = { + from: moment(from).startOf('hour').toISOString(), + to: timerange.to, + interval: timerange.interval, + }; + const duration = moment.duration(to.diff(from)); + return { timeRangeAdjusted: newTimerange, duration }; + } +}; diff --git a/x-pack/plugins/security_solution/public/transforms/utils/create_indices_from_prefix.ts b/x-pack/plugins/security_solution/public/transforms/utils/create_indices_from_prefix.ts new file mode 100644 index 0000000000000..1f3f3959f2aa3 --- /dev/null +++ b/x-pack/plugins/security_solution/public/transforms/utils/create_indices_from_prefix.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ELASTIC_NAME } from '../../../common/constants'; + +export const createIndicesFromPrefix = ({ + transformIndices, + prefix, +}: { + transformIndices: string[]; + prefix: string; +}): string[] => { + return transformIndices.map((index) => `.${ELASTIC_NAME}_${prefix}_${index}`); +}; diff --git a/x-pack/plugins/security_solution/public/transforms/utils/get_settings_match.ts b/x-pack/plugins/security_solution/public/transforms/utils/get_settings_match.ts new file mode 100644 index 0000000000000..1546dddb60ca9 --- /dev/null +++ b/x-pack/plugins/security_solution/public/transforms/utils/get_settings_match.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { TransformConfigSchema } from '../../../common/transforms/types'; + +export const getSettingsMatch = ({ + indices, + transformSettings, +}: { + indices: string[]; + transformSettings: TransformConfigSchema; +}): TransformConfigSchema['settings'][0] | undefined => { + const removeAllSubtractedIndices = indices.filter((index) => !index.startsWith('-')).sort(); + return transformSettings.settings.find((setting) => { + const match = setting.data_sources.some((dataSource) => { + return dataSource.sort().join() === removeAllSubtractedIndices.join(); + }); + if (match) { + return setting; + } else { + return undefined; + } + }); +}; diff --git a/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes.ts b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes.ts new file mode 100644 index 0000000000000..6e327457a683d --- /dev/null +++ b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getTransformChangesForHosts } from './get_transform_changes_for_hosts'; +import { getTransformChangesForKpi } from './get_transform_changes_for_kpi'; +import { getTransformChangesForMatrixHistogram } from './get_transform_changes_for_matrix_histogram'; +import { getTransformChangesForNetwork } from './get_transform_changes_for_network'; +import { GetTransformChanges } from './types'; + +export const getTransformChanges: GetTransformChanges = ({ + factoryQueryType, + settings, + histogramType, +}) => { + const kpiTransform = getTransformChangesForKpi({ factoryQueryType, settings }); + if (kpiTransform != null) { + return kpiTransform; + } + + const hostTransform = getTransformChangesForHosts({ factoryQueryType, settings }); + if (hostTransform != null) { + return hostTransform; + } + + const networkTransform = getTransformChangesForNetwork({ + factoryQueryType, + settings, + }); + if (networkTransform != null) { + return networkTransform; + } + + const matrixHistogram = getTransformChangesForMatrixHistogram({ + factoryQueryType, + settings, + histogramType, + }); + if (matrixHistogram != null) { + return matrixHistogram; + } + + // nothing matches + return undefined; +}; diff --git a/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_hosts.ts b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_hosts.ts new file mode 100644 index 0000000000000..fef884dd6761f --- /dev/null +++ b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_hosts.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { HostsQueries } from '../../../common/search_strategy'; +import { createIndicesFromPrefix } from './create_indices_from_prefix'; +import { GetTransformChanges } from './types'; + +export const getTransformChangesForHosts: GetTransformChanges = ({ + factoryQueryType, + settings, +}) => { + switch (factoryQueryType) { + case HostsQueries.hosts: { + return { + indices: createIndicesFromPrefix({ + prefix: settings.prefix, + transformIndices: ['host_ent*'], + }), + factoryQueryType: HostsQueries.hostsEntities, + }; + } + case HostsQueries.authentications: { + return { + indices: createIndicesFromPrefix({ + prefix: settings.prefix, + transformIndices: ['user_ent*'], + }), + factoryQueryType: HostsQueries.authenticationsEntities, + }; + } + default: { + return undefined; + } + } +}; diff --git a/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_kpi.ts b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_kpi.ts new file mode 100644 index 0000000000000..620e794dd0a77 --- /dev/null +++ b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_kpi.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { HostsKpiQueries } from '../../../common/search_strategy'; +import { createIndicesFromPrefix } from './create_indices_from_prefix'; +import { GetTransformChanges } from './types'; + +export const getTransformChangesForKpi: GetTransformChanges = ({ factoryQueryType, settings }) => { + switch (factoryQueryType) { + case HostsKpiQueries.kpiHosts: { + return { + indices: createIndicesFromPrefix({ + prefix: settings.prefix, + transformIndices: ['host_ent*'], + }), + factoryQueryType: HostsKpiQueries.kpiHostsEntities, + }; + } + case HostsKpiQueries.kpiAuthentications: { + return { + indices: createIndicesFromPrefix({ + prefix: settings.prefix, + transformIndices: ['user_ent*'], + }), + factoryQueryType: HostsKpiQueries.kpiAuthenticationsEntities, + }; + } + case HostsKpiQueries.kpiUniqueIps: { + return { + indices: createIndicesFromPrefix({ + prefix: settings.prefix, + transformIndices: ['src_ip_ent*', 'dest_ip_ent*'], + }), + factoryQueryType: HostsKpiQueries.kpiUniqueIpsEntities, + }; + } + default: { + return undefined; + } + } +}; diff --git a/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_matrix_histogram.ts b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_matrix_histogram.ts new file mode 100644 index 0000000000000..fca29b2e97d87 --- /dev/null +++ b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_matrix_histogram.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + MatrixHistogramQuery, + MatrixHistogramQueryEntities, + MatrixHistogramType, +} from '../../../common/search_strategy'; +import { createIndicesFromPrefix } from './create_indices_from_prefix'; +import { GetTransformChanges } from './types'; + +export const getTransformChangesForMatrixHistogram: GetTransformChanges = ({ + factoryQueryType, + settings, + histogramType, +}) => { + switch (factoryQueryType) { + case MatrixHistogramQuery: { + switch (histogramType) { + case MatrixHistogramType.authentications: { + return { + indices: createIndicesFromPrefix({ + prefix: settings.prefix, + transformIndices: ['user_met*'], + }), + factoryQueryType: MatrixHistogramQueryEntities, + histogramType: MatrixHistogramType.authenticationsEntities, + }; + } + default: { + return undefined; + } + } + } + default: { + return undefined; + } + } +}; diff --git a/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_network.ts b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_network.ts new file mode 100644 index 0000000000000..1b1ef815f82bc --- /dev/null +++ b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_network.ts @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { NetworkKpiQueries, NetworkQueries } from '../../../common/search_strategy'; +import { createIndicesFromPrefix } from './create_indices_from_prefix'; +import { GetTransformChanges } from './types'; + +export const getTransformChangesForNetwork: GetTransformChanges = ({ + factoryQueryType, + settings, +}) => { + switch (factoryQueryType) { + case NetworkQueries.topCountries: { + return { + indices: createIndicesFromPrefix({ + prefix: settings.prefix, + transformIndices: ['src_iso_ent*', 'dest_iso_ent*'], + }), + factoryQueryType: NetworkQueries.topCountriesEntities, + }; + } + case NetworkQueries.topNFlow: { + return { + indices: createIndicesFromPrefix({ + prefix: settings.prefix, + transformIndices: ['src_ip_ent*', 'dest_ip_ent*'], + }), + factoryQueryType: NetworkQueries.topNFlowEntities, + }; + } + case NetworkKpiQueries.dns: { + return { + indices: createIndicesFromPrefix({ + prefix: settings.prefix, + transformIndices: ['ip_met*'], + }), + factoryQueryType: NetworkKpiQueries.dnsEntities, + }; + } + case NetworkKpiQueries.networkEvents: { + return { + indices: createIndicesFromPrefix({ + prefix: settings.prefix, + transformIndices: ['ip_met*'], + }), + factoryQueryType: NetworkKpiQueries.networkEventsEntities, + }; + } + case NetworkKpiQueries.tlsHandshakes: { + return { + indices: createIndicesFromPrefix({ + prefix: settings.prefix, + transformIndices: ['ip_met*'], + }), + factoryQueryType: NetworkKpiQueries.tlsHandshakesEntities, + }; + } + case NetworkKpiQueries.uniquePrivateIps: { + return { + indices: createIndicesFromPrefix({ + prefix: settings.prefix, + transformIndices: ['src_ip_ent*', 'dest_ip_ent*'], + }), + factoryQueryType: NetworkKpiQueries.uniquePrivateIpsEntities, + }; + } + default: { + return undefined; + } + } +}; diff --git a/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_if_they_exist.ts b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_if_they_exist.ts new file mode 100644 index 0000000000000..20e0b6a31d2c3 --- /dev/null +++ b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_if_they_exist.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { adjustTimeRange } from './adjust_timerange'; +import { getSettingsMatch } from './get_settings_match'; +import { getTransformChanges } from './get_transform_changes'; +import { isFilterQueryCompatible } from './is_filter_query_compatible'; +import { GetTransformChangesIfTheyExist } from './types'; + +// TODO: Add the other switches here such as the disabling of a widget/factory type +// or if a transform is disabled, then this cannot use the query +export const getTransformChangesIfTheyExist: GetTransformChangesIfTheyExist = ({ + factoryQueryType, + indices, + transformSettings, + filterQuery, + histogramType, + timerange, +}) => { + if (!transformSettings.enabled) { + // Early return if we are not enabled + return { factoryQueryType, indices, timerange }; + } + + if (!isFilterQueryCompatible(filterQuery)) { + // Early return if the filter query is not compatible + return { factoryQueryType, indices, timerange }; + } + + const { timeRangeAdjusted, duration } = adjustTimeRange(timerange); + + if (timeRangeAdjusted == null || duration == null || duration.asHours() < 1) { + // Early return if we are less than hour of time or from is something not as we expect + // and as we should just use raw events instead of summaries + return { factoryQueryType, indices, timerange }; + } + + const settings = getSettingsMatch({ indices, transformSettings }); + if (settings == null) { + // early return if none of the settings match + return { factoryQueryType, indices, timerange }; + } + + const transform = getTransformChanges({ factoryQueryType, settings, histogramType }); + if (transform) { + return { ...transform, timerange: timeRangeAdjusted }; + } + + // nothing matched, return what was sent in unchanged + return { factoryQueryType, indices, timerange }; +}; diff --git a/x-pack/plugins/security_solution/public/transforms/utils/index.ts b/x-pack/plugins/security_solution/public/transforms/utils/index.ts new file mode 100644 index 0000000000000..8cacb0035204a --- /dev/null +++ b/x-pack/plugins/security_solution/public/transforms/utils/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export * from './adjust_timerange'; +export * from './create_indices_from_prefix'; +export * from './get_settings_match'; +export * from './get_transform_changes_for_hosts'; +export * from './get_transform_changes_for_kpi'; +export * from './get_transform_changes_for_matrix_histogram'; +export * from './get_transform_changes_for_network'; +export * from './get_transform_changes_if_they_exist'; +export * from './get_transform_changes'; +export * from './is_filter_query_compatible'; +export * from './types'; diff --git a/x-pack/plugins/security_solution/public/transforms/utils/is_filter_query_compatible.ts b/x-pack/plugins/security_solution/public/transforms/utils/is_filter_query_compatible.ts new file mode 100644 index 0000000000000..31c264bce3cd3 --- /dev/null +++ b/x-pack/plugins/security_solution/public/transforms/utils/is_filter_query_compatible.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ESQuery } from '../../../common/typed_json'; + +export const isFilterQueryCompatible = (filterQuery: ESQuery | string | undefined) => { + if (filterQuery === undefined) { + return true; + } else if (typeof filterQuery === 'string') { + return ( + filterQuery === '{"bool":{"must":[],"filter":[{"match_all":{}}],"should":[],"must_not":[]}}' + ); + } else { + // TODO: Can we check here and return if it matches a string or other signature? + return false; + } +}; diff --git a/x-pack/plugins/security_solution/public/transforms/utils/types.ts b/x-pack/plugins/security_solution/public/transforms/utils/types.ts new file mode 100644 index 0000000000000..96dc1ee228bd0 --- /dev/null +++ b/x-pack/plugins/security_solution/public/transforms/utils/types.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { TimerangeInput } from '../../../common/search_strategy/common'; +import { ESQuery } from '../../../common/typed_json'; +import { TransformConfigSchema } from '../../../common/transforms/types'; +import { + FactoryQueryTypes, + MatrixHistogramType, +} from '../../../common/search_strategy/security_solution'; + +export type GetTransformChanges = ({ + factoryQueryType, + settings, + histogramType, +}: { + factoryQueryType: FactoryQueryTypes; + settings: TransformConfigSchema['settings'][0]; + histogramType?: MatrixHistogramType; +}) => + | { + indices: string[]; + factoryQueryType: FactoryQueryTypes; + histogramType?: MatrixHistogramType; + } + | undefined; + +export type GetTransformChangesIfTheyExist = ({ + factoryQueryType, + indices, + filterQuery, + histogramType, + timerange, +}: { + factoryQueryType: FactoryQueryTypes; + indices: string[]; + transformSettings: TransformConfigSchema; + filterQuery: ESQuery | string | undefined; + histogramType?: MatrixHistogramType; + timerange: TimerangeInput; +}) => { + indices: string[]; + factoryQueryType: FactoryQueryTypes; + histogramType?: MatrixHistogramType; + timerange: TimerangeInput; +}; diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 8987c8be00cf0..3497041e776e5 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -162,13 +162,14 @@ export class Plugin implements IPlugin => Promise.resolve(config), - experimentalFeatures: parseExperimentalConfigValue(config.enableExperimental), + experimentalFeatures, }; initUsageCollectors({ diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/index.ts index 9df74a492c22a..7c318b24a7a35 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/index.ts @@ -20,6 +20,7 @@ import { inspectStringifyObject } from '../../../../../utils/build_query'; import { SecuritySolutionFactory } from '../../types'; import { buildHostsQuery } from './query.all_hosts.dsl'; import { formatHostEdgesData, HOSTS_FIELDS } from './helpers'; +import { buildHostsQueryEntities } from './query.all_hosts_entities.dsl'; export const allHosts: SecuritySolutionFactory = { buildDsl: (options: HostsRequestOptions) => { @@ -60,3 +61,43 @@ export const allHosts: SecuritySolutionFactory = { }; }, }; + +export const allHostsEntities: SecuritySolutionFactory = { + buildDsl: (options: HostsRequestOptions) => { + if (options.pagination && options.pagination.querySize >= DEFAULT_MAX_TABLE_QUERY_SIZE) { + throw new Error(`No query size above ${DEFAULT_MAX_TABLE_QUERY_SIZE}`); + } + return buildHostsQueryEntities(options); + }, + parse: async ( + options: HostsRequestOptions, + response: IEsSearchResponse + ): Promise => { + const { activePage, cursorStart, fakePossibleCount, querySize } = options.pagination; + const totalCount = getOr(0, 'aggregations.host_count.value', response.rawResponse); + const buckets: HostAggEsItem[] = getOr( + [], + 'aggregations.host_data.buckets', + response.rawResponse + ); + const hostsEdges = buckets.map((bucket) => formatHostEdgesData(HOSTS_FIELDS, bucket)); + const fakeTotalCount = fakePossibleCount <= totalCount ? fakePossibleCount : totalCount; + const edges = hostsEdges.splice(cursorStart, querySize - cursorStart); + const inspect = { + dsl: [inspectStringifyObject(buildHostsQueryEntities(options))], + }; + const showMorePagesIndicator = totalCount > fakeTotalCount; + + return { + ...response, + inspect, + edges, + totalCount, + pageInfo: { + activePage: activePage ?? 0, + fakeTotalCount, + showMorePagesIndicator, + }, + }; + }, +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/query.all_hosts_entities.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/query.all_hosts_entities.dsl.ts new file mode 100644 index 0000000000000..1c338998e3b65 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/query.all_hosts_entities.dsl.ts @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isEmpty } from 'lodash/fp'; +import { ISearchRequestParams } from '../../../../../../../../../src/plugins/data/common'; +import { + Direction, + HostsRequestOptions, + SortField, + HostsFields, +} from '../../../../../../common/search_strategy'; +import { createQueryFilterClauses } from '../../../../../utils/build_query'; +import { assertUnreachable } from '../../../../../../common/utility_types'; + +export const buildHostsQueryEntities = ({ + defaultIndex, + docValueFields, + filterQuery, + pagination: { querySize }, + sort, + timerange: { from, to }, +}: HostsRequestOptions): ISearchRequestParams => { + const filter = [ + ...createQueryFilterClauses(filterQuery), + { + range: { + '@timestamp': { + gte: from, + lte: to, + format: 'strict_date_optional_time', + }, + }, + }, + ]; + + const agg = { host_count: { cardinality: { field: 'host.name' } } }; + + const dslQuery = { + allowNoIndices: true, + index: defaultIndex, + ignoreUnavailable: true, + track_total_hits: false, + body: { + ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), + aggregations: { + ...agg, + host_data: { + terms: { size: querySize, field: 'host.name', order: getQueryOrder(sort) }, + aggs: { + lastSeen: { max: { field: '@timestamp' } }, + os: { + top_hits: { + size: 1, + sort: [ + { + '@timestamp': { + order: 'desc' as const, + }, + }, + ], + _source: { + includes: ['host.os.*'], + }, + }, + }, + }, + }, + }, + query: { bool: { filter } }, + size: 0, + }, + }; + + return dslQuery; +}; + +type QueryOrder = { lastSeen: Direction } | { _key: Direction }; + +const getQueryOrder = (sort: SortField): QueryOrder => { + switch (sort.field) { + case HostsFields.lastSeen: + return { lastSeen: sort.direction }; + case HostsFields.hostName: + return { _key: sort.direction }; + default: + return assertUnreachable(sort.field); + } +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/dsl/query_entities.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/dsl/query_entities.dsl.ts new file mode 100644 index 0000000000000..51d80152d65c0 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/dsl/query_entities.dsl.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isEmpty } from 'lodash/fp'; +import type { estypes } from '@elastic/elasticsearch'; + +import { HostAuthenticationsRequestOptions } from '../../../../../../../common/search_strategy/security_solution/hosts/authentications'; + +import { createQueryFilterClauses } from '../../../../../../utils/build_query'; + +export const buildQueryEntities = ({ + filterQuery, + timerange: { from, to }, + pagination: { querySize }, + defaultIndex, + docValueFields, +}: HostAuthenticationsRequestOptions) => { + const filter = [ + ...createQueryFilterClauses(filterQuery), + { + range: { + '@timestamp': { + gte: from, + lte: to, + format: 'strict_date_optional_time', + }, + }, + }, + ]; + + const agg = { + user_count: { + cardinality: { + field: 'user.name', + }, + }, + }; + + const dslQuery = { + allowNoIndices: true, + index: defaultIndex, + ignoreUnavailable: true, + body: { + ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), + aggregations: { + ...agg, + group_by_users: { + terms: { + size: querySize, + field: 'user.name', + order: [{ successes: 'desc' }, { failures: 'desc' }] as estypes.TermsAggregationOrder, + }, + aggs: { + failures: { + sum: { + field: 'metrics.event.authentication.failure.value_count', + }, + }, + successes: { + sum: { + field: 'metrics.event.authentication.success.value_count', + }, + }, + }, + }, + }, + query: { + bool: { + filter, + }, + }, + size: 0, + }, + track_total_hits: false, + }; + + return dslQuery; +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/helpers.ts index 807b78cb9cdd2..7517d112aebdc 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/helpers.ts @@ -97,3 +97,70 @@ export const getHits = (response: StrategyResponseT successes: bucket.successes.doc_count, }) ); + +export const getHitsEntities = (response: StrategyResponseType) => + getOr([], 'aggregations.group_by_users.buckets', response.rawResponse).map( + (bucket: AuthenticationBucket) => ({ + _id: getOr( + `${bucket.key}+${bucket.doc_count}`, + 'failures.lastFailure.hits.hits[0].id', + bucket + ), + _source: { + lastSuccess: getOr(null, 'successes.lastSuccess.hits.hits[0]._source', bucket), + lastFailure: getOr(null, 'failures.lastFailure.hits.hits[0]._source', bucket), + }, + user: bucket.key, + failures: bucket.failures.value, + successes: bucket.successes.value, + }) + ); + +export const formatAuthenticationEntitiesData = ( + fields: readonly string[] = authenticationsFields, + hit: AuthenticationHit, + fieldMap: Readonly> +): AuthenticationsEdges => { + return fields.reduce( + (flattenedFields, fieldName) => { + if (hit.cursor) { + flattenedFields.cursor.value = hit.cursor; + } + flattenedFields.node = { + ...flattenedFields.node, + ...{ + _id: hit._id, + user: { name: [hit.user] }, + failures: hit.failures, + successes: hit.successes, + }, + }; + const mergedResult = mergeFieldsWithHit(fieldName, flattenedFields, fieldMap, hit); + const fieldPath = `node.${fieldName}`; + const fieldValue = get(fieldPath, mergedResult); + if (!isEmpty(fieldValue)) { + return set( + fieldPath, + toObjectArrayOfStrings(fieldValue).map(({ str }) => str), + mergedResult + ); + } else { + return mergedResult; + } + }, + { + node: { + failures: 0, + successes: 0, + _id: '', + user: { + name: [''], + }, + }, + cursor: { + value: '', + tiebreaker: null, + }, + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/index.tsx b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/index.tsx index 437d9d09a9b85..9e85eefe21e8a 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/index.tsx +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/index.tsx @@ -21,7 +21,16 @@ import { import { inspectStringifyObject } from '../../../../../utils/build_query'; import { SecuritySolutionFactory } from '../../types'; import { auditdFieldsMap, buildQuery as buildAuthenticationQuery } from './dsl/query.dsl'; -import { authenticationsFields, formatAuthenticationData, getHits } from './helpers'; + +import { buildQueryEntities as buildAuthenticationQueryEntities } from './dsl/query_entities.dsl'; + +import { + authenticationsFields, + formatAuthenticationData, + formatAuthenticationEntitiesData, + getHits, + getHitsEntities, +} from './helpers'; export const authentications: SecuritySolutionFactory = { buildDsl: (options: HostAuthenticationsRequestOptions) => { @@ -63,3 +72,44 @@ export const authentications: SecuritySolutionFactory = { + buildDsl: (options: HostAuthenticationsRequestOptions) => { + if (options.pagination && options.pagination.querySize >= DEFAULT_MAX_TABLE_QUERY_SIZE) { + throw new Error(`No query size above ${DEFAULT_MAX_TABLE_QUERY_SIZE}`); + } + + return buildAuthenticationQueryEntities(options); + }, + parse: async ( + options: HostAuthenticationsRequestOptions, + response: IEsSearchResponse + ): Promise => { + const { activePage, cursorStart, fakePossibleCount, querySize } = options.pagination; + const totalCount = getOr(0, 'aggregations.user_count.value', response.rawResponse); + + const fakeTotalCount = fakePossibleCount <= totalCount ? fakePossibleCount : totalCount; + const hits: AuthenticationHit[] = getHitsEntities(response); + const authenticationEdges: AuthenticationsEdges[] = hits.map((hit) => + formatAuthenticationEntitiesData(authenticationsFields, hit, auditdFieldsMap) + ); + + const edges = authenticationEdges.splice(cursorStart, querySize - cursorStart); + const inspect = { + dsl: [inspectStringifyObject(buildAuthenticationQueryEntities(options))], + }; + const showMorePagesIndicator = totalCount > fakeTotalCount; + + return { + ...response, + inspect, + edges, + totalCount, + pageInfo: { + activePage: activePage ?? 0, + fakeTotalCount, + showMorePagesIndicator, + }, + }; + }, +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/index.test.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/index.test.ts index 5575b4fb487e7..fbe1ac6413bef 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/index.test.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/index.test.ts @@ -12,10 +12,10 @@ import { hostDetails } from './details'; import { hostOverview } from './overview'; import { firstOrLastSeenHost } from './last_first_seen'; import { uncommonProcesses } from './uncommon_processes'; -import { authentications } from './authentications'; -import { hostsKpiAuthentications } from './kpi/authentications'; -import { hostsKpiHosts } from './kpi/hosts'; -import { hostsKpiUniqueIps } from './kpi/unique_ips'; +import { authentications, authenticationsEntities } from './authentications'; +import { hostsKpiAuthentications, hostsKpiAuthenticationsEntities } from './kpi/authentications'; +import { hostsKpiHosts, hostsKpiHostsEntities } from './kpi/hosts'; +import { hostsKpiUniqueIps, hostsKpiUniqueIpsEntities } from './kpi/unique_ips'; jest.mock('./all'); jest.mock('./details'); @@ -36,8 +36,12 @@ describe('hostsFactory', () => { [HostsQueries.firstOrLastSeen]: firstOrLastSeenHost, [HostsQueries.uncommonProcesses]: uncommonProcesses, [HostsQueries.authentications]: authentications, + [HostsQueries.authenticationsEntities]: authenticationsEntities, [HostsKpiQueries.kpiAuthentications]: hostsKpiAuthentications, + [HostsKpiQueries.kpiAuthenticationsEntities]: hostsKpiAuthenticationsEntities, [HostsKpiQueries.kpiHosts]: hostsKpiHosts, + [HostsKpiQueries.kpiHostsEntities]: hostsKpiHostsEntities, + [HostsKpiQueries.kpiUniqueIpsEntities]: hostsKpiUniqueIpsEntities, [HostsKpiQueries.kpiUniqueIps]: hostsKpiUniqueIps, }; expect(hostsFactory).toEqual(expectedHostsFactory); diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/index.ts index 5cee547a6b365..cd95a38ec3092 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/index.ts @@ -12,15 +12,15 @@ import { } from '../../../../../common/search_strategy/security_solution'; import { SecuritySolutionFactory } from '../types'; -import { allHosts } from './all'; +import { allHosts, allHostsEntities } from './all'; import { hostDetails } from './details'; import { hostOverview } from './overview'; import { firstOrLastSeenHost } from './last_first_seen'; import { uncommonProcesses } from './uncommon_processes'; -import { authentications } from './authentications'; -import { hostsKpiAuthentications } from './kpi/authentications'; -import { hostsKpiHosts } from './kpi/hosts'; -import { hostsKpiUniqueIps } from './kpi/unique_ips'; +import { authentications, authenticationsEntities } from './authentications'; +import { hostsKpiAuthentications, hostsKpiAuthenticationsEntities } from './kpi/authentications'; +import { hostsKpiHosts, hostsKpiHostsEntities } from './kpi/hosts'; +import { hostsKpiUniqueIps, hostsKpiUniqueIpsEntities } from './kpi/unique_ips'; export const hostsFactory: Record< HostsQueries | HostsKpiQueries, @@ -28,11 +28,16 @@ export const hostsFactory: Record< > = { [HostsQueries.details]: hostDetails, [HostsQueries.hosts]: allHosts, + [HostsQueries.hostsEntities]: allHostsEntities, [HostsQueries.overview]: hostOverview, [HostsQueries.firstOrLastSeen]: firstOrLastSeenHost, [HostsQueries.uncommonProcesses]: uncommonProcesses, [HostsQueries.authentications]: authentications, + [HostsQueries.authenticationsEntities]: authenticationsEntities, [HostsKpiQueries.kpiAuthentications]: hostsKpiAuthentications, + [HostsKpiQueries.kpiAuthenticationsEntities]: hostsKpiAuthenticationsEntities, [HostsKpiQueries.kpiHosts]: hostsKpiHosts, + [HostsKpiQueries.kpiHostsEntities]: hostsKpiHostsEntities, [HostsKpiQueries.kpiUniqueIps]: hostsKpiUniqueIps, + [HostsKpiQueries.kpiUniqueIpsEntities]: hostsKpiUniqueIpsEntities, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/authentications/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/authentications/helpers.ts index 34b4a682d42de..6b81dd177bbf6 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/authentications/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/authentications/helpers.ts @@ -20,3 +20,13 @@ export const formatAuthenticationsHistogramData = ( y: count.doc_count, })) : null; + +export const formatAuthenticationsHistogramDataEntities = ( + data: Array> +): HostsKpiHistogramData[] | null => + data && data.length > 0 + ? data.map(({ key, count }) => ({ + x: key, + y: count.value, + })) + : null; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/authentications/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/authentications/index.ts index bf411744fc4ef..90c01a2346eab 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/authentications/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/authentications/index.ts @@ -16,7 +16,11 @@ import { import { inspectStringifyObject } from '../../../../../../utils/build_query'; import { SecuritySolutionFactory } from '../../../types'; import { buildHostsKpiAuthenticationsQuery } from './query.hosts_kpi_authentications.dsl'; -import { formatAuthenticationsHistogramData } from './helpers'; +import { + formatAuthenticationsHistogramData, + formatAuthenticationsHistogramDataEntities, +} from './helpers'; +import { buildHostsKpiAuthenticationsQueryEntities } from './query.hosts_kpi_authentications_entities.dsl'; export const hostsKpiAuthentications: SecuritySolutionFactory = { buildDsl: (options: HostsKpiAuthenticationsRequestOptions) => @@ -62,3 +66,48 @@ export const hostsKpiAuthentications: SecuritySolutionFactory = { + buildDsl: (options: HostsKpiAuthenticationsRequestOptions) => + buildHostsKpiAuthenticationsQueryEntities(options), + parse: async ( + options: HostsKpiAuthenticationsRequestOptions, + response: IEsSearchResponse + ): Promise => { + const inspect = { + dsl: [inspectStringifyObject(buildHostsKpiAuthenticationsQueryEntities(options))], + }; + + const authenticationsSuccessHistogram = getOr( + null, + 'aggregations.authentication_success_histogram.buckets', + response.rawResponse + ); + const authenticationsFailureHistogram = getOr( + null, + 'aggregations.authentication_failure_histogram.buckets', + response.rawResponse + ); + + return { + ...response, + inspect, + authenticationsSuccess: getOr( + null, + 'aggregations.authentication_success.value', + response.rawResponse + ), + authenticationsSuccessHistogram: formatAuthenticationsHistogramDataEntities( + authenticationsSuccessHistogram + ), + authenticationsFailure: getOr( + null, + 'aggregations.authentication_failure.value', + response.rawResponse + ), + authenticationsFailureHistogram: formatAuthenticationsHistogramDataEntities( + authenticationsFailureHistogram + ), + }; + }, +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/authentications/query.hosts_kpi_authentications_entities.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/authentications/query.hosts_kpi_authentications_entities.dsl.ts new file mode 100644 index 0000000000000..cff09f2354d31 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/authentications/query.hosts_kpi_authentications_entities.dsl.ts @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { HostsKpiAuthenticationsRequestOptions } from '../../../../../../../common/search_strategy/security_solution/hosts'; +import { createQueryFilterClauses } from '../../../../../../utils/build_query'; + +export const buildHostsKpiAuthenticationsQueryEntities = ({ + filterQuery, + timerange: { from, to }, + defaultIndex, +}: HostsKpiAuthenticationsRequestOptions) => { + const filter = [ + ...createQueryFilterClauses(filterQuery), + { + range: { + '@timestamp': { + gte: from, + lte: to, + format: 'strict_date_optional_time', + }, + }, + }, + ]; + + const dslQuery = { + index: defaultIndex, + allowNoIndices: true, + ignoreUnavailable: true, + track_total_hits: false, + body: { + aggs: { + authentication_success: { + sum: { + field: 'metrics.event.authentication.success.value_count', + }, + }, + authentication_success_histogram: { + auto_date_histogram: { + field: '@timestamp', + buckets: 6, + }, + aggs: { + count: { + sum: { + field: 'metrics.event.authentication.success.value_count', + }, + }, + }, + }, + authentication_failure: { + sum: { + field: 'metrics.event.authentication.failure.value_count', + }, + }, + authentication_failure_histogram: { + auto_date_histogram: { + field: '@timestamp', + buckets: 6, + }, + aggs: { + count: { + sum: { + field: 'metrics.event.authentication.failure.value_count', + }, + }, + }, + }, + }, + query: { + bool: { + filter, + }, + }, + size: 0, + }, + }; + + return dslQuery; +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/hosts/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/hosts/index.ts index 5f1eb9ab0fbfa..6bf2ccd5a2739 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/hosts/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/hosts/index.ts @@ -17,6 +17,7 @@ import { inspectStringifyObject } from '../../../../../../utils/build_query'; import { SecuritySolutionFactory } from '../../../types'; import { buildHostsKpiHostsQuery } from './query.hosts_kpi_hosts.dsl'; import { formatGeneralHistogramData } from '../common'; +import { buildHostsKpiHostsQueryEntities } from './query.hosts_kpi_hosts_entities.dsl'; export const hostsKpiHosts: SecuritySolutionFactory = { buildDsl: (options: HostsKpiHostsRequestOptions) => buildHostsKpiHostsQuery(options), @@ -41,3 +42,27 @@ export const hostsKpiHosts: SecuritySolutionFactory = }; }, }; + +export const hostsKpiHostsEntities: SecuritySolutionFactory = { + buildDsl: (options: HostsKpiHostsRequestOptions) => buildHostsKpiHostsQueryEntities(options), + parse: async ( + options: HostsKpiHostsRequestOptions, + response: IEsSearchResponse + ): Promise => { + const inspect = { + dsl: [inspectStringifyObject(buildHostsKpiHostsQueryEntities(options))], + }; + + const hostsHistogram = getOr( + null, + 'aggregations.hosts_histogram.buckets', + response.rawResponse + ); + return { + ...response, + inspect, + hosts: getOr(null, 'aggregations.hosts.value', response.rawResponse), + hostsHistogram: formatGeneralHistogramData(hostsHistogram), + }; + }, +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/hosts/query.hosts_kpi_hosts_entities.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/hosts/query.hosts_kpi_hosts_entities.dsl.ts new file mode 100644 index 0000000000000..972ead9a6538e --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/hosts/query.hosts_kpi_hosts_entities.dsl.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { HostsKpiHostsRequestOptions } from '../../../../../../../common/search_strategy/security_solution/hosts'; +import { createQueryFilterClauses } from '../../../../../../utils/build_query'; + +export const buildHostsKpiHostsQueryEntities = ({ + filterQuery, + timerange: { from, to }, + defaultIndex, +}: HostsKpiHostsRequestOptions) => { + const filter = [ + ...createQueryFilterClauses(filterQuery), + { + range: { + '@timestamp': { + gte: from, + lte: to, + format: 'strict_date_optional_time', + }, + }, + }, + ]; + + const dslQuery = { + index: defaultIndex, + allowNoIndices: true, + ignoreUnavailable: true, + track_total_hits: false, + body: { + aggregations: { + hosts: { + cardinality: { + field: 'host.name', + }, + }, + hosts_histogram: { + auto_date_histogram: { + field: '@timestamp', + buckets: 6, + }, + aggs: { + count: { + cardinality: { + field: 'host.name', + }, + }, + }, + }, + }, + query: { + bool: { + filter, + }, + }, + size: 0, + }, + }; + + return dslQuery; +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/unique_ips/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/unique_ips/index.ts index 89bb6c09fc49c..83849f886163e 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/unique_ips/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/unique_ips/index.ts @@ -17,6 +17,7 @@ import { inspectStringifyObject } from '../../../../../../utils/build_query'; import { SecuritySolutionFactory } from '../../../types'; import { buildHostsKpiUniqueIpsQuery } from './query.hosts_kpi_unique_ips.dsl'; import { formatGeneralHistogramData } from '../common'; +import { buildHostsKpiUniqueIpsQueryEntities } from './query.hosts_kpi_unique_ips_entities.dsl'; export const hostsKpiUniqueIps: SecuritySolutionFactory = { buildDsl: (options: HostsKpiUniqueIpsRequestOptions) => buildHostsKpiUniqueIpsQuery(options), @@ -54,3 +55,41 @@ export const hostsKpiUniqueIps: SecuritySolutionFactory = { + buildDsl: (options: HostsKpiUniqueIpsRequestOptions) => + buildHostsKpiUniqueIpsQueryEntities(options), + parse: async ( + options: HostsKpiUniqueIpsRequestOptions, + response: IEsSearchResponse + ): Promise => { + const inspect = { + dsl: [inspectStringifyObject(buildHostsKpiUniqueIpsQueryEntities(options))], + }; + + const uniqueSourceIpsHistogram = getOr( + null, + 'aggregations.unique_source_ips_histogram.buckets', + response.rawResponse + ); + + const uniqueDestinationIpsHistogram = getOr( + null, + 'aggregations.unique_destination_ips_histogram.buckets', + response.rawResponse + ); + + return { + ...response, + inspect, + uniqueSourceIps: getOr(null, 'aggregations.unique_source_ips.value', response.rawResponse), + uniqueSourceIpsHistogram: formatGeneralHistogramData(uniqueSourceIpsHistogram), + uniqueDestinationIps: getOr( + null, + 'aggregations.unique_destination_ips.value', + response.rawResponse + ), + uniqueDestinationIpsHistogram: formatGeneralHistogramData(uniqueDestinationIpsHistogram), + }; + }, +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/unique_ips/query.hosts_kpi_unique_ips_entities.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/unique_ips/query.hosts_kpi_unique_ips_entities.dsl.ts new file mode 100644 index 0000000000000..2a55c34238d70 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/unique_ips/query.hosts_kpi_unique_ips_entities.dsl.ts @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { HostsKpiUniqueIpsRequestOptions } from '../../../../../../../common/search_strategy/security_solution/hosts'; +import { createQueryFilterClauses } from '../../../../../../utils/build_query'; + +export const buildHostsKpiUniqueIpsQueryEntities = ({ + filterQuery, + timerange: { from, to }, + defaultIndex, +}: HostsKpiUniqueIpsRequestOptions) => { + const filter = [ + ...createQueryFilterClauses(filterQuery), + { + range: { + '@timestamp': { + gte: from, + lte: to, + format: 'strict_date_optional_time', + }, + }, + }, + ]; + + const dslQuery = { + index: defaultIndex, + allowNoIndices: true, + ignoreUnavailable: true, + track_total_hits: false, + body: { + aggregations: { + unique_source_ips: { + cardinality: { + field: 'source.ip', + }, + }, + unique_source_ips_histogram: { + auto_date_histogram: { + field: '@timestamp', + buckets: 6, + }, + aggs: { + count: { + cardinality: { + field: 'source.ip', + }, + }, + }, + }, + unique_destination_ips: { + cardinality: { + field: 'destination.ip', + }, + }, + unique_destination_ips_histogram: { + auto_date_histogram: { + field: '@timestamp', + buckets: 6, + }, + aggs: { + count: { + cardinality: { + field: 'destination.ip', + }, + }, + }, + }, + }, + query: { + bool: { + filter, + }, + }, + size: 0, + }, + }; + + return dslQuery; +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/index.ts index 49e09e3915673..c147b32be2c00 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/index.ts @@ -5,10 +5,19 @@ * 2.0. */ +import { getEntitiesParser } from '../helpers'; import { buildAuthenticationsHistogramQuery } from './query.authentications_histogram.dsl'; +import { buildAuthenticationsHistogramQueryEntities } from './query.authentications_histogram_entities.dsl'; export const authenticationsMatrixHistogramConfig = { buildDsl: buildAuthenticationsHistogramQuery, aggName: 'aggregations.eventActionGroup.buckets', parseKey: 'events.buckets', }; + +export const authenticationsMatrixHistogramEntitiesConfig = { + buildDsl: buildAuthenticationsHistogramQueryEntities, + aggName: 'aggregations.events.buckets', + parseKey: 'events.buckets', + parser: getEntitiesParser, +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/query.authentications_histogram_entities.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/query.authentications_histogram_entities.dsl.ts new file mode 100644 index 0000000000000..c66a0d6c11b94 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/query.authentications_histogram_entities.dsl.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; + +import { createQueryFilterClauses } from '../../../../../utils/build_query'; +import { MatrixHistogramRequestOptions } from '../../../../../../common/search_strategy/security_solution/matrix_histogram'; + +export const buildAuthenticationsHistogramQueryEntities = ({ + filterQuery, + timerange: { from, to }, + defaultIndex, + stackByField = 'event.outcome', // TODO: Remove this field if not used +}: MatrixHistogramRequestOptions) => { + const filter = [ + ...createQueryFilterClauses(filterQuery), + { + range: { + '@timestamp': { + gte: from, + lte: to, + format: 'strict_date_optional_time', + }, + }, + }, + ]; + + const getHistogramAggregation = () => { + const histogramTimestampField = '@timestamp'; + const dateHistogram = { + date_histogram: { + field: histogramTimestampField, + calendar_interval: '1h', + min_doc_count: 0, + extended_bounds: { + min: moment(from).valueOf(), + max: moment(to).valueOf(), + }, + }, + aggs: { + failure: { + sum: { + field: 'metrics.event.authentication.failure.value_count', + }, + }, + success: { + sum: { + field: 'metrics.event.authentication.success.value_count', + }, + }, + }, + }; + return { events: dateHistogram }; + }; + + const dslQuery = { + index: defaultIndex, + allowNoIndices: true, + ignoreUnavailable: true, + track_total_hits: true, + body: { + aggregations: getHistogramAggregation(), + query: { + bool: { + filter, + }, + }, + size: 0, + }, + }; + + return dslQuery; +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/helpers.ts index aa6b85d795443..c8ede95d166c7 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/helpers.ts @@ -32,3 +32,19 @@ export const getGenericData = ( return result; }; + +export const getEntitiesParser = ( + data: MatrixHistogramParseData, + keyBucket: string // TODO: Remove this keyBucket if it is not being used. +): MatrixHistogramData[] => { + let result: MatrixHistogramData[] = []; + data.forEach((bucketData: unknown) => { + const successValue = get('success.value', bucketData); + const failureValue = get('failure.value', bucketData); + const key = get('key', bucketData); + const histDataSuccess = { x: key, y: successValue, g: 'success' }; + const histDataFailure = { x: key, y: failureValue, g: 'failure' }; + result = [...result, histDataFailure, histDataSuccess]; + }); + return result; +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/index.ts index 40b22a31691b6..3be1e6cb1dfe0 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/index.ts @@ -15,13 +15,17 @@ import { MatrixHistogramQuery, MatrixHistogramType, MatrixHistogramDataConfig, + MatrixHistogramQueryEntities, } from '../../../../../common/search_strategy/security_solution'; import { inspectStringifyObject } from '../../../../utils/build_query'; import { SecuritySolutionFactory } from '../types'; import { getGenericData } from './helpers'; import { alertsMatrixHistogramConfig } from './alerts'; import { anomaliesMatrixHistogramConfig } from './anomalies'; -import { authenticationsMatrixHistogramConfig } from './authentications'; +import { + authenticationsMatrixHistogramConfig, + authenticationsMatrixHistogramEntitiesConfig, +} from './authentications'; import { dnsMatrixHistogramConfig } from './dns'; import { eventsMatrixHistogramConfig } from './events'; @@ -29,6 +33,7 @@ const matrixHistogramConfig: MatrixHistogramDataConfig = { [MatrixHistogramType.alerts]: alertsMatrixHistogramConfig, [MatrixHistogramType.anomalies]: anomaliesMatrixHistogramConfig, [MatrixHistogramType.authentications]: authenticationsMatrixHistogramConfig, + [MatrixHistogramType.authenticationsEntities]: authenticationsMatrixHistogramEntitiesConfig, [MatrixHistogramType.dns]: dnsMatrixHistogramConfig, [MatrixHistogramType.events]: eventsMatrixHistogramConfig, }; @@ -69,9 +74,46 @@ export const matrixHistogram: SecuritySolutionFactory = { + buildDsl: (options: MatrixHistogramRequestOptions) => { + const myConfig = getOr(null, options.histogramType, matrixHistogramConfig); + if (myConfig == null) { + throw new Error(`This histogram type ${options.histogramType} is unknown to the server side`); + } + return myConfig.buildDsl(options); + }, + parse: async ( + options: MatrixHistogramRequestOptions, + response: IEsSearchResponse + ): Promise => { + const myConfig = getOr(null, options.histogramType, matrixHistogramConfig); + if (myConfig == null) { + throw new Error(`This histogram type ${options.histogramType} is unknown to the server side`); + } + const totalCount = response.rawResponse.hits.total || 0; + const matrixHistogramData = getOr([], myConfig.aggName, response.rawResponse); + const inspect = { + dsl: [inspectStringifyObject(myConfig.buildDsl(options))], + }; + const dataParser = myConfig.parser ?? getGenericData; + + return { + ...response, + inspect, + matrixHistogramData: dataParser( + matrixHistogramData, + myConfig.parseKey + ), + // @ts-expect-error code doesn't handle TotalHits + totalCount, + }; + }, +}; + export const matrixHistogramFactory: Record< - typeof MatrixHistogramQuery, + typeof MatrixHistogramQuery | typeof MatrixHistogramQueryEntities, SecuritySolutionFactory > = { [MatrixHistogramQuery]: matrixHistogram, + [MatrixHistogramQueryEntities]: matrixHistogramEntities, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.test.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.test.ts index d3621ef22bf2a..3e770cbedaed6 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.test.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.test.ts @@ -19,11 +19,14 @@ import { networkTls } from './tls'; import { networkTopCountries } from './top_countries'; import { networkTopNFlow } from './top_n_flow'; import { networkUsers } from './users'; -import { networkKpiDns } from './kpi/dns'; -import { networkKpiNetworkEvents } from './kpi/network_events'; -import { networkKpiTlsHandshakes } from './kpi/tls_handshakes'; +import { networkKpiDns, networkKpiDnsEntities } from './kpi/dns'; +import { networkKpiNetworkEvents, networkKpiNetworkEventsEntities } from './kpi/network_events'; +import { networkKpiTlsHandshakes, networkKpiTlsHandshakesEntities } from './kpi/tls_handshakes'; import { networkKpiUniqueFlows } from './kpi/unique_flows'; -import { networkKpiUniquePrivateIps } from './kpi/unique_private_ips'; +import { + networkKpiUniquePrivateIps, + networkKpiUniquePrivateIpsEntities, +} from './kpi/unique_private_ips'; jest.mock('./details'); jest.mock('./dns'); @@ -51,10 +54,14 @@ describe('networkFactory', () => { [NetworkQueries.topNFlow]: networkTopNFlow, [NetworkQueries.users]: networkUsers, [NetworkKpiQueries.dns]: networkKpiDns, + [NetworkKpiQueries.dnsEntities]: networkKpiDnsEntities, [NetworkKpiQueries.networkEvents]: networkKpiNetworkEvents, + [NetworkKpiQueries.networkEventsEntities]: networkKpiNetworkEventsEntities, + [NetworkKpiQueries.tlsHandshakesEntities]: networkKpiTlsHandshakesEntities, [NetworkKpiQueries.tlsHandshakes]: networkKpiTlsHandshakes, [NetworkKpiQueries.uniqueFlows]: networkKpiUniqueFlows, [NetworkKpiQueries.uniquePrivateIps]: networkKpiUniquePrivateIps, + [NetworkKpiQueries.uniquePrivateIpsEntities]: networkKpiUniquePrivateIpsEntities, }; expect(networkFactory).toEqual(expectedNetworkFactory); }); diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.ts index 9d9940247eb30..3a7c4951ea878 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.ts @@ -11,19 +11,22 @@ import { NetworkKpiQueries, } from '../../../../../common/search_strategy/security_solution'; -import { networkKpiDns } from './kpi/dns'; -import { networkKpiNetworkEvents } from './kpi/network_events'; -import { networkKpiTlsHandshakes } from './kpi/tls_handshakes'; +import { networkKpiDns, networkKpiDnsEntities } from './kpi/dns'; +import { networkKpiNetworkEvents, networkKpiNetworkEventsEntities } from './kpi/network_events'; +import { networkKpiTlsHandshakes, networkKpiTlsHandshakesEntities } from './kpi/tls_handshakes'; import { networkKpiUniqueFlows } from './kpi/unique_flows'; -import { networkKpiUniquePrivateIps } from './kpi/unique_private_ips'; +import { + networkKpiUniquePrivateIps, + networkKpiUniquePrivateIpsEntities, +} from './kpi/unique_private_ips'; import { SecuritySolutionFactory } from '../types'; import { networkDetails } from './details'; import { networkDns } from './dns'; import { networkHttp } from './http'; import { networkOverview } from './overview'; import { networkTls } from './tls'; -import { networkTopCountries } from './top_countries'; -import { networkTopNFlow } from './top_n_flow'; +import { networkTopCountries, networkTopCountriesEntities } from './top_countries'; +import { networkTopNFlow, networkTopNFlowEntities } from './top_n_flow'; import { networkUsers } from './users'; export const networkFactory: Record< @@ -36,11 +39,17 @@ export const networkFactory: Record< [NetworkQueries.overview]: networkOverview, [NetworkQueries.tls]: networkTls, [NetworkQueries.topCountries]: networkTopCountries, + [NetworkQueries.topCountriesEntities]: networkTopCountriesEntities, [NetworkQueries.topNFlow]: networkTopNFlow, + [NetworkQueries.topNFlowEntities]: networkTopNFlowEntities, [NetworkQueries.users]: networkUsers, [NetworkKpiQueries.dns]: networkKpiDns, + [NetworkKpiQueries.dnsEntities]: networkKpiDnsEntities, [NetworkKpiQueries.networkEvents]: networkKpiNetworkEvents, + [NetworkKpiQueries.networkEventsEntities]: networkKpiNetworkEventsEntities, [NetworkKpiQueries.tlsHandshakes]: networkKpiTlsHandshakes, + [NetworkKpiQueries.tlsHandshakesEntities]: networkKpiTlsHandshakesEntities, [NetworkKpiQueries.uniqueFlows]: networkKpiUniqueFlows, [NetworkKpiQueries.uniquePrivateIps]: networkKpiUniquePrivateIps, + [NetworkKpiQueries.uniquePrivateIpsEntities]: networkKpiUniquePrivateIpsEntities, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/dns/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/dns/index.ts index 7ef0e6e303528..c5298c4498cda 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/dns/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/dns/index.ts @@ -13,6 +13,7 @@ import { } from '../../../../../../../common/search_strategy/security_solution/network'; import { inspectStringifyObject } from '../../../../../../utils/build_query'; import { SecuritySolutionFactory } from '../../../types'; +import { buildDnsQueryEntities } from './query.network_kip_dns_entities.dsl'; import { buildDnsQuery } from './query.network_kpi_dns.dsl'; export const networkKpiDns: SecuritySolutionFactory = { @@ -33,3 +34,21 @@ export const networkKpiDns: SecuritySolutionFactory = { }; }, }; + +export const networkKpiDnsEntities: SecuritySolutionFactory = { + buildDsl: (options: NetworkKpiDnsRequestOptions) => buildDnsQueryEntities(options), + parse: async ( + options: NetworkKpiDnsRequestOptions, + response: IEsSearchResponse + ): Promise => { + const inspect = { + dsl: [inspectStringifyObject(buildDnsQueryEntities(options))], + }; + return { + ...response, + inspect, + // @ts-expect-error code doesn't handle TotalHits + dnsQueries: response.rawResponse.aggregations?.dns?.value ?? null, + }; + }, +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/dns/query.network_kip_dns_entities.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/dns/query.network_kip_dns_entities.dsl.ts new file mode 100644 index 0000000000000..75b32af4b01f5 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/dns/query.network_kip_dns_entities.dsl.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { NetworkKpiDnsRequestOptions } from '../../../../../../../common/search_strategy/security_solution/network'; +import { createQueryFilterClauses } from '../../../../../../utils/build_query'; + +export const buildDnsQueryEntities = ({ + filterQuery, + timerange: { from, to }, + defaultIndex, +}: NetworkKpiDnsRequestOptions) => { + const filter = [ + ...createQueryFilterClauses(filterQuery), + { + range: { + '@timestamp': { + gte: from, + lte: to, + format: 'strict_date_optional_time', + }, + }, + }, + ]; + + const dslQuery = { + index: defaultIndex, + allowNoIndices: true, + ignoreUnavailable: true, + track_total_hits: false, + body: { + aggs: { + dns: { + sum: { + field: 'metrics.dns.queries.value_count', + }, + }, + }, + query: { + bool: { + filter, + }, + }, + size: 0, + }, + }; + + return dslQuery; +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/index.ts index 2a18bf3b5de86..326d4f33e7a93 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/index.ts @@ -11,19 +11,26 @@ import { } from '../../../../../../common/search_strategy/security_solution'; import { SecuritySolutionFactory } from '../../types'; -import { networkKpiDns } from './dns'; -import { networkKpiNetworkEvents } from './network_events'; -import { networkKpiTlsHandshakes } from './tls_handshakes'; +import { networkKpiDns, networkKpiDnsEntities } from './dns'; +import { networkKpiNetworkEvents, networkKpiNetworkEventsEntities } from './network_events'; +import { networkKpiTlsHandshakes, networkKpiTlsHandshakesEntities } from './tls_handshakes'; import { networkKpiUniqueFlows } from './unique_flows'; -import { networkKpiUniquePrivateIps } from './unique_private_ips'; +import { + networkKpiUniquePrivateIps, + networkKpiUniquePrivateIpsEntities, +} from './unique_private_ips'; export const networkKpiFactory: Record< NetworkKpiQueries, SecuritySolutionFactory > = { [NetworkKpiQueries.dns]: networkKpiDns, + [NetworkKpiQueries.dnsEntities]: networkKpiDnsEntities, [NetworkKpiQueries.networkEvents]: networkKpiNetworkEvents, + [NetworkKpiQueries.networkEventsEntities]: networkKpiNetworkEventsEntities, [NetworkKpiQueries.tlsHandshakes]: networkKpiTlsHandshakes, + [NetworkKpiQueries.tlsHandshakesEntities]: networkKpiTlsHandshakesEntities, [NetworkKpiQueries.uniqueFlows]: networkKpiUniqueFlows, [NetworkKpiQueries.uniquePrivateIps]: networkKpiUniquePrivateIps, + [NetworkKpiQueries.uniquePrivateIpsEntities]: networkKpiUniquePrivateIpsEntities, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/network_events/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/network_events/index.ts index 5327a2396cdac..5f4db9591dbf6 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/network_events/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/network_events/index.ts @@ -14,6 +14,7 @@ import { import { inspectStringifyObject } from '../../../../../../utils/build_query'; import { SecuritySolutionFactory } from '../../../types'; import { buildNetworkEventsQuery } from './query.network_kpi_network_events.dsl'; +import { buildNetworkEventsQueryEntities } from './query.network_kpi_network_events_entities.dsl'; export const networkKpiNetworkEvents: SecuritySolutionFactory = { buildDsl: (options: NetworkKpiNetworkEventsRequestOptions) => buildNetworkEventsQuery(options), @@ -33,3 +34,23 @@ export const networkKpiNetworkEvents: SecuritySolutionFactory = { + buildDsl: (options: NetworkKpiNetworkEventsRequestOptions) => + buildNetworkEventsQueryEntities(options), + parse: async ( + options: NetworkKpiNetworkEventsRequestOptions, + response: IEsSearchResponse + ): Promise => { + const inspect = { + dsl: [inspectStringifyObject(buildNetworkEventsQueryEntities(options))], + }; + + return { + ...response, + inspect, + // @ts-expect-error code doesn't handle TotalHits + networkEvents: response.rawResponse.aggregations?.events?.value ?? null, + }; + }, +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/network_events/query.network_kpi_network_events_entities.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/network_events/query.network_kpi_network_events_entities.dsl.ts new file mode 100644 index 0000000000000..6311bb6ea2039 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/network_events/query.network_kpi_network_events_entities.dsl.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { NetworkKpiNetworkEventsRequestOptions } from '../../../../../../../common/search_strategy/security_solution/network'; +import { createQueryFilterClauses } from '../../../../../../utils/build_query'; + +export const buildNetworkEventsQueryEntities = ({ + filterQuery, + timerange: { from, to }, + defaultIndex, +}: NetworkKpiNetworkEventsRequestOptions) => { + const filter = [ + ...createQueryFilterClauses(filterQuery), + { + range: { + '@timestamp': { + gte: from, + lte: to, + format: 'strict_date_optional_time', + }, + }, + }, + ]; + + const dslQuery = { + index: defaultIndex, + allowNoIndices: true, + ignoreUnavailable: true, + track_total_hits: false, + body: { + aggs: { + events: { + sum: { + field: 'metrics.network.events.value_count', + }, + }, + }, + query: { + bool: { + filter, + }, + }, + size: 0, + }, + }; + + return dslQuery; +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/tls_handshakes/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/tls_handshakes/index.ts index 17c2c4cf64981..016abdb10f935 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/tls_handshakes/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/tls_handshakes/index.ts @@ -14,6 +14,7 @@ import { import { inspectStringifyObject } from '../../../../../../utils/build_query'; import { SecuritySolutionFactory } from '../../../types'; import { buildTlsHandshakeQuery } from './query.network_kpi_tls_handshakes.dsl'; +import { buildTlsHandshakeQueryEntities } from './query.network_kpi_tls_handshakes_entities.dsl'; export const networkKpiTlsHandshakes: SecuritySolutionFactory = { buildDsl: (options: NetworkKpiTlsHandshakesRequestOptions) => buildTlsHandshakeQuery(options), @@ -33,3 +34,23 @@ export const networkKpiTlsHandshakes: SecuritySolutionFactory = { + buildDsl: (options: NetworkKpiTlsHandshakesRequestOptions) => + buildTlsHandshakeQueryEntities(options), + parse: async ( + options: NetworkKpiTlsHandshakesRequestOptions, + response: IEsSearchResponse + ): Promise => { + const inspect = { + dsl: [inspectStringifyObject(buildTlsHandshakeQueryEntities(options))], + }; + + return { + ...response, + inspect, + // @ts-expect-error code doesn't handle TotalHits + tlsHandshakes: response.rawResponse.aggregations?.tls?.value ?? null, + }; + }, +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/tls_handshakes/query.network_kpi_tls_handshakes_entities.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/tls_handshakes/query.network_kpi_tls_handshakes_entities.dsl.ts new file mode 100644 index 0000000000000..5b0ac92b35049 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/tls_handshakes/query.network_kpi_tls_handshakes_entities.dsl.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { NetworkKpiTlsHandshakesRequestOptions } from '../../../../../../../common/search_strategy/security_solution/network'; +import { createQueryFilterClauses } from '../../../../../../utils/build_query'; + +export const buildTlsHandshakeQueryEntities = ({ + filterQuery, + timerange: { from, to }, + defaultIndex, +}: NetworkKpiTlsHandshakesRequestOptions) => { + const filter = [ + ...createQueryFilterClauses(filterQuery), + { + range: { + '@timestamp': { + gte: from, + lte: to, + format: 'strict_date_optional_time', + }, + }, + }, + ]; + + const dslQuery = { + index: defaultIndex, + allowNoIndices: true, + ignoreUnavailable: true, + track_total_hits: false, + body: { + aggs: { + tls: { + sum: { + field: 'metrics.network.tls.version.value_count', + }, + }, + }, + query: { + bool: { + filter, + }, + }, + size: 0, + }, + }; + + return dslQuery; +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/unique_private_ips/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/unique_private_ips/index.ts index a6b9255cf6c95..6eaa9ac8f2214 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/unique_private_ips/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/unique_private_ips/index.ts @@ -17,6 +17,7 @@ import { inspectStringifyObject } from '../../../../../../utils/build_query'; import { SecuritySolutionFactory } from '../../../types'; import { formatHistogramData } from '../common'; import { buildUniquePrivateIpsQuery } from './query.network_kpi_unique_private_ips.dsl'; +import { buildUniquePrivateIpsQueryEntities } from './query.network_kpi_unique_private_ips_entities.dsl'; export const networkKpiUniquePrivateIps: SecuritySolutionFactory = { // @ts-expect-error auto_date_histogram.buckets is incompatible @@ -61,3 +62,47 @@ export const networkKpiUniquePrivateIps: SecuritySolutionFactory = { + // @ts-expect-error auto_date_histogram.buckets is incompatible + buildDsl: (options: NetworkKpiUniquePrivateIpsRequestOptions) => + buildUniquePrivateIpsQueryEntities(options), + parse: async ( + options: NetworkKpiUniquePrivateIpsRequestOptions, + response: IEsSearchResponse + ): Promise => { + const inspect = { + dsl: [inspectStringifyObject(buildUniquePrivateIpsQueryEntities(options))], + }; + + const uniqueSourcePrivateIpsHistogram = getOr( + null, + 'aggregations.source.histogram.buckets', + response.rawResponse + ); + const uniqueDestinationPrivateIpsHistogram = getOr( + null, + 'aggregations.destination.histogram.buckets', + response.rawResponse + ); + + return { + ...response, + inspect, + uniqueSourcePrivateIps: getOr( + null, + 'aggregations.source.unique_private_ips.value', + response.rawResponse + ), + uniqueDestinationPrivateIps: getOr( + null, + 'aggregations.destination.unique_private_ips.value', + response.rawResponse + ), + uniqueSourcePrivateIpsHistogram: formatHistogramData(uniqueSourcePrivateIpsHistogram), + uniqueDestinationPrivateIpsHistogram: formatHistogramData( + uniqueDestinationPrivateIpsHistogram + ), + }; + }, +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/unique_private_ips/query.network_kpi_unique_private_ips_entities.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/unique_private_ips/query.network_kpi_unique_private_ips_entities.dsl.ts new file mode 100644 index 0000000000000..a56cf4c3d1ced --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/unique_private_ips/query.network_kpi_unique_private_ips_entities.dsl.ts @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + NetworkKpiUniquePrivateIpsRequestOptions, + UniquePrivateAttributeQuery, +} from '../../../../../../../common/search_strategy/security_solution/network'; +import { createQueryFilterClauses } from '../../../../../../utils/build_query'; + +const getUniquePrivateIpsFilter = (attrQuery: UniquePrivateAttributeQuery) => ({ + bool: { + should: [ + { + term: { + [`${attrQuery}.ip`]: '10.0.0.0/8', + }, + }, + { + term: { + [`${attrQuery}.ip`]: '192.168.0.0/16', + }, + }, + { + term: { + [`${attrQuery}.ip`]: '172.16.0.0/12', + }, + }, + { + term: { + [`${attrQuery}.ip`]: 'fd00::/8', + }, + }, + ], + minimum_should_match: 1, + }, +}); + +const getAggs = (attrQuery: 'source' | 'destination') => ({ + [attrQuery]: { + filter: getUniquePrivateIpsFilter(attrQuery), + aggs: { + unique_private_ips: { + cardinality: { + field: `${attrQuery}.ip`, + }, + }, + histogram: { + auto_date_histogram: { + field: '@timestamp', + buckets: '6', + }, + aggs: { + count: { + cardinality: { + field: `${attrQuery}.ip`, + }, + }, + }, + }, + }, + }, +}); + +export const buildUniquePrivateIpsQueryEntities = ({ + filterQuery, + timerange: { from, to }, + defaultIndex, +}: NetworkKpiUniquePrivateIpsRequestOptions) => { + const filter = [ + ...createQueryFilterClauses(filterQuery), + { + range: { + '@timestamp': { + gte: from, + lte: to, + format: 'strict_date_optional_time', + }, + }, + }, + ]; + + const dslQuery = { + allowNoIndices: true, + index: defaultIndex, + ignoreUnavailable: true, + track_total_hits: false, + body: { + aggregations: { + ...getAggs('source'), + ...getAggs('destination'), + }, + query: { + bool: { + filter, + }, + }, + size: 0, + }, + }; + + return dslQuery; +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/index.ts index 6a53771085789..80c2050fd1423 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/index.ts @@ -22,6 +22,7 @@ import { SecuritySolutionFactory } from '../../types'; import { getTopCountriesEdges } from './helpers'; import { buildTopCountriesQuery } from './query.top_countries_network.dsl'; +import { buildTopCountriesQueryEntities } from './query.top_countries_network_entities.dsl'; export const networkTopCountries: SecuritySolutionFactory = { buildDsl: (options: NetworkTopCountriesRequestOptions) => { @@ -60,3 +61,41 @@ export const networkTopCountries: SecuritySolutionFactory = { + buildDsl: (options: NetworkTopCountriesRequestOptions) => { + if (options.pagination && options.pagination.querySize >= DEFAULT_MAX_TABLE_QUERY_SIZE) { + throw new Error(`No query size above ${DEFAULT_MAX_TABLE_QUERY_SIZE}`); + } + return buildTopCountriesQueryEntities(options); + }, + parse: async ( + options: NetworkTopCountriesRequestOptions, + response: IEsSearchResponse + ): Promise => { + const { activePage, cursorStart, fakePossibleCount, querySize } = options.pagination; + const totalCount = getOr(0, 'aggregations.top_countries_count.value', response.rawResponse); + const networkTopCountriesEdges: NetworkTopCountriesEdges[] = getTopCountriesEdges( + response, + options + ); + const fakeTotalCount = fakePossibleCount <= totalCount ? fakePossibleCount : totalCount; + const edges = networkTopCountriesEdges.splice(cursorStart, querySize - cursorStart); + const inspect = { + dsl: [inspectStringifyObject(buildTopCountriesQueryEntities(options))], + }; + const showMorePagesIndicator = totalCount > fakeTotalCount; + + return { + ...response, + edges, + inspect, + pageInfo: { + activePage: activePage ?? 0, + fakeTotalCount, + showMorePagesIndicator, + }, + totalCount, + }; + }, +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/query.top_countries_network_entities.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/query.top_countries_network_entities.dsl.ts new file mode 100644 index 0000000000000..d661bfa0d6707 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/query.top_countries_network_entities.dsl.ts @@ -0,0 +1,162 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createQueryFilterClauses } from '../../../../../utils/build_query'; +import { assertUnreachable } from '../../../../../../common/utility_types'; +import { + Direction, + FlowTargetSourceDest, + NetworkTopTablesFields, + NetworkTopCountriesRequestOptions, + SortField, +} from '../../../../../../common/search_strategy'; + +// TODO: This is the same as the other one, so move this into helpers. +const getCountAgg = (flowTarget: FlowTargetSourceDest) => ({ + top_countries_count: { + cardinality: { + field: `${flowTarget}.geo.country_iso_code`, + }, + }, +}); + +export const buildTopCountriesQueryEntities = ({ + defaultIndex, + filterQuery, + flowTarget, + sort, + pagination: { querySize }, + timerange: { from, to }, + ip, +}: NetworkTopCountriesRequestOptions) => { + const filter = [ + ...createQueryFilterClauses(filterQuery), + { + range: { + '@timestamp': { + gte: from, + lte: to, + format: 'strict_date_optional_time', + }, + }, + }, + ]; + + const dslQuery = { + allowNoIndices: true, + index: defaultIndex, + ignoreUnavailable: true, + body: { + aggregations: { + ...getCountAgg(flowTarget), + ...getFlowTargetAggs(sort, flowTarget, querySize), + }, + query: { + bool: ip + ? { + filter, + should: [ + { + term: { + [`${getOppositeField(flowTarget)}.ip`]: ip, + }, + }, + ], + minimum_should_match: 1, + } + : { + filter, + }, + }, + }, + size: 0, + track_total_hits: false, + }; + return dslQuery; +}; + +const getFlowTargetAggs = ( + sort: SortField, + flowTarget: FlowTargetSourceDest, + querySize: number +) => ({ + [flowTarget]: { + terms: { + field: `${flowTarget}.geo.country_iso_code`, + size: querySize, + order: { + ...getQueryOrder(sort), + }, + }, + aggs: { + bytes_in: { + sum: { + field: `metrics.${getOppositeField(flowTarget)}.bytes.sum`, + }, + }, + bytes_out: { + sum: { + field: `metrics.${flowTarget}.bytes.sum`, + }, + }, + flows: { + // TODO: Should we use max here and/or do a hybrid with a max here for performance? + avg: { + field: 'metrics.network.community_id.cardinality', + }, + }, + source_ips: { + avg: { + field: 'metrics.source.ip.cardinality', + }, + }, + destination_ips: { + avg: { + field: 'metrics.destination.ip.cardinality', + }, + }, + }, + }, +}); + +// TODO: This is the same as the other one, so move this to helpers and use it from there. +export const getOppositeField = (flowTarget: FlowTargetSourceDest): FlowTargetSourceDest => { + switch (flowTarget) { + case FlowTargetSourceDest.source: + return FlowTargetSourceDest.destination; + case FlowTargetSourceDest.destination: + return FlowTargetSourceDest.source; + } + assertUnreachable(flowTarget); +}; + +// TODO: This is the same as the other one, so move this to helpers and use it from there. +type QueryOrder = + | { bytes_in: Direction } + | { bytes_out: Direction } + | { flows: Direction } + | { destination_ips: Direction } + | { source_ips: Direction }; + +// TODO: This is the same as the other one, so move this to helpers and use it from there. +const getQueryOrder = ( + networkTopCountriesSortField: SortField +): QueryOrder => { + switch (networkTopCountriesSortField.field) { + case NetworkTopTablesFields.bytes_in: + return { bytes_in: networkTopCountriesSortField.direction }; + case NetworkTopTablesFields.bytes_out: + return { bytes_out: networkTopCountriesSortField.direction }; + case NetworkTopTablesFields.flows: + return { flows: networkTopCountriesSortField.direction }; + case NetworkTopTablesFields.destination_ips: + return { destination_ips: networkTopCountriesSortField.direction }; + case NetworkTopTablesFields.source_ips: + return { source_ips: networkTopCountriesSortField.direction }; + } + assertUnreachable(networkTopCountriesSortField.field); +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/index.ts index d26316cba8222..31529f6771941 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/index.ts @@ -22,6 +22,7 @@ import { SecuritySolutionFactory } from '../../types'; import { getTopNFlowEdges } from './helpers'; import { buildTopNFlowQuery } from './query.top_n_flow_network.dsl'; +import { buildTopNFlowQueryEntities } from './query.top_n_flow_network_entities.dsl'; export const networkTopNFlow: SecuritySolutionFactory = { buildDsl: (options: NetworkTopNFlowRequestOptions) => { @@ -57,3 +58,38 @@ export const networkTopNFlow: SecuritySolutionFactory = }; }, }; + +export const networkTopNFlowEntities: SecuritySolutionFactory = { + buildDsl: (options: NetworkTopNFlowRequestOptions) => { + if (options.pagination && options.pagination.querySize >= DEFAULT_MAX_TABLE_QUERY_SIZE) { + throw new Error(`No query size above ${DEFAULT_MAX_TABLE_QUERY_SIZE}`); + } + return buildTopNFlowQueryEntities(options); + }, + parse: async ( + options: NetworkTopNFlowRequestOptions, + response: IEsSearchResponse + ): Promise => { + const { activePage, cursorStart, fakePossibleCount, querySize } = options.pagination; + const totalCount = getOr(0, 'aggregations.top_n_flow_count.value', response.rawResponse); + const networkTopNFlowEdges: NetworkTopNFlowEdges[] = getTopNFlowEdges(response, options); + const fakeTotalCount = fakePossibleCount <= totalCount ? fakePossibleCount : totalCount; + const edges = networkTopNFlowEdges.splice(cursorStart, querySize - cursorStart); + const inspect = { + dsl: [inspectStringifyObject(buildTopNFlowQueryEntities(options))], + }; + const showMorePagesIndicator = totalCount > fakeTotalCount; + + return { + ...response, + edges, + inspect, + pageInfo: { + activePage: activePage ?? 0, + fakeTotalCount, + showMorePagesIndicator, + }, + totalCount, + }; + }, +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/query.top_n_flow_network_entities.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/query.top_n_flow_network_entities.dsl.ts new file mode 100644 index 0000000000000..3ea3c6f363de0 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/query.top_n_flow_network_entities.dsl.ts @@ -0,0 +1,165 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + SortField, + FlowTargetSourceDest, + NetworkTopTablesFields, + NetworkTopNFlowRequestOptions, +} from '../../../../../../common/search_strategy'; +import { createQueryFilterClauses } from '../../../../../utils/build_query'; +import { getOppositeField } from '../helpers'; +import { getQueryOrder } from './helpers'; + +// TODO: This is the same as the other one, so move this into helpers. +const getCountAgg = (flowTarget: FlowTargetSourceDest) => ({ + top_n_flow_count: { + cardinality: { + field: `${flowTarget}.ip`, + }, + }, +}); + +export const buildTopNFlowQueryEntities = ({ + defaultIndex, + filterQuery, + flowTarget, + sort, + pagination: { querySize }, + timerange: { from, to }, + ip, +}: NetworkTopNFlowRequestOptions) => { + const filter = [ + ...createQueryFilterClauses(filterQuery), + { + range: { + '@timestamp': { + gte: from, + lte: to, + format: 'strict_date_optional_time', + }, + }, + }, + ]; + + const dslQuery = { + allowNoIndices: true, + index: defaultIndex, + ignoreUnavailable: true, + body: { + aggregations: { + ...getCountAgg(flowTarget), + ...getFlowTargetAggs(sort, flowTarget, querySize), + }, + query: { + bool: ip + ? { + filter, + should: [ + { + term: { + [`${getOppositeField(flowTarget)}.ip`]: ip, + }, + }, + ], + minimum_should_match: 1, + } + : { + filter, + }, + }, + }, + size: 0, + track_total_hits: false, + }; + return dslQuery; +}; + +const getFlowTargetAggs = ( + sort: SortField, + flowTarget: FlowTargetSourceDest, + querySize: number +) => ({ + [flowTarget]: { + terms: { + field: `${flowTarget}.ip`, + size: querySize, + order: { + ...getQueryOrder(sort), + }, + }, + aggs: { + bytes_in: { + sum: { + field: `metrics.${getOppositeField(flowTarget)}.bytes.sum`, + }, + }, + bytes_out: { + sum: { + field: `metrics.${flowTarget}.bytes.sum`, + }, + }, + domain: { + terms: { + field: `${flowTarget}.domain`, + order: { + timestamp: 'desc', + }, + }, + aggs: { + timestamp: { + max: { + field: '@timestamp', + }, + }, + }, + }, + location: { + filter: { + exists: { + field: `${flowTarget}.geo`, + }, + }, + aggs: { + top_geo: { + top_hits: { + _source: `${flowTarget}.geo.*`, + size: 1, + }, + }, + }, + }, + autonomous_system: { + filter: { + exists: { + field: `${flowTarget}.as`, + }, + }, + aggs: { + top_as: { + top_hits: { + _source: `${flowTarget}.as.*`, + size: 1, + }, + }, + }, + }, + flows: { + avg: { + // TODO: Should we use a max here along with a hybrid query? + field: 'metrics.network.community_id.cardinality', + }, + }, + [`${getOppositeField(flowTarget)}_ips`]: { + avg: { + // TODO: Should we use a max here along with a hybrid query? + field: `metrics.${getOppositeField(flowTarget)}.ip.cardinality`, + }, + }, + }, + }, +}); diff --git a/x-pack/plugins/security_solution/server/ui_settings.ts b/x-pack/plugins/security_solution/server/ui_settings.ts index ab220f0f3c295..259c0f2ae2f92 100644 --- a/x-pack/plugins/security_solution/server/ui_settings.ts +++ b/x-pack/plugins/security_solution/server/ui_settings.ts @@ -29,9 +29,16 @@ import { DEFAULT_RULE_REFRESH_INTERVAL_ON, DEFAULT_RULE_REFRESH_INTERVAL_VALUE, DEFAULT_RULE_REFRESH_IDLE_VALUE, + DEFAULT_TRANSFORMS, + DEFAULT_TRANSFORMS_SETTING, } from '../common/constants'; +import { transformConfigSchema } from '../common/transforms/types'; +import { ExperimentalFeatures } from '../common/experimental_features'; -export const initUiSettings = (uiSettings: CoreSetup['uiSettings']) => { +export const initUiSettings = ( + uiSettings: CoreSetup['uiSettings'], + experimentalFeatures: ExperimentalFeatures +) => { uiSettings.register({ [DEFAULT_APP_REFRESH_INTERVAL]: { type: 'json', @@ -181,5 +188,25 @@ export const initUiSettings = (uiSettings: CoreSetup['uiSettings']) => { }) ), }, + // TODO: Remove this check once the experimental flag is removed + ...(experimentalFeatures.metricsEntitiesEnabled + ? { + [DEFAULT_TRANSFORMS]: { + name: i18n.translate('xpack.securitySolution.uiSettings.transforms', { + defaultMessage: 'Default transforms to use', + }), + value: DEFAULT_TRANSFORMS_SETTING, + type: 'json', + description: i18n.translate('xpack.securitySolution.uiSettings.transformDescription', { + // TODO: Add a hyperlink to documentation about this feature + defaultMessage: 'Experimental: Enable an application cache through transforms', + }), + sensitive: true, + category: [APP_ID], + requiresPageReload: false, + schema: transformConfigSchema, + }, + } + : {}), }); }; diff --git a/x-pack/test/tsconfig.json b/x-pack/test/tsconfig.json index 9eaa4b93214b2..ab4bdb654e98d 100644 --- a/x-pack/test/tsconfig.json +++ b/x-pack/test/tsconfig.json @@ -70,6 +70,7 @@ { "path": "../plugins/license_management/tsconfig.json" }, { "path": "../plugins/licensing/tsconfig.json" }, { "path": "../plugins/logstash/tsconfig.json" }, + { "path": "../plugins/metrics_entities/tsconfig.json" }, { "path": "../plugins/ml/tsconfig.json" }, { "path": "../plugins/monitoring/tsconfig.json" }, { "path": "../plugins/observability/tsconfig.json" },