diff --git a/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts b/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts index ab15212431401..a4cca4455a274 100644 --- a/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts +++ b/x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts @@ -30,6 +30,7 @@ describe('Fleet - packageToPackagePolicy', () => { index_pattern: [], map: [], lens: [], + ml_module: [], }, elasticsearch: { ingest_pipeline: [], diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 5ea997d217888..80fabd51613ae 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -50,6 +50,7 @@ export enum KibanaAssetType { indexPattern = 'index_pattern', map = 'map', lens = 'lens', + mlModule = 'ml_module', } /* @@ -62,6 +63,7 @@ export enum KibanaSavedObjectType { indexPattern = 'index-pattern', map = 'map', lens = 'lens', + mlModule = 'ml-module', } export enum ElasticsearchAssetType { diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/epm/constants.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/epm/constants.tsx index de7e16e1e5d2b..ea19a330adfee 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/epm/constants.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/epm/constants.tsx @@ -33,6 +33,7 @@ export const AssetTitleMap: Record = { map: 'Map', data_stream_ilm_policy: 'Data Stream ILM Policy', lens: 'Lens', + ml_module: 'ML Module', }; export const ServiceTitleMap: Record = { @@ -47,6 +48,7 @@ export const AssetIcons: Record = { visualization: 'visualizeApp', map: 'emsApp', lens: 'lensApp', + ml_module: 'mlApp', }; export const ServiceIcons: Record = { diff --git a/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts b/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts index 4196138a2534f..bfcc40e18fe80 100644 --- a/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts @@ -37,6 +37,7 @@ const KibanaSavedObjectTypeMapping: Record { diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts index 8d8dad7b24d83..abc91a973e6b6 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts @@ -394,6 +394,11 @@ const expectAssetsInstalled = ({ id: 'sample_lens', }); expect(resLens.id).equal('sample_lens'); + const resMlModule = await kibanaServer.savedObjects.get({ + type: 'ml-module', + id: 'sample_ml_module', + }); + expect(resMlModule.id).equal('sample_ml_module'); const resIndexPattern = await kibanaServer.savedObjects.get({ type: 'index-pattern', id: 'test-*', @@ -459,6 +464,10 @@ const expectAssetsInstalled = ({ id: 'sample_lens', type: 'lens', }, + { + id: 'sample_ml_module', + type: 'ml-module', + }, { id: 'sample_search', type: 'search', @@ -526,6 +535,7 @@ const expectAssetsInstalled = ({ { id: '47758dc2-979d-5fbe-a2bd-9eded68a5a43', type: 'epm-packages-assets' }, { id: '318959c9-997b-5a14-b328-9fc7355b4b74', type: 'epm-packages-assets' }, { id: 'e21b59b5-eb76-5ab0-bef2-1c8e379e6197', type: 'epm-packages-assets' }, + { id: '4c758d70-ecf1-56b3-b704-6d8374841b34', type: 'epm-packages-assets' }, { id: 'e786cbd9-0f3b-5a0b-82a6-db25145ebf58', type: 'epm-packages-assets' }, { id: '53c94591-aa33-591d-8200-cd524c2a0561', type: 'epm-packages-assets' }, { id: 'b658d2d4-752e-54b8-afc2-4c76155c1466', type: 'epm-packages-assets' }, diff --git a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts index 2950436fab416..1a559ac5a5c75 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts @@ -296,6 +296,10 @@ export default function (providerContext: FtrProviderContext) { id: 'sample_lens', type: 'lens', }, + { + id: 'sample_ml_module', + type: 'ml-module', + }, ], installed_es: [ { @@ -344,6 +348,7 @@ export default function (providerContext: FtrProviderContext) { { id: '48e582df-b1d2-5f88-b6ea-ba1fafd3a569', type: 'epm-packages-assets' }, { id: 'bf3b0b65-9fdc-53c6-a9ca-e76140e56490', type: 'epm-packages-assets' }, { id: '7f4c5aca-b4f5-5f0a-95af-051da37513fc', type: 'epm-packages-assets' }, + { id: '4281a436-45a8-54ab-9724-fda6849f789d', type: 'epm-packages-assets' }, { id: '2e56f08b-1d06-55ed-abee-4708e1ccf0aa', type: 'epm-packages-assets' }, { id: 'c7bf1a39-e057-58a0-afde-fb4b48751d8c', type: 'epm-packages-assets' }, { id: '8c665f28-a439-5f43-b5fd-8fda7b576735', type: 'epm-packages-assets' }, diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/all_assets/0.1.0/kibana/ml_module/sample_ml_module.json b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/all_assets/0.1.0/kibana/ml_module/sample_ml_module.json new file mode 100644 index 0000000000000..ff7e3478d03e3 --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/all_assets/0.1.0/kibana/ml_module/sample_ml_module.json @@ -0,0 +1,403 @@ +{ + "attributes": { + "id": "sample_ml_module", + "title": "Nginx access logs", + "description": "Find unusual activity in HTTP access logs from filebeat (ECS).", + "type": "Web Access Logs", + "logo": { + "icon": "logoNginx" + }, + "defaultIndexPattern": "filebeat-*", + "query": { + "bool": { + "filter": [ + { + "term": { + "event.dataset": "nginx.access" + } + }, + { + "exists": { + "field": "source.address" + } + }, + { + "exists": { + "field": "url.original" + } + }, + { + "exists": { + "field": "http.response.status_code" + } + } + ] + } + }, + "jobs": [ + { + "id": "visitor_rate_ecs", + "config": { + "groups": [ + "nginx" + ], + "description": "HTTP Access Logs: Detect unusual visitor rates (ECS)", + "analysis_config": { + "bucket_span": "15m", + "summary_count_field_name": "dc_source_address", + "detectors": [ + { + "detector_description": "Nginx access visitor rate", + "function": "non_zero_count" + } + ], + "influencers": [] + }, + "analysis_limits": { + "model_memory_limit": "10mb" + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "model_plot_config": { + "enabled": true + }, + "custom_settings": { + "created_by": "ml-module-nginx-access", + "custom_urls": [ + { + "url_name": "Raw data", + "url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(event.dataset:(query:\u0027nginx.access\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } + } + }, + { + "id": "status_code_rate_ecs", + "config": { + "groups": [ + "nginx" + ], + "description": "HTTP Access Logs: Detect unusual status code rates (ECS)", + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "Nginx access status code rate", + "function": "count", + "partition_field_name": "http.response.status_code" + } + ], + "influencers": [ + "http.response.status_code", + "source.address" + ] + }, + "analysis_limits": { + "model_memory_limit": "100mb" + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "model_plot_config": { + "enabled": true + }, + "custom_settings": { + "created_by": "ml-module-nginx-access", + "custom_urls": [ + { + "url_name": "Investigate status code", + "url_value": "dashboards#/view/ml_http_access_explorer_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(event.dataset:(query:\u0027nginx.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:http.response.status_code,negate:!f,params:(query:\u0027$http.response.status_code$\u0027),type:phrase,value:\u0027$http.response.status_code$\u0027),query:(match:(http.response.status_code:(query:\u0027$http.response.status_code$\u0027,type:phrase))))),query:(language:kuery,query:\u0027\u0027))" + }, + { + "url_name": "Raw data", + "url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(event.dataset:(query:\u0027nginx.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:http.response.status_code,negate:!f,params:(query:\u0027$http.response.status_code$\u0027),type:phrase,value:\u0027$http.response.status_code$\u0027),query:(match:(http.response.status_code:(query:\u0027$http.response.status_code$\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } + } + }, + { + "id": "source_ip_url_count_ecs", + "config": { + "groups": [ + "nginx" + ], + "description": "HTTP Access Logs: Detect unusual source IPs - high distinct count of URLs (ECS)", + "analysis_config": { + "bucket_span": "1h", + "detectors": [ + { + "detector_description": "Nginx access source IP high dc URL", + "function": "high_distinct_count", + "field_name": "url.original", + "over_field_name": "source.address" + } + ], + "influencers": [ + "source.address" + ] + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "custom_settings": { + "created_by": "ml-module-nginx-access", + "custom_urls": [ + { + "url_name": "Investigate source IP", + "url_value": "dashboards#/view/ml_http_access_explorer_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(event.dataset:(query:\u0027nginx.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,params:(query:\u0027$source.address$\u0027),type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),query:(language:kuery,query:\u0027\u0027))" + }, + { + "url_name": "Raw data", + "url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(event.dataset:(query:\u0027nginx.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,params:(query:\u0027$source.address$\u0027),type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } + } + }, + { + "id": "source_ip_request_rate_ecs", + "config": { + "groups": [ + "nginx" + ], + "description": "HTTP Access Logs: Detect unusual source IPs - high request rates (ECS)", + "analysis_config": { + "bucket_span": "1h", + "detectors": [ + { + "detector_description": "Nginx access source IP high count", + "function": "high_count", + "over_field_name": "source.address" + } + ], + "influencers": [ + "source.address" + ] + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "custom_settings": { + "created_by": "ml-module-nginx-access", + "custom_urls": [ + { + "url_name": "Investigate source IP", + "url_value": "dashboards#/view/ml_http_access_explorer_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(event.dataset:(query:\u0027nginx.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,params:(query:\u0027$source.address$\u0027),type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),query:(language:kuery,query:\u0027\u0027))" + }, + { + "url_name": "Raw data", + "url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(event.dataset:(query:\u0027nginx.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,params:(query:\u0027$source.address$\u0027),type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } + } + }, + { + "id": "low_request_rate_ecs", + "config": { + "groups": [ + "nginx" + ], + "description": "HTTP Access Logs: Detect low request rates (ECS)", + "analysis_config": { + "bucket_span": "15m", + "summary_count_field_name": "doc_count", + "detectors": [ + { + "detector_description": "Nginx access low request rate", + "function": "low_count" + } + ], + "influencers": [] + }, + "analysis_limits": { + "model_memory_limit": "10mb" + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "model_plot_config": { + "enabled": true + }, + "custom_settings": { + "created_by": "ml-module-nginx-access", + "custom_urls": [ + { + "url_name": "Raw data", + "url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(event.dataset:(query:\u0027nginx.access\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } + } + } + ], + "datafeeds": [ + { + "id": "datafeed-visitor_rate_ecs", + "job_id": "visitor_rate_ecs", + "config": { + "job_id": "visitor_rate_ecs", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { + "term": { + "event.dataset": "nginx.access" + } + } + ] + } + }, + "aggregations": { + "buckets": { + "date_histogram": { + "field": "@timestamp", + "fixed_interval": "15m", + "offset": 0, + "order": { + "_key": "asc" + }, + "keyed": false, + "min_doc_count": 0 + }, + "aggregations": { + "@timestamp": { + "max": { + "field": "@timestamp" + } + }, + "dc_source_address": { + "cardinality": { + "field": "source.address" + } + } + } + } + } + } + }, + { + "id": "datafeed-status_code_rate_ecs", + "job_id": "status_code_rate_ecs", + "config": { + "job_id": "status_code_rate_ecs", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { + "term": { + "event.dataset": "nginx.access" + } + } + ] + } + } + } + }, + { + "id": "datafeed-source_ip_url_count_ecs", + "job_id": "source_ip_url_count_ecs", + "config": { + "job_id": "source_ip_url_count_ecs", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { + "term": { + "event.dataset": "nginx.access" + } + } + ] + } + } + } + }, + { + "id": "datafeed-source_ip_request_rate_ecs", + "job_id": "source_ip_request_rate_ecs", + "config": { + "job_id": "source_ip_request_rate_ecs", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { + "term": { + "event.dataset": "nginx.access" + } + } + ] + } + } + } + }, + { + "id": "datafeed-low_request_rate_ecs", + "job_id": "low_request_rate_ecs", + "config": { + "job_id": "low_request_rate_ecs", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { + "term": { + "event.dataset": "nginx.access" + } + } + ] + } + }, + "aggregations": { + "buckets": { + "date_histogram": { + "field": "@timestamp", + "fixed_interval": "15m", + "offset": 0, + "order": { + "_key": "asc" + }, + "keyed": false, + "min_doc_count": 0 + }, + "aggregations": { + "@timestamp": { + "max": { + "field": "@timestamp" + } + } + } + } + } + } + } + ] + }, + "id": "sample_ml_module", + "migrationVersion": { + "search": "7.9.3" + }, + "references": [], + "type": "ml-module" +} diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/all_assets/0.2.0/kibana/ml_module/sample_ml_module.json b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/all_assets/0.2.0/kibana/ml_module/sample_ml_module.json new file mode 100644 index 0000000000000..ff7e3478d03e3 --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/all_assets/0.2.0/kibana/ml_module/sample_ml_module.json @@ -0,0 +1,403 @@ +{ + "attributes": { + "id": "sample_ml_module", + "title": "Nginx access logs", + "description": "Find unusual activity in HTTP access logs from filebeat (ECS).", + "type": "Web Access Logs", + "logo": { + "icon": "logoNginx" + }, + "defaultIndexPattern": "filebeat-*", + "query": { + "bool": { + "filter": [ + { + "term": { + "event.dataset": "nginx.access" + } + }, + { + "exists": { + "field": "source.address" + } + }, + { + "exists": { + "field": "url.original" + } + }, + { + "exists": { + "field": "http.response.status_code" + } + } + ] + } + }, + "jobs": [ + { + "id": "visitor_rate_ecs", + "config": { + "groups": [ + "nginx" + ], + "description": "HTTP Access Logs: Detect unusual visitor rates (ECS)", + "analysis_config": { + "bucket_span": "15m", + "summary_count_field_name": "dc_source_address", + "detectors": [ + { + "detector_description": "Nginx access visitor rate", + "function": "non_zero_count" + } + ], + "influencers": [] + }, + "analysis_limits": { + "model_memory_limit": "10mb" + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "model_plot_config": { + "enabled": true + }, + "custom_settings": { + "created_by": "ml-module-nginx-access", + "custom_urls": [ + { + "url_name": "Raw data", + "url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(event.dataset:(query:\u0027nginx.access\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } + } + }, + { + "id": "status_code_rate_ecs", + "config": { + "groups": [ + "nginx" + ], + "description": "HTTP Access Logs: Detect unusual status code rates (ECS)", + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "Nginx access status code rate", + "function": "count", + "partition_field_name": "http.response.status_code" + } + ], + "influencers": [ + "http.response.status_code", + "source.address" + ] + }, + "analysis_limits": { + "model_memory_limit": "100mb" + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "model_plot_config": { + "enabled": true + }, + "custom_settings": { + "created_by": "ml-module-nginx-access", + "custom_urls": [ + { + "url_name": "Investigate status code", + "url_value": "dashboards#/view/ml_http_access_explorer_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(event.dataset:(query:\u0027nginx.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:http.response.status_code,negate:!f,params:(query:\u0027$http.response.status_code$\u0027),type:phrase,value:\u0027$http.response.status_code$\u0027),query:(match:(http.response.status_code:(query:\u0027$http.response.status_code$\u0027,type:phrase))))),query:(language:kuery,query:\u0027\u0027))" + }, + { + "url_name": "Raw data", + "url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(event.dataset:(query:\u0027nginx.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:http.response.status_code,negate:!f,params:(query:\u0027$http.response.status_code$\u0027),type:phrase,value:\u0027$http.response.status_code$\u0027),query:(match:(http.response.status_code:(query:\u0027$http.response.status_code$\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } + } + }, + { + "id": "source_ip_url_count_ecs", + "config": { + "groups": [ + "nginx" + ], + "description": "HTTP Access Logs: Detect unusual source IPs - high distinct count of URLs (ECS)", + "analysis_config": { + "bucket_span": "1h", + "detectors": [ + { + "detector_description": "Nginx access source IP high dc URL", + "function": "high_distinct_count", + "field_name": "url.original", + "over_field_name": "source.address" + } + ], + "influencers": [ + "source.address" + ] + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "custom_settings": { + "created_by": "ml-module-nginx-access", + "custom_urls": [ + { + "url_name": "Investigate source IP", + "url_value": "dashboards#/view/ml_http_access_explorer_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(event.dataset:(query:\u0027nginx.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,params:(query:\u0027$source.address$\u0027),type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),query:(language:kuery,query:\u0027\u0027))" + }, + { + "url_name": "Raw data", + "url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(event.dataset:(query:\u0027nginx.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,params:(query:\u0027$source.address$\u0027),type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } + } + }, + { + "id": "source_ip_request_rate_ecs", + "config": { + "groups": [ + "nginx" + ], + "description": "HTTP Access Logs: Detect unusual source IPs - high request rates (ECS)", + "analysis_config": { + "bucket_span": "1h", + "detectors": [ + { + "detector_description": "Nginx access source IP high count", + "function": "high_count", + "over_field_name": "source.address" + } + ], + "influencers": [ + "source.address" + ] + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "custom_settings": { + "created_by": "ml-module-nginx-access", + "custom_urls": [ + { + "url_name": "Investigate source IP", + "url_value": "dashboards#/view/ml_http_access_explorer_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(event.dataset:(query:\u0027nginx.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,params:(query:\u0027$source.address$\u0027),type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),query:(language:kuery,query:\u0027\u0027))" + }, + { + "url_name": "Raw data", + "url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(event.dataset:(query:\u0027nginx.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,params:(query:\u0027$source.address$\u0027),type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } + } + }, + { + "id": "low_request_rate_ecs", + "config": { + "groups": [ + "nginx" + ], + "description": "HTTP Access Logs: Detect low request rates (ECS)", + "analysis_config": { + "bucket_span": "15m", + "summary_count_field_name": "doc_count", + "detectors": [ + { + "detector_description": "Nginx access low request rate", + "function": "low_count" + } + ], + "influencers": [] + }, + "analysis_limits": { + "model_memory_limit": "10mb" + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "model_plot_config": { + "enabled": true + }, + "custom_settings": { + "created_by": "ml-module-nginx-access", + "custom_urls": [ + { + "url_name": "Raw data", + "url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(event.dataset:(query:\u0027nginx.access\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } + } + } + ], + "datafeeds": [ + { + "id": "datafeed-visitor_rate_ecs", + "job_id": "visitor_rate_ecs", + "config": { + "job_id": "visitor_rate_ecs", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { + "term": { + "event.dataset": "nginx.access" + } + } + ] + } + }, + "aggregations": { + "buckets": { + "date_histogram": { + "field": "@timestamp", + "fixed_interval": "15m", + "offset": 0, + "order": { + "_key": "asc" + }, + "keyed": false, + "min_doc_count": 0 + }, + "aggregations": { + "@timestamp": { + "max": { + "field": "@timestamp" + } + }, + "dc_source_address": { + "cardinality": { + "field": "source.address" + } + } + } + } + } + } + }, + { + "id": "datafeed-status_code_rate_ecs", + "job_id": "status_code_rate_ecs", + "config": { + "job_id": "status_code_rate_ecs", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { + "term": { + "event.dataset": "nginx.access" + } + } + ] + } + } + } + }, + { + "id": "datafeed-source_ip_url_count_ecs", + "job_id": "source_ip_url_count_ecs", + "config": { + "job_id": "source_ip_url_count_ecs", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { + "term": { + "event.dataset": "nginx.access" + } + } + ] + } + } + } + }, + { + "id": "datafeed-source_ip_request_rate_ecs", + "job_id": "source_ip_request_rate_ecs", + "config": { + "job_id": "source_ip_request_rate_ecs", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { + "term": { + "event.dataset": "nginx.access" + } + } + ] + } + } + } + }, + { + "id": "datafeed-low_request_rate_ecs", + "job_id": "low_request_rate_ecs", + "config": { + "job_id": "low_request_rate_ecs", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { + "term": { + "event.dataset": "nginx.access" + } + } + ] + } + }, + "aggregations": { + "buckets": { + "date_histogram": { + "field": "@timestamp", + "fixed_interval": "15m", + "offset": 0, + "order": { + "_key": "asc" + }, + "keyed": false, + "min_doc_count": 0 + }, + "aggregations": { + "@timestamp": { + "max": { + "field": "@timestamp" + } + } + } + } + } + } + } + ] + }, + "id": "sample_ml_module", + "migrationVersion": { + "search": "7.9.3" + }, + "references": [], + "type": "ml-module" +}